diff --git a/.gitignore b/.gitignore index fb5a710..c3d916e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,10 @@ dist/ # Never commit either - the share url grants read access to the artifact. handoff-bundle/ SHARE-URL.txt + +# deploy/hub.env holds one deployment's real addresses: mesh IPs, the control +# plane URL, SSH fallbacks. Copy deploy/hub.env.example and fill it in locally. +deploy/hub.env + +# npm pack output: the install artifact deploy/install-hub.sh consumes. +*.tgz diff --git a/AGENTS.md b/AGENTS.md index fe3d2d4..8d76838 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,8 @@ node dist/cli.js run examples/hello.yaml # smoke test, zero cost - **Tests never spawn a real agent CLI.** No test may execute `claude`, `codex`, `opencode`, `enclave`, or `git`, and no test may make a network request. Adapter tests parse fixture strings; engine tests inject stub adapters through the registry argument; `src/handoff/` routes every spawn - including `opencode export` and `enclave push` - through an injected `Exec` seam that tests replace with a fake. - **Never invent cost numbers.** If a CLI does not report a price, record `0`. Do not derive cost from a token count and a price table anywhere in this codebase. - **Checkpoint after every edge crossing.** Not at the end of a batch, not at the end of the run. `CheckpointStore.save` writes a temp file and renames it; never write `state.json` in place. -- **The event log is append-only and unbuffered.** A killed process must leave a readable JSONL log. Do not add buffering or rewrite past lines. +- **The event log is append-only and unbuffered.** This rule governs `.loomgraph/runs//events.jsonl` on the machine that ran the graph. A killed process must leave a readable JSONL log. Do not add buffering or rewrite past lines. +- **The hub's truth is SQLite.** `lg-hub export --jsonl` is a derived, rebuildable artifact. Never make a JSONL file on the hub authoritative, never make the laptop's log a database. - **Graph validation stays loud.** Unknown node ids, cycles, missing budgets and unknown adapters throw with the offending node id in the message. Do not downgrade a validation error to a warning. - **No LLM SDK dependency.** The agent CLIs are the runtime. Adding an API client to `dependencies` is out of scope for this project. - **Adapter output is a contract.** Every adapter returns `{ ok, text, costUsd, raw, error }`. Cost is recorded even when the run failed, because budget accounting depends on it. @@ -36,13 +37,20 @@ src/core/ types, store, events, graph, budget, engine (no CLI concerns) src/adapters/ one file per executor, plus the registry src/commands/ CLI command implementations and pure renderers src/handoff/ the `lg-handoff` bin: session readers, secret scanner, brief renderer +src/hub/ the `lg-hub` bin: HTTP API and SQLite database +src/team/ client-side code for the team hub examples/ graph files that must stay valid (`lg validate`) ``` Nothing under `src/handoff/` may import from `src/core/` or `src/adapters/`. The subtree -owns its own enclave helpers so it stays extractable into a sibling package with a `git mv`. -That is why `buildEnclavePushArgs` exists twice and neither copy should be deduplicated -into a shared module. +owns its own enclave helpers, so it stays extractable into a sibling package with a `git mv`; +once extracted, the hub depends on that sibling package, not on this repo. That is why +`buildEnclavePushArgs` exists twice and neither copy should be deduplicated into a shared +module. + +`src/hub/` and `src/team/` may import from `src/core/` and `src/handoff/scan.ts`; the arrow +never reverses. Do not fork the scanner - the `buildEnclavePushArgs`-exists-twice precedent +covers argv builders, not security gates. Inside `src/handoff/`, one file per job, and the data flows one way: diff --git a/README.md b/README.md index fc94908..f82dc78 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,15 @@ It does not call a model itself. Your agent CLIs are the runtime. ## Install +Not published to npm. Build it from a clone and link the binaries: + ```bash -npm i -g loomgraph +git clone https://github.com/datj9/loomgraph && cd loomgraph +npm install && npm run build +npm link # or: npm pack && npm i -g ./loomgraph-0.1.0.tgz ``` -Requires Node >= 22. The binary is `lg`. +Requires Node >= 22. Three binaries land on your PATH: `lg`, `lg-handoff` and `lg-hub`. ## 60-second quickstart @@ -486,18 +490,159 @@ always means "looked at and found nothing". headings. There is no model call, so `pack` works offline and cannot invent a claim. - **`private` visibility only.** A transcript is production data. `--visibility org` and `public` are refused. -- **No signal bus, inbox, or daemon.** That would contradict "Not a workflow server" - below, and an inbox that starts an agent on someone else's laptop is a different product - with a much harder threat model. +- **No inbox.** An inbox that starts an agent on someone else's laptop is a different + product with a much harder threat model, and phase 1 ships no inbox - that is phase 3. + There is a daemon now, and the Hub section below is precise about what it does. Known rough edge: the `enclave share create --json` parser accepts several plausible field names because that stdout shape has not yet been captured from a real invocation. +## The hub + +The third binary is `lg-hub`: an HTTP API in front of one SQLite database. It +stores what members push and serves reads out of that store - it never runs an agent. +Agents run on the member's own machine, started by the member; the hub has no way to +start one, and that absence is the design. A daemon that can only store and route is a +store, not a scheduler. + +### Set one up + +On the hub host: + +```bash +lg-hub init # create the data dir and hub.db +lg-hub member add alice # prints alice's token once - write it down +lg-hub serve # binds 127.0.0.1:8369, web UI on the same origin +``` + +That database runs in WAL mode, so the hub's state on disk is **three** files, not +one: `hub.db`, `hub.db-wal` and `hub.db-shm`. Copying `hub.db` alone while the +server is running gives you a backup missing every committed write still in the +WAL. Use `VACUUM INTO` (or stop the service first). + +Onboarding a colleague is two grants, not one — a NetBird peer in the members +group, and a hub token. `deploy/enroll-member.sh` does the first, +`lg-hub member add` the second, and +[docs/hub-operations.md](docs/hub-operations.md) is the operator runbook for +both. Hand the new member [docs/member-quickstart.md](docs/member-quickstart.md) +(the commands) and [docs/hub-onboarding.md](docs/hub-onboarding.md) (what +syncing shares). + +On a member machine: + +```bash +lg enroll http://10.0.0.5:8369 # identity lives in ~/.config/loomgraph/hub.json +lg sync --enable # opt this repo in: .loomgraph/hub.json +lg run examples/hello.yaml +lg sync # push one run +lg sync --all # or every run under .loomgraph/runs/ +``` + +`lg sync --enable` is a hard gate, not a hint: without `.loomgraph/hub.json` in +the repo, `lg sync ` and `lg sync --all` both refuse and push nothing. The +hub cannot delete an event once ingested, so opting in has to be a deliberate act +per repo rather than something a forgotten flag decides for you. + +The rest of the hub-facing surface: `lg-hub member revoke ` and +`lg-hub member ls` for the roster, `lg-hub export --jsonl` to print the raw stored +lines to stdout for grepping, and `lg-hub export --out ` to write one +`runs///events.jsonl` per run. + +### What the hub receives + +A sync pushes two things, and **both** are filtered on the way out: + +**The run state projection.** Built field by field, never as a filtered copy of the +full state, so there is no field a `vars` value or a node `output` could ride in on: + +- `vars` reach the hub as key names only. +- node `output` never reaches the hub. +- node `error` is control-stripped, path-rewritten, secret-masked, and capped at 200 + characters. + +**The run's event lines.** These are *not* pushed verbatim. Each line is rebuilt +against a per-kind allowlist of `data` fields before it leaves the machine; a field +not on the list is dropped, and a line whose `kind` the allowlist does not know is +dropped whole. Fields carrying operator or environment text - `node_finished.error` +and `run_finished.error` (an adapter folds the agent's full result text and stderr +into these), `run_started.cwd`, the interpolated `human_requested.question`, and +`human_resolved.answer` - go through the same strip/rewrite/mask/cap as a node error. + +The filtering happens at **push** time, not at emission. Your local +`.loomgraph/runs//events.jsonl` keeps its raw values for debugging; only the +copy crossing to the hub is sanitised. `lg-hub export` still reproduces the *ingested* +lines byte for byte - those lines simply no longer carry secrets. + +Same error before and after: + +``` +in : command exited with code 1: /home/alice/work/repo/run.sh: token sk-ant-api03-… rejected +out: command exited with code 1: ${REPO_ROOT}/run.sh: token sk-a... rejected +``` + +### Tokens are possession-equals-identity + +Anyone holding a member token is that member to the hub. `member add` prints a token +once and the store keeps only its hash, so it can never be printed again - treat it +like an SSH key, and `member revoke ` is the off switch. + +### `serve` refuses a non-loopback bind + +`lg-hub serve` binds `127.0.0.1` by default and refuses any other host unless +`--behind-tls-proxy` is passed. A bearer token over plaintext non-loopback HTTP is +exactly the credential shape this project's own scanner has a rule for, so a bind that +would put the token on the wire without TLS is an error rather than an option. + +### The web UI ships, and it is on by default + +`lg-hub serve` serves a self-contained web UI on the same origin as the API - one +embedded HTML document, no build step, no external requests. It renders runs, the +activity feed and the member roster using `textContent` only, so an untrusted +transcript cannot inject markup. Pass `--no-ui` for an API-only bind. + +It authenticates with a bearer token you paste, and **keeps that token in +`localStorage`**. On the default loopback bind that is fine. Behind +`--behind-tls-proxy` on a plaintext `http://` origin it is not: the token sits in +browser storage on an origin anyone on that network can impersonate. Terminate TLS in +front of it, or run `--no-ui`. + +### Two caveats, stated up front + +**Phase 1 does not mask on egress.** The projection is the only gate; whatever does +reach the hub is served back as stored. A repository whose var *key names* are +themselves sensitive should keep sync disabled - redaction-on-read is phase 2. + +**The masking is an allowlist, not a proof.** Error masking reuses `lg-handoff`'s +`SCAN_RULES`, so it catches the shapes those rules know and nothing else. During +development a test canary shaped `AKIA` plus 18 more characters passed through +unmasked, because the rule is `\bAKIA[0-9A-Z]{16}\b` - exactly 20 characters. The +canary was malformed rather than the rule being wrong, but that is exactly the point: a +shape the rules do not cover reaches a team-readable field unmasked. + +### Verified against a live hub + +Both of these were run end to end against the built binary with a live hub: + +- **The export is byte-identical to the local log.** `lg-hub export` reproduces the + ingested lines exactly; it does not re-encode them. +- **A dead hub changes neither the exit code nor the node outcomes.** The hub was + killed mid-exercise; the run was unaffected. + +### What phase 1 does not ship + +- No inbox - that is phase 3. +- No briefs on the hub, no encryption at rest, and no redaction on read - all phase 2. + Nothing in phase 1 is encrypted. +- No full transcripts, ever. The handoff refusal stands unchanged: a transcript is a + credential dump, and syncing to the hub does not soften that. + ## What this is not - **Not a model, and not an SDK for one.** loomgraph makes zero API calls of its own and has no LLM SDK dependency. - **Not a replacement for your agent CLI.** It shells out to the CLI you already installed and authenticated. -- **Not a workflow server.** No daemon, no web UI, no cloud, no plugin system in v0.1. +- **Not a workflow server.** A daemon ships in phase 1 - `lg-hub` - but it stores and + routes, and never runs an agent. It ships a web UI over its own store - reads, plus member add/revoke; no + cloud, and no plugin system in v0.1. `lg report --publish` does not change that: it writes a static file and shells out to the `enclave` cli the same way a node shells out to `claude`. If `enclave` is not installed the diff --git a/deploy/backup-hub.sh b/deploy/backup-hub.sh new file mode 100755 index 0000000..0c40f5a --- /dev/null +++ b/deploy/backup-hub.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash +# +# backup-hub.sh - WAL-safe, verified backup of the loomgraph hub database. +# +# hub.db runs PRAGMA journal_mode=WAL (src/hub/storage.ts), so it has -wal and +# -shm sidecars and a plain `cp hub.db` produces an INCONSISTENT snapshot: +# committed pages that still live only in the WAL are silently lost. This +# script uses `VACUUM INTO`, which takes a read transaction against the live +# database and writes a fully-checkpointed, self-contained copy. +# +# The store is also a hash chain - row_hash = sha256(prev_hash || json), with +# the head in chain_head (src/hub/storage.ts) - so a torn or partial copy can +# open cleanly and still be corrupt. Every copy is therefore verified before it +# is kept: +# (a) the copy opens, +# (b) PRAGMA integrity_check returns ok, +# (c) the hash chain verifies end to end, from the 32-zero-byte genesis to +# the value stored in chain_head. +# Any failure renames the copy to *.rejected and exits non-zero. +# +# This runs against a LIVE server, and storage.ts sets no busy_timeout on any +# connection, so a lock conflict surfaces immediately as SQLITE_BUSY. The +# snapshot therefore sets its own busy_timeout and retries with exponential +# backoff, and gives up with exit code 3 rather than emitting a partial copy. +# +# Every database access runs as the lghub service user: node:sqlite creates +# hub.db-wal / hub.db-shm on demand, and root-owned sidecars would lock the +# service out of its own store. +# +# Restore is a documented MANUAL procedure, deliberately not a flag here. +# +# Exit codes: 0 ok | 1 failure or failed verification | 2 usage | 3 could not +# acquire a lock (retryable; safe for cron to treat as "try again later"). +# +# Usage: sudo ./backup-hub.sh +# Overrides (env): LGHUB_DATA_DIR, LGHUB_BACKUP_DIR, LGHUB_RETAIN, LGHUB_USER, +# LGHUB_BUSY_TIMEOUT_MS, LGHUB_BUSY_RETRIES + +set -euo pipefail + +readonly DATA_DIR="${LGHUB_DATA_DIR:-/var/lib/lghub}" +readonly BACKUP_DIR="${LGHUB_BACKUP_DIR:-/var/backups/lghub}" +readonly SERVICE_USER="${LGHUB_USER:-lghub}" +readonly RETAIN="${LGHUB_RETAIN:-14}" +readonly DB_PATH="${DATA_DIR}/hub.db" +# storage.ts sets no busy_timeout anywhere, so a lock conflict with the running +# server surfaces instantly as SQLITE_BUSY. This script runs against a live +# server by design, so it sets its own timeout and retries with backoff. +readonly BUSY_TIMEOUT_MS="${LGHUB_BUSY_TIMEOUT_MS:-10000}" +readonly BUSY_RETRIES="${LGHUB_BUSY_RETRIES:-5}" +# Exit codes: 0 ok, 1 failed/verification failed, 2 usage, 3 could not acquire +# a lock (retryable - safe for a cron job to treat as "try again later"). +readonly EXIT_BUSY=3 + +log() { printf 'backup-hub: %s\n' "$*"; } +die() { printf 'backup-hub: FATAL %s\n' "$*" >&2; exit 1; } + +WORK_DIR="" +VERIFIER_PATH="" +cleanup() { + if [ -n "$WORK_DIR" ] && [ -d "$WORK_DIR" ]; then + rm -rf -- "$WORK_DIR" + fi +} +trap cleanup EXIT + +# Run a command as the hub service user so any SQLite sidecar files touched +# during the read stay lghub-owned. Running the read as root can leave a +# root-owned -wal/-shm behind and lock the service out of its own database. +run_as_hub() { + if [ "$(id -un)" = "$SERVICE_USER" ]; then + "$@" + elif [ "$(id -u)" -eq 0 ]; then + runuser -u "$SERVICE_USER" -- "$@" + else + die "must run as root or as ${SERVICE_USER} (current user: $(id -un))" + fi +} + +preflight() { + local cmd + for cmd in node find sort; do + command -v "$cmd" >/dev/null 2>&1 || die "missing required command: ${cmd}" + done + if [ "$(id -un)" != "$SERVICE_USER" ] && [ "$(id -u)" -ne 0 ]; then + die "must run as root or as ${SERVICE_USER} (current user: $(id -un))" + fi + [ -f "$DB_PATH" ] || die "database not found: ${DB_PATH}" + if ! [[ "$RETAIN" =~ ^[0-9]+$ ]] || [ "$RETAIN" -lt 1 ]; then + die "LGHUB_RETAIN must be a positive integer, got: ${RETAIN}" + fi + if ! [[ "$BUSY_RETRIES" =~ ^[0-9]+$ ]] || [ "$BUSY_RETRIES" -lt 1 ]; then + die "LGHUB_BUSY_RETRIES must be a positive integer, got: ${BUSY_RETRIES}" + fi + if ! [[ "$BUSY_TIMEOUT_MS" =~ ^[0-9]+$ ]]; then + die "LGHUB_BUSY_TIMEOUT_MS must be a non-negative integer, got: ${BUSY_TIMEOUT_MS}" + fi + if [ ! -d "$BACKUP_DIR" ]; then + if [ "$(id -u)" -eq 0 ]; then + install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$BACKUP_DIR" + log "created ${BACKUP_DIR} (0750 ${SERVICE_USER}:${SERVICE_USER})" + else + die "backup directory does not exist and cannot be created as a non-root user: ${BACKUP_DIR}" + fi + fi +} + +# Sets WORK_DIR and VERIFIER_PATH. Must NOT be called in a command +# substitution: the subshell would discard WORK_DIR and leak the temp dir past +# the EXIT trap. +write_verifier() { + WORK_DIR="$(mktemp -d)" + # World-readable so the unprivileged service user can read the script when + # this runs under runuser. It contains no secrets. + chmod 0755 "$WORK_DIR" + VERIFIER_PATH="${WORK_DIR}/vacuum-and-verify.mjs" + cat >"$VERIFIER_PATH" <<'VERIFIER_EOF' +// VACUUM INTO a live WAL database, then verify the copy: it opens, passes +// PRAGMA integrity_check, and its hash chain is continuous end to end. +// +// Chain construction copied from src/hub/storage.ts: +// genesis = 32 zero bytes (chain_head seed) +// row_hash = sha256(prev_hash || json) json is the client line, utf8, verbatim +// chain_head.head = row_hash of the most recently inserted event +// Insertion order is rowid order: `events` is an ordinary rowid table and +// triggers forbid UPDATE and DELETE, so rowids are append-only. +import { DatabaseSync } from "node:sqlite"; +import { createHash } from "node:crypto"; +import { rmSync } from "node:fs"; + +const [srcPath, dstPath, busyTimeoutMsArg, retriesArg] = process.argv.slice(2); +if (!srcPath || !dstPath) { + console.error("usage: vacuum-and-verify.mjs [busyTimeoutMs] [retries]"); + process.exit(2); +} +const busyTimeoutMs = Number(busyTimeoutMsArg ?? 10000); +const maxAttempts = Number(retriesArg ?? 5); + +function fail(message) { + console.error(`backup-hub: VERIFY FAILED: ${message}`); + process.exit(1); +} + +function toBuffer(value, what) { + if (value === null || value === undefined) fail(`${what} is NULL`); + return Buffer.from(value); +} + +/** + * src/hub/storage.ts sets no busy_timeout on any connection, so a lock + * conflict surfaces immediately as SQLITE_BUSY rather than waiting. This + * script runs against a live server by design, so it must expect that and + * retry rather than emit a partial or missing snapshot. + * + * Observed shape from node:sqlite: code ERR_SQLITE_ERROR, errcode 5, + * errstr "database is locked". 261 = SQLITE_BUSY_SNAPSHOT, 517 = + * SQLITE_BUSY_TIMEOUT. + */ +function isBusy(err) { + const code = err?.errcode; + if (code === 5 || code === 261 || code === 517) return true; + return /database is locked|database table is locked/i.test(String(err?.message ?? "")); +} + +/** Synchronous sleep: this script is deliberately straight-line. */ +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +// 1. Snapshot. VACUUM INTO reads the live database under a read transaction +// and writes a checkpointed, standalone copy - no -wal/-shm needed. +// It either completes or throws; a throw leaves a partial file behind, +// which is removed before the next attempt so no torn copy can survive. +let snapshotted = false; +for (let attempt = 1; attempt <= maxAttempts && !snapshotted; attempt += 1) { + let src; + try { + src = new DatabaseSync(srcPath, { readOnly: true }); + src.exec(`PRAGMA busy_timeout = ${Math.trunc(busyTimeoutMs)}`); + src.exec(`VACUUM INTO '${dstPath.replace(/'/g, "''")}'`); + snapshotted = true; + } catch (err) { + rmSync(dstPath, { force: true }); + if (!isBusy(err)) { + console.error( + `backup-hub: VACUUM INTO failed: ${err instanceof Error ? err.message : String(err)}`, + ); + process.exit(1); + } + if (attempt === maxAttempts) { + console.error( + `backup-hub: BUSY: could not acquire a read lock on ${srcPath} after ${maxAttempts} ` + + `attempt(s) with a ${busyTimeoutMs}ms busy_timeout each. No backup was produced. ` + + `Something is holding a long write lock - check the hub server and any manual ` + + `lg-hub command.`, + ); + process.exit(3); + } + const backoffMs = Math.min(1000 * 2 ** (attempt - 1), 16000); + console.error( + `backup-hub: SQLITE_BUSY on attempt ${attempt}/${maxAttempts}; retrying in ${backoffMs}ms`, + ); + sleepSync(backoffMs); + } finally { + if (src !== undefined) src.close(); + } +} + +// 2. The copy opens, and 3. integrity_check / chain continuity. +let dst; +try { + dst = new DatabaseSync(dstPath, { readOnly: true }); + + const integrity = dst.prepare("PRAGMA integrity_check").all(); + const verdict = integrity.map((r) => String(r.integrity_check)).join("; "); + if (integrity.length !== 1 || verdict !== "ok") { + fail(`PRAGMA integrity_check returned: ${verdict}`); + } + + const genesis = Buffer.alloc(32); + let expected = genesis; + let count = 0; + const rows = dst + .prepare("SELECT rowid AS rid, json, prev_hash, row_hash FROM events ORDER BY rowid") + .iterate(); + for (const row of rows) { + const prev = toBuffer(row.prev_hash, `events.prev_hash at rowid ${row.rid}`); + const stored = toBuffer(row.row_hash, `events.row_hash at rowid ${row.rid}`); + if (!prev.equals(expected)) { + fail( + `chain break at rowid ${row.rid}: stored prev_hash ${prev.toString("hex")} ` + + `!= previous row_hash ${expected.toString("hex")}`, + ); + } + const computed = createHash("sha256").update(expected).update(String(row.json), "utf8").digest(); + if (!computed.equals(stored)) { + fail( + `chain break at rowid ${row.rid}: stored row_hash ${stored.toString("hex")} ` + + `!= sha256(prev_hash || json) ${computed.toString("hex")}`, + ); + } + expected = computed; + count += 1; + } + + const headRow = dst.prepare("SELECT head FROM chain_head WHERE id=1").get(); + if (headRow === undefined) fail("chain_head has no row with id=1"); + const head = toBuffer(headRow.head, "chain_head.head"); + if (!head.equals(expected)) { + fail( + `chain_head ${head.toString("hex")} does not match the last row hash ` + + `${expected.toString("hex")} (${count} event(s) walked)`, + ); + } + + console.log( + `backup-hub: verified - integrity_check ok, ${count} event(s), chain head ${head.toString("hex")}`, + ); +} catch (err) { + console.error(`backup-hub: verification error: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); +} finally { + if (dst !== undefined) dst.close(); +} +VERIFIER_EOF + chmod 0644 "$VERIFIER_PATH" +} + +prune() { + local stale + # Names are UTC timestamps, so lexical sort is chronological. + stale="$(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'hub-*.db' -printf '%f\n' | + sort -r | tail -n "+$((RETAIN + 1))" || true)" + if [ -z "$stale" ]; then + log "retention: ${RETAIN} copies kept, nothing to prune" + return 0 + fi + local name + while IFS= read -r name; do + [ -n "$name" ] || continue + rm -f -- "${BACKUP_DIR}/${name}" + log "pruned ${name}" + done <<<"$stale" +} + +main() { + preflight + + local stamp out + write_verifier + stamp="$(date -u +%Y%m%dT%H%M%SZ)" + out="${BACKUP_DIR}/hub-${stamp}.db" + + if [ -e "$out" ]; then + die "backup target already exists: ${out}" + fi + + log "snapshotting ${DB_PATH} -> ${out}" + local rc=0 + run_as_hub node --disable-warning=ExperimentalWarning \ + "$VERIFIER_PATH" "$DB_PATH" "$out" "$BUSY_TIMEOUT_MS" "$BUSY_RETRIES" || rc=$? + if [ "$rc" -ne 0 ]; then + if [ "$rc" -eq "$EXIT_BUSY" ]; then + # The verifier already removed any partial file before giving up. + printf 'backup-hub: FATAL could not acquire a lock on %s; NO backup was produced. Retry later.\n' \ + "$DB_PATH" >&2 + exit "$EXIT_BUSY" + fi + if [ -e "$out" ]; then + mv -- "$out" "${out}.rejected" + die "backup verification failed; copy kept for inspection at ${out}.rejected (it is NOT a usable backup)" + fi + die "backup failed before a copy was produced" + fi + + chmod 0640 "$out" + log "backup complete: ${out} ($(du -h -- "$out" | cut -f1))" + prune +} + +main "$@" diff --git a/deploy/enroll-member.sh b/deploy/enroll-member.sh new file mode 100755 index 0000000..a17d725 --- /dev/null +++ b/deploy/enroll-member.sh @@ -0,0 +1,293 @@ +#!/usr/bin/env bash +# +# enroll-member.sh - the NetBird half of adding a colleague to the hub. +# +# A member needs TWO independent grants, and this script does only the first: +# +# 1. NETWORK - their peer is in the loomgraph-members group, so packets to +# the hub port are allowed by the ACL. THIS SCRIPT. +# 2. IDENTITY - a hub token, so the hub answers them. +# `lg-hub member add `, run on the hub host. +# +# Either alone is useless: a token without mesh access cannot reach the port, +# and mesh access without a token gets a 401. Revoking someone means undoing +# both - `lg-hub member revoke ` AND removing their peer from the group. +# +# Default mode is DRY-RUN. Nothing is written without --apply. +# +# deploy/enroll-member.sh --new alice dry-run a setup key +# deploy/enroll-member.sh --new alice --apply create it, print it once +# deploy/enroll-member.sh --peer alices-mbp --apply add an EXISTING peer +# deploy/enroll-member.sh --list show the group's members +# +# The management API token is read from NETBIRD_TOKEN, an `nbtoken` helper, or +# the macOS Keychain, and is never printed, logged, or passed on a command line. +# +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${LOOMGRAPH_HUB_ENV:-$SCRIPT_DIR/hub.env}" +if [ -f "$ENV_FILE" ]; then + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a +fi + +API_BASE="${NETBIRD_API:-}" +KEYCHAIN_SERVICE="${NETBIRD_KEYCHAIN_SERVICE:-netbird-pat}" +GROUP_MEMBERS="${LOOMGRAPH_MEMBERS_GROUP:-loomgraph-members}" +HUB_IP="${LOOMGRAPH_HUB_IP:-}" +HUB_PORT="${LOOMGRAPH_HUB_PORT:-8369}" + +# A setup key that never expires is a credential with no end date sitting in +# somebody's chat history. One-off plus a short window means a leaked key is +# useless by the time anyone finds it. +KEY_EXPIRY_SECONDS="${LOOMGRAPH_SETUP_KEY_EXPIRY:-86400}" + +MODE="dry-run" +ACTION="" +TARGET="" +TOKEN="" + +log() { printf '%s\n' "$*"; } +info() { printf ' %s\n' "$*"; } +die() { printf 'ERROR %s\n' "$*" >&2; exit 1; } +step() { printf '\n== %s\n' "$*"; } + +usage() { + cat <<'USAGE' +Usage: enroll-member.sh (--new | --peer | --list) [--apply] + + --new Create a one-off setup key that auto-joins the new peer to + the members group. Use for a colleague with no peer yet. + The key is printed ONCE and cannot be recovered. + --peer Add an EXISTING peer to the members group, by peer name, + hostname or mesh IP. Use when they are already on the mesh. + --list Print the members group's current peers. Read-only. + --apply Actually write. Without it, everything is a dry-run. + +Configuration comes from deploy/hub.env - see hub.env.example. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --new) ACTION="new"; TARGET="${2:-}"; [ -n "$TARGET" ] || die "--new needs a name"; shift ;; + --peer) ACTION="peer"; TARGET="${2:-}"; [ -n "$TARGET" ] || die "--peer needs a name or IP"; shift ;; + --list) ACTION="list" ;; + --apply) MODE="apply" ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; die "unknown argument: $1" ;; + esac + shift +done + +[ -n "$ACTION" ] || { usage >&2; die "pick one of --new, --peer or --list"; } +[ -n "$API_BASE" ] || die "NETBIRD_API is not set. Fill in $ENV_FILE (copy hub.env.example)." + +load_token() { + if [ -n "${NETBIRD_TOKEN:-}" ]; then + TOKEN="$NETBIRD_TOKEN" + elif command -v nbtoken >/dev/null 2>&1; then + TOKEN="$(nbtoken)" || die "the nbtoken helper failed; check it or set NETBIRD_TOKEN" + elif command -v security >/dev/null 2>&1; then + TOKEN="$(security find-generic-password -a "$USER" -s "$KEYCHAIN_SERVICE" -w 2>/dev/null)" || + die "no Keychain item for service \"$KEYCHAIN_SERVICE\" / account \"$USER\"; add it or set NETBIRD_TOKEN" + else + die "no token source: set NETBIRD_TOKEN, install nbtoken, or store the PAT in the Keychain" + fi + [ -n "$TOKEN" ] || die "the API token resolved to an empty string" +} + +# The token goes to curl through a --config file on STDIN. Never as -H on the +# command line: argv is world-readable in `ps` for the life of the process. +api() { + local method="$1" path="$2" body="${3:-}" + { + printf 'url = "%s%s"\n' "$API_BASE" "$path" + printf 'header = "Authorization: Token %s"\n' "$TOKEN" + printf 'header = "Content-Type: application/json"\n' + printf 'request = "%s"\n' "$method" + printf 'silent\nshow-error\nfail\n' + if [ -n "$body" ]; then + printf 'data-binary = "%s"\n' "@-" + fi + } > "$CURL_CONFIG" + + if [ -n "$body" ]; then + printf '%s' "$body" | curl --config "$CURL_CONFIG" || + die "$method $path failed" + else + curl --config "$CURL_CONFIG" < /dev/null || die "$method $path failed" + fi +} + +WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/enroll-member.XXXXXX")" +CURL_CONFIG="$WORK_DIR/curl.conf" +trap 'rm -rf "$WORK_DIR"' EXIT + +group_id() { + api GET /groups > "$WORK_DIR/groups.json" + GROUP_NAME="$GROUP_MEMBERS" python3 - "$WORK_DIR/groups.json" <<'PY' +import json, os, sys +want = os.environ["GROUP_NAME"] +for g in json.load(open(sys.argv[1])): + if g["name"] == want: + print(g["id"]); raise SystemExit(0) +raise SystemExit(1) +PY +} + +main() { + load_token + log "API: $API_BASE" + + local gid + gid="$(group_id)" || die "group \"$GROUP_MEMBERS\" does not exist - run netbird-acl.sh first" + + case "$ACTION" in + list) + step "peers in \"$GROUP_MEMBERS\" ($gid)" + api GET /peers > "$WORK_DIR/peers.json" + GROUP_ID="$gid" python3 - "$WORK_DIR/groups.json" "$WORK_DIR/peers.json" <<'PY' +import json, os, sys +gid = os.environ["GROUP_ID"] +groups = json.load(open(sys.argv[1])) +peers = {p["id"]: p for p in json.load(open(sys.argv[2]))} +members = next(g for g in groups if g["id"] == gid).get("peers") or [] +if not members: + print(" (none yet)") +for m in members: + pid = m["id"] if isinstance(m, dict) else m + p = peers.get(pid, {}) + print(" %-24s %-16s %s" % (p.get("name", pid), p.get("ip", "?"), + "connected" if p.get("connected") else "offline")) +PY + ;; + + new) + step "one-off setup key \"$TARGET\", auto-joining \"$GROUP_MEMBERS\"" + local body + body="$(GROUP_ID="$gid" NAME="$TARGET" EXPIRY="$KEY_EXPIRY_SECONDS" python3 -c ' +import json, os +print(json.dumps({ + "name": os.environ["NAME"], + "type": "one-off", + "expires_in": int(os.environ["EXPIRY"]), + "usage_limit": 1, + "ephemeral": False, + "auto_groups": [os.environ["GROUP_ID"]], +}))')" + if [ "$MODE" != "apply" ]; then + info "DRY-RUN POST ${API_BASE}/setup-keys" + info "body: $body" + info "re-run with --apply to create it" + return 0 + fi + api POST /setup-keys "$body" > "$WORK_DIR/key.json" + local key + key="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["key"])' "$WORK_DIR/key.json")" + print_member_instructions "$TARGET" "$key" + ;; + + peer) + step "add existing peer \"$TARGET\" to \"$GROUP_MEMBERS\"" + api GET /peers > "$WORK_DIR/peers.json" + local pid + pid="$(REF="$TARGET" python3 - "$WORK_DIR/peers.json" <<'PY' +import json, os, sys +ref = os.environ["REF"] +hits = [p for p in json.load(open(sys.argv[1])) + if ref in (p.get("name"), p.get("hostname"), p.get("ip"), p.get("id"))] +if len(hits) != 1: + raise SystemExit(1) +print(hits[0]["id"]) +PY +)" || die "no single peer matched \"$TARGET\" - check the name with: netbird-acl.sh --verify, or the NetBird console" + + # A group PUT REPLACES the peer list, so the existing members are read + # and sent back with the new one appended. Sending just the new id would + # silently evict everyone already in the group. + local body + body="$(GROUP_ID="$gid" NEW_PEER="$pid" NAME="$GROUP_MEMBERS" python3 - "$WORK_DIR/groups.json" <<'PY' +import json, os, sys +gid, new = os.environ["GROUP_ID"], os.environ["NEW_PEER"] +g = next(x for x in json.load(open(sys.argv[1])) if x["id"] == gid) +peers = [p["id"] if isinstance(p, dict) else p for p in (g.get("peers") or [])] +if new not in peers: + peers.append(new) +print(json.dumps({"name": os.environ["NAME"], "peers": peers})) +PY +)" + if [ "$MODE" != "apply" ]; then + info "DRY-RUN PUT ${API_BASE}/groups/${gid}" + info "body: $body" + info "re-run with --apply" + return 0 + fi + api PUT "/groups/${gid}" "$body" > /dev/null + info "peer \"$TARGET\" ($pid) is now in \"$GROUP_MEMBERS\"" + printf '\nThey still need a hub token. On the hub host:\n' + printf ' sudo -u lghub lg-hub member add --data-dir /var/lib/lghub\n' + ;; + esac +} + +print_member_instructions() { + local name="$1" key="$2" + cat < + +5. Opt in PER REPOSITORY. Nothing syncs until they do this, in each repo: + + lg sync --enable + + Have them read docs/hub-onboarding.md BEFORE this step, not after. What + reaches the hub is readable by every member and can never be deleted. + +---------------------------------------------------------------------------- + +Then, from THEIR machine, prove the ACL holds: + + deploy/enroll-member.sh --list + deploy/netbird-acl.sh --verify --from-member + +The second one is the only way to check the negative criteria: a member peer +must reach the hub on tcp/${HUB_PORT} and nothing else, including your MacBooks. +It proves nothing from the operator machine, which is in "personal macbooks" +and is supposed to reach everything. +EOF +} + +main diff --git a/deploy/hub.env.example b/deploy/hub.env.example new file mode 100644 index 0000000..f653505 --- /dev/null +++ b/deploy/hub.env.example @@ -0,0 +1,79 @@ +# deploy/hub.env - YOUR deployment's real values. COPY, DO NOT EDIT THIS FILE. +# +# cp deploy/hub.env.example deploy/hub.env # then fill it in +# +# This file is SOURCED by bash (`set -a; . hub.env`). Quote any value containing +# a space - `NAME=two words` makes the shell try to run `words` as a command. +# +# `deploy/hub.env` is gitignored on purpose. None of these are secrets in the +# credential sense - they are addresses - but a public repo does not need a map +# of your mesh, and the scripts refuse to run on someone else's defaults. +# +# Every script under deploy/ sources this file if it exists, then falls back to +# the environment. A variable marked REQUIRED aborts the script when unset, with +# a message naming it - that is deliberate: a deployment script that guesses an +# IP address converges the wrong network. + +# --- NetBird control plane ------------------------------------------------- + +# REQUIRED. Base URL of your NetBird management API, including /api. +# Self-hosted: https://netbird.example.com/api +# NetBird cloud: https://api.netbird.io/api +NETBIRD_API= + +# Keychain service name the token is stored under (macOS). The token itself is +# NEVER put in this file - netbird-acl.sh reads it from the Keychain or an +# `nbtoken` helper and passes it to curl on stdin. +NETBIRD_KEYCHAIN_SERVICE=netbird-pat + +# --- Peers ----------------------------------------------------------------- + +# REQUIRED. Mesh IP of the peer that runs lg-hub. +LOOMGRAPH_HUB_IP= + +# Display name for that peer in log output. Cosmetic only. +LOOMGRAPH_HUB_PEER_NAME="the hub peer" + +# Mesh IP of a peer the operator keeps SSH to besides the hub. Leave empty if +# you have none: the sandbox group and its policy are then skipped entirely. +LOOMGRAPH_SANDBOX_IP= + +# REQUIRED for `--verify --from-member`. Comma-separated mesh IPs that a member +# peer must NOT be able to reach. These are the negative acceptance criteria. +LOOMGRAPH_MAC_IPS= + +# --- Hub service ----------------------------------------------------------- + +LOOMGRAPH_HUB_PORT=8369 +LOOMGRAPH_HEALTH_PATH=/v1/health + +# Regex the /v1/health BODY must match. Leave empty until you have seen the real +# payload once - the built-in check is a heuristic, and a status code is not a +# health check here (the hub serves its UI for any non-/v1 GET, so /healthz +# returns 200 HTML even when the API is dead). +# +# PYTHON `re` SYNTAX, not POSIX ERE - the check runs through re.search. A POSIX +# class like [[:space:]] is not a character class there, it is a nested set, and +# it fails to match while only emitting a FutureWarning. Use \s, \d, \w. +# A real payload looks like {"ok":true,"version":"0.1.0"}, so: +# LOOMGRAPH_HEALTH_EXPECT='"ok"\s*:\s*true' +LOOMGRAPH_HEALTH_EXPECT= + +# --- Lockout fallbacks ----------------------------------------------------- + +# Optional but strongly recommended: a host:port that still reaches the hub +# WITHOUT the mesh, used in the warning printed before the All -> All policy is +# disabled. Empty means the warning says you have no fallback - which may be +# true, and you should know it before continuing. +LOOMGRAPH_HUB_PUBLIC_SSH= + +# Optional. `user@host` for the by-hand post-apply check of mesh SSH. +LOOMGRAPH_SANDBOX_SSH= +LOOMGRAPH_HUB_SSH= + +# --- One-off cleanup ------------------------------------------------------- + +# Optional. Exact name of a stale policy to delete during --apply. Empty skips +# that step. NetBird names auto-created policies +# "Temporary access policy for peer ", so this value needs quoting. +NETBIRD_DEAD_POLICY= diff --git a/deploy/install-hub.sh b/deploy/install-hub.sh new file mode 100755 index 0000000..ac99312 --- /dev/null +++ b/deploy/install-hub.sh @@ -0,0 +1,476 @@ +#!/usr/bin/env bash +# +# install-hub.sh - provision the loomgraph hub on the mesh host. +# +# Idempotent: safe to re-run. A second run against an already-provisioned host +# changes nothing and reports "no changes". +# +# Assumes node >= 22.13 and npm are ALREADY installed (developed against node +# v22.22.1, npm 10.9.4, systemd 257, Ubuntu 25.04). This script never installs +# or upgrades node. +# +# Refuses to proceed if: +# - node < 22.13 +# - port 8369 is already bound by something that is not lg-hub.service +# - the mesh interface is absent, or LOOMGRAPH_HUB_IP is not assigned to it +# +# PACKAGE SOURCE - read this before running. +# +# loomgraph is NOT published to the public npm registry: fetching +# https://registry.npmjs.org/loomgraph returns "Not found". `npm i -g loomgraph` +# (as the README currently documents) cannot work, so this script installs from +# a LOCAL artifact by default and never falls back to the registry for the +# loomgraph package itself. +# +# Build the artifact on a machine with the repo checked out: +# +# npm ci +# npm run build # tsup -> dist/ ; the bins point at dist/, so this +# # is mandatory, `npm pack` will not build for you +# npm pack # produces loomgraph-.tgz in the repo root +# scp loomgraph-.tgz :/tmp/ +# +# Then, on the host: +# +# sudo LOOMGRAPH_PACKAGE=/tmp/loomgraph-.tgz ./install-hub.sh +# +# With no LOOMGRAPH_PACKAGE set, the script looks for loomgraph-.tgz +# beside itself, in the repo root, and in the current directory, then falls back +# to a built repo working tree (one containing dist/hub/cli.js). If it finds +# none of those it exits with instructions rather than letting npm emit a bare +# 404. Whichever source is used, its version must equal the pinned version. +# +# Note: loomgraph's own runtime dependencies (commander, execa, yaml, zod) still +# come from the registry, so the host needs network access to it. +# +# Usage: sudo LOOMGRAPH_PACKAGE=/tmp/loomgraph-0.1.0.tgz ./install-hub.sh +# Overrides (env): LOOMGRAPH_VERSION, LOOMGRAPH_PACKAGE + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +readonly SERVICE_NAME="lg-hub.service" +readonly SERVICE_USER="lghub" +readonly SERVICE_GROUP="lghub" +readonly DATA_DIR="/var/lib/lghub" +readonly UNIT_SRC="${SCRIPT_DIR}/lg-hub.service" +readonly UNIT_DST="/etc/systemd/system/${SERVICE_NAME}" +readonly BIN_PATH="/usr/bin/lg-hub" +# Deployment addresses come from deploy/hub.env (gitignored; copy +# deploy/hub.env.example). Nothing host-specific is baked into this script - a +# provisioning script that defaults to someone else's mesh IP binds the wrong +# address and the unit fails to start with a confusing error. +ENV_FILE="${LOOMGRAPH_HUB_ENV:-${SCRIPT_DIR}/hub.env}" +if [ -f "$ENV_FILE" ]; then + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a +fi + +readonly MESH_IFACE="${LOOMGRAPH_MESH_IFACE:-wt0}" +readonly MESH_IP="${LOOMGRAPH_HUB_IP:-}" +readonly HUB_PORT="${LOOMGRAPH_HUB_PORT:-8369}" +readonly MIN_NODE_MAJOR=22 +readonly MIN_NODE_MINOR=13 + +# Pinned version fallback. The repo's package.json wins when this script is run +# from a checkout; never @latest either way. +DEFAULT_VERSION="0.1.0" + +CHANGED=0 +# Set by resolve_package_source. A global rather than a command substitution so +# that a failure inside it exits the script instead of a subshell. +PACKAGE_SOURCE="" + +log() { printf 'install-hub: %s\n' "$*"; } +step() { printf 'install-hub: [changed] %s\n' "$*"; CHANGED=$((CHANGED + 1)); } +die() { printf 'install-hub: FATAL %s\n' "$*" >&2; exit 1; } + +# --- preconditions ---------------------------------------------------------- + +require_root() { + if [ "$(id -u)" -ne 0 ]; then + die "must run as root (try: sudo $0)" + fi +} + +require_commands() { + local missing=() + local cmd + for cmd in node npm tar ss ip systemctl useradd groupadd install; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing+=("$cmd") + fi + done + if [ "${#missing[@]}" -gt 0 ]; then + die "missing required commands: ${missing[*]}" + fi + if ! command -v runuser >/dev/null 2>&1 && ! command -v sudo >/dev/null 2>&1; then + die "need runuser or sudo to drop privileges to ${SERVICE_USER}; neither is installed" + fi +} + +# Never run lg-hub as root. node:sqlite creates hub.db-wal and hub.db-shm +# alongside the database on first write; if root creates them, they end up +# root-owned and the service - which runs as lghub under ProtectSystem=strict - +# can no longer write its own store. Every lg-hub invocation goes through here. +run_as_hub() { + if command -v runuser >/dev/null 2>&1; then + runuser -u "$SERVICE_USER" -- "$@" + else + sudo -u "$SERVICE_USER" -- "$@" + fi +} + +check_node_version() { + local raw major minor + raw="$(node -v)" # e.g. v22.22.1 + raw="${raw#v}" + major="${raw%%.*}" + minor="${raw#*.}" + minor="${minor%%.*}" + if ! [[ "$major" =~ ^[0-9]+$ && "$minor" =~ ^[0-9]+$ ]]; then + die "could not parse node version from 'node -v' output: $(node -v)" + fi + if [ "$major" -lt "$MIN_NODE_MAJOR" ] || + { [ "$major" -eq "$MIN_NODE_MAJOR" ] && [ "$minor" -lt "$MIN_NODE_MINOR" ]; }; then + die "node $(node -v) is too old; loomgraph requires >= ${MIN_NODE_MAJOR}.${MIN_NODE_MINOR} (node:sqlite is only usable unflagged from 22.13). This script does not install node." + fi + log "node $(node -v) satisfies >= ${MIN_NODE_MAJOR}.${MIN_NODE_MINOR}" +} + +check_config() { + if [ -z "$MESH_IP" ]; then + die "LOOMGRAPH_HUB_IP is not set (the mesh address lg-hub binds). Copy deploy/hub.env.example to ${ENV_FILE} and fill it in, or export it for this run." + fi +} + +check_mesh() { + if ! ip -o link show dev "$MESH_IFACE" >/dev/null 2>&1; then + die "interface ${MESH_IFACE} does not exist - NetBird is not up. The unit binds ${MESH_IP} and will not start without it." + fi + if ! ip -o -4 addr show dev "$MESH_IFACE" | grep -Fq " ${MESH_IP}/"; then + die "${MESH_IP} is not assigned to ${MESH_IFACE}. Check 'ip -4 addr show dev ${MESH_IFACE}' and the NetBird peer configuration." + fi + log "${MESH_IP} is assigned to ${MESH_IFACE}" +} + +check_port_free() { + local holders + holders="$(ss -ltnH "sport = :${HUB_PORT}" 2>/dev/null || true)" + if [ -z "$holders" ]; then + log "port ${HUB_PORT} is free" + return 0 + fi + # Our own service already listening is the expected state on a re-run. + if systemctl is-active --quiet "$SERVICE_NAME"; then + log "port ${HUB_PORT} is held by ${SERVICE_NAME} (already provisioned)" + return 0 + fi + printf '%s\n' "$holders" >&2 + die "port ${HUB_PORT} is already bound by a process that is not ${SERVICE_NAME} (listeners above). Refusing to provision over it." +} + +# --- provisioning steps ----------------------------------------------------- + +resolve_version() { + local pkg_json="${SCRIPT_DIR}/../package.json" + if [ -n "${LOOMGRAPH_VERSION:-}" ]; then + printf '%s' "$LOOMGRAPH_VERSION" + return 0 + fi + if [ -f "$pkg_json" ]; then + local v + v="$(node -e 'const p=require(process.argv[1]); process.stdout.write(String(p.version ?? ""))' "$pkg_json")" + if [ -n "$v" ]; then + printf '%s' "$v" + return 0 + fi + fi + printf '%s' "$DEFAULT_VERSION" +} + +installed_version() { + local root + root="$(npm root -g 2>/dev/null || true)" + if [ -z "$root" ] || [ ! -f "${root}/loomgraph/package.json" ]; then + return 0 + fi + node -e 'const p=require(process.argv[1]); process.stdout.write(String(p.version ?? ""))' \ + "${root}/loomgraph/package.json" 2>/dev/null || true +} + +ensure_user() { + if ! getent group "$SERVICE_GROUP" >/dev/null 2>&1; then + groupadd --system "$SERVICE_GROUP" + step "created system group ${SERVICE_GROUP}" + fi + if ! getent passwd "$SERVICE_USER" >/dev/null 2>&1; then + useradd --system \ + --gid "$SERVICE_GROUP" \ + --home-dir "$DATA_DIR" \ + --no-create-home \ + --shell /usr/sbin/nologin \ + --comment "loomgraph hub service account" \ + "$SERVICE_USER" + step "created system user ${SERVICE_USER} (no login shell)" + fi +} + +ensure_data_dir() { + if [ ! -d "$DATA_DIR" ]; then + install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$DATA_DIR" + step "created ${DATA_DIR} (0750 ${SERVICE_USER}:${SERVICE_GROUP})" + return 0 + fi + local mode owner + mode="$(stat -c '%a' "$DATA_DIR")" + owner="$(stat -c '%U:%G' "$DATA_DIR")" + if [ "$mode" != "750" ]; then + chmod 0750 "$DATA_DIR" + step "fixed ${DATA_DIR} mode ${mode} -> 750" + fi + if [ "$owner" != "${SERVICE_USER}:${SERVICE_GROUP}" ]; then + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$DATA_DIR" + step "fixed ${DATA_DIR} owner ${owner} -> ${SERVICE_USER}:${SERVICE_GROUP}" + fi +} + +# Read the version out of a packed tarball without unpacking it to disk. +tarball_version() { + tar -xzOf "$1" package/package.json 2>/dev/null | + node -e 'let s="";process.stdin.on("data",(d)=>{s+=d;}).on("end",()=>{try{process.stdout.write(String(JSON.parse(s).version ?? ""));}catch{}});' +} + +# Sets PACKAGE_SOURCE. Never resolves to the public registry on its own: +# loomgraph is not published there. +resolve_package_source() { + local want="$1" candidate found + PACKAGE_SOURCE="" + + if [ -n "${LOOMGRAPH_PACKAGE:-}" ]; then + if [ -e "$LOOMGRAPH_PACKAGE" ]; then + PACKAGE_SOURCE="$LOOMGRAPH_PACKAGE" + log "package source: ${PACKAGE_SOURCE} (from LOOMGRAPH_PACKAGE)" + return 0 + fi + case "$LOOMGRAPH_PACKAGE" in + /* | ./* | ../* | *.tgz | *.tar.gz) + die "LOOMGRAPH_PACKAGE points at a path that does not exist: ${LOOMGRAPH_PACKAGE}" + ;; + esac + # Not a path: treat as an npm spec. Opt-in only, and only meaningful + # against a private registry. + log "WARNING: LOOMGRAPH_PACKAGE='${LOOMGRAPH_PACKAGE}' is not an existing path, so it is" + log " being passed to npm as a package spec. loomgraph is NOT on the public" + log " registry, so this only works against a private/mirrored one." + PACKAGE_SOURCE="$LOOMGRAPH_PACKAGE" + return 0 + fi + + for candidate in \ + "${SCRIPT_DIR}/loomgraph-${want}.tgz" \ + "${SCRIPT_DIR}/../loomgraph-${want}.tgz" \ + "./loomgraph-${want}.tgz"; do + if [ -f "$candidate" ]; then + PACKAGE_SOURCE="$candidate" + log "package source: ${PACKAGE_SOURCE} (auto-discovered tarball)" + return 0 + fi + done + + # A built working tree is acceptable; an unbuilt one is not - the bins point + # at dist/ and npm will not run the build for us (there is no prepare script). + if [ -f "${SCRIPT_DIR}/../package.json" ]; then + found="$(cd -- "${SCRIPT_DIR}/.." && pwd)" + if [ -f "${found}/dist/hub/cli.js" ]; then + PACKAGE_SOURCE="$found" + log "package source: ${PACKAGE_SOURCE} (built repo working tree)" + return 0 + fi + die "found a repo working tree at ${found} but ${found}/dist/hub/cli.js is missing - run 'npm ci && npm run build' there first, or build a tarball with 'npm pack' and pass LOOMGRAPH_PACKAGE=/path/to/loomgraph-${want}.tgz" + fi + + die "no loomgraph package to install. loomgraph is NOT published to npm, so there is nothing to fetch. On a machine with the repo: 'npm ci && npm run build && npm pack' produces loomgraph-${want}.tgz; copy it to this host and re-run with LOOMGRAPH_PACKAGE=/path/to/loomgraph-${want}.tgz" +} + +ensure_package() { + local want have src_version + want="$1" + have="$(installed_version)" + if [ "$have" = "$want" ]; then + log "loomgraph ${want} already installed globally" + return 0 + fi + + resolve_package_source "$want" + + # The pin has to mean something: check the artifact really is the version we + # think we are installing, before npm touches the system. + if [ -f "$PACKAGE_SOURCE" ]; then + src_version="$(tarball_version "$PACKAGE_SOURCE")" + if [ -z "$src_version" ]; then + die "could not read a version from ${PACKAGE_SOURCE}; is it an 'npm pack' tarball?" + fi + if [ "$src_version" != "$want" ]; then + die "version mismatch: ${PACKAGE_SOURCE} contains loomgraph ${src_version}, but the pinned version is ${want}. Set LOOMGRAPH_VERSION=${src_version} if that tarball is what you mean to install." + fi + elif [ -d "$PACKAGE_SOURCE" ]; then + src_version="$(node -e 'const p=require(process.argv[1]); process.stdout.write(String(p.version ?? ""))' "${PACKAGE_SOURCE}/package.json")" + if [ "$src_version" != "$want" ]; then + die "version mismatch: ${PACKAGE_SOURCE} is loomgraph ${src_version}, but the pinned version is ${want}" + fi + fi + + log "installing loomgraph ${want} from ${PACKAGE_SOURCE} (pinned; never @latest)" + npm install -g --no-fund --no-audit -- "$PACKAGE_SOURCE" + have="$(installed_version)" + if [ "$have" != "$want" ]; then + die "after installing ${PACKAGE_SOURCE} the global loomgraph reports version '${have:-}', expected '${want}'" + fi + step "installed loomgraph ${want} globally from ${PACKAGE_SOURCE}" +} + +ensure_bin_path() { + local prefix real + prefix="$(npm prefix -g)" + real="${prefix}/bin/lg-hub" + if [ ! -x "$real" ]; then + die "npm reports global prefix ${prefix} but ${real} is missing or not executable" + fi + if [ "$real" = "$BIN_PATH" ]; then + log "${BIN_PATH} provided directly by the npm global prefix" + return 0 + fi + if [ -L "$BIN_PATH" ] && [ "$(readlink -f "$BIN_PATH")" = "$(readlink -f "$real")" ]; then + log "${BIN_PATH} already links to ${real}" + return 0 + fi + if [ -e "$BIN_PATH" ] && [ ! -L "$BIN_PATH" ]; then + die "${BIN_PATH} exists and is not a symlink; refusing to replace it. The unit's ExecStart expects ${BIN_PATH}." + fi + ln -sfn "$real" "$BIN_PATH" + step "linked ${BIN_PATH} -> ${real}" +} + +ensure_db() { + if [ -f "${DATA_DIR}/hub.db" ]; then + log "${DATA_DIR}/hub.db already exists" + check_store_ownership + return 0 + fi + # --data-dir is passed explicitly: resolveDataDir() would otherwise fall back + # to $HOME/.local/share/loomgraph-hub, which is not where the unit looks. + run_as_hub "$BIN_PATH" init --data-dir "$DATA_DIR" + if [ ! -f "${DATA_DIR}/hub.db" ]; then + die "'lg-hub init' completed but ${DATA_DIR}/hub.db was not created" + fi + step "initialised ${DATA_DIR}/hub.db as ${SERVICE_USER}" + check_store_ownership +} + +# A root-owned hub.db-wal or hub.db-shm means somebody ran lg-hub as root. The +# service cannot write through it, so surface it rather than let the unit fail +# with an opaque SQLITE_CANTOPEN later. +check_store_ownership() { + local f owner stray=0 + for f in "${DATA_DIR}/hub.db" "${DATA_DIR}/hub.db-wal" "${DATA_DIR}/hub.db-shm"; do + [ -e "$f" ] || continue + owner="$(stat -c '%U' "$f")" + if [ "$owner" != "$SERVICE_USER" ]; then + printf 'install-hub: %s is owned by %s, expected %s\n' "$f" "$owner" "$SERVICE_USER" >&2 + stray=1 + fi + done + if [ "$stray" -eq 1 ]; then + die "store files are not owned by ${SERVICE_USER} (see above). Something ran lg-hub as root. Stop the service, 'chown -R ${SERVICE_USER}:${SERVICE_GROUP} ${DATA_DIR}', and re-run." + fi +} + +ensure_unit() { + if [ ! -f "$UNIT_SRC" ]; then + die "unit file not found next to this script: ${UNIT_SRC}" + fi + + # The unit ships as a TEMPLATE: @MESH_IP@ and @HUB_PORT@ are substituted here + # so the installed file carries concrete values. systemd could expand an + # EnvironmentFile instead, but then `systemctl cat` shows a variable and the + # operator has to go find what it resolved to - exactly the wrong trade when + # the value being hidden is the bind address. + local rendered + rendered="$(mktemp)" + # shellcheck disable=SC2064 # expand now: the path must survive this function + trap "rm -f '${rendered}'" RETURN + sed -e "s|@MESH_IP@|${MESH_IP}|g" -e "s|@HUB_PORT@|${HUB_PORT}|g" "$UNIT_SRC" > "$rendered" + if grep -q '@MESH_IP@\|@HUB_PORT@' "$rendered"; then + die "unit template still contains an unsubstituted placeholder after rendering" + fi + + local unit_changed=0 + if ! cmp -s "$rendered" "$UNIT_DST"; then + install -o root -g root -m 0644 "$rendered" "$UNIT_DST" + systemctl daemon-reload + step "installed ${UNIT_DST}" + unit_changed=1 + else + log "${UNIT_DST} already up to date" + fi + + if ! systemctl is-enabled --quiet "$SERVICE_NAME" 2>/dev/null; then + systemctl enable "$SERVICE_NAME" + step "enabled ${SERVICE_NAME}" + fi + + if ! systemctl is-active --quiet "$SERVICE_NAME"; then + systemctl start "$SERVICE_NAME" + step "started ${SERVICE_NAME}" + elif [ "$unit_changed" -eq 1 ] || [ "$CHANGED" -gt 0 ]; then + systemctl restart "$SERVICE_NAME" + step "restarted ${SERVICE_NAME} (unit or package changed)" + else + log "${SERVICE_NAME} already running with the current unit and package" + fi +} + +report() { + if [ "$CHANGED" -eq 0 ]; then + log "no changes - host already provisioned" + else + log "${CHANGED} change(s) applied" + fi + log "verify with:" + log " systemctl status ${SERVICE_NAME}" + log " ss -tln | grep ${HUB_PORT} # expect ${MESH_IP}:${HUB_PORT}, nothing on 0.0.0.0:${HUB_PORT}" + log "" + log "the web UI is disabled (--no-ui in the unit); the JSON API is the only surface." + log "NEVER run lg-hub as root - it would leave root-owned hub.db-wal/-shm and lock" + log "the service out of its own store. Add a member like this:" + log " runuser -u ${SERVICE_USER} -- ${BIN_PATH} member add --data-dir ${DATA_DIR}" + log "note: the store sets no busy_timeout, so a member/export command issued while" + log "the server is mid-ingest can fail with SQLITE_BUSY. Re-run it if it does." +} + +main() { + require_root + require_commands + check_config + check_node_version + check_mesh + check_port_free + + local version + version="$(resolve_version)" + log "pinned loomgraph version: ${version}" + + ensure_user + ensure_data_dir + ensure_package "$version" + ensure_bin_path + ensure_db + ensure_unit + report +} + +main "$@" diff --git a/deploy/lg-hub.service b/deploy/lg-hub.service new file mode 100644 index 0000000..4d21ce3 --- /dev/null +++ b/deploy/lg-hub.service @@ -0,0 +1,86 @@ +[Unit] +Description=loomgraph team hub (lg-hub) +After=network-online.target netbird.service +Wants=network-online.target netbird.service + +[Service] +Type=simple +User=lghub +Group=lghub + +# DATA DIRECTORY - must be explicit. +# resolveDataDir() (src/hub/serve.ts) falls back to $HOME/.local/share/ +# loomgraph-hub when neither --data-dir nor LOOMGRAPH_HUB_DIR is set. That +# default is broken for this unit: lghub is a system account whose home IS +# /var/lib/lghub, and ProtectHome=yes makes the usual home paths unreadable +# anyway. --data-dir on ExecStart below is authoritative; this env var is a +# backstop so any lg-hub process started in this unit's environment lands on +# the same store even if ExecStart is later edited. +Environment=LOOMGRAPH_HUB_DIR=/var/lib/lghub + +# --------------------------------------------------------------------------- +# READ THIS BEFORE CHANGING --behind-tls-proxy OR THE BIND ADDRESS. +# +# There is NO TLS proxy in front of this service. Nothing terminates TLS for +# it, and nothing is planned to. The flag's name is misleading; it is passed +# for the reason below, not because a proxy exists. +# +# @MESH_IP@ - rendered by install-hub.sh from LOOMGRAPH_HUB_IP - is a NetBird +# (WireGuard) mesh address. Every packet between mesh peers is already +# encrypted by WireGuard (ChaCha20-Poly1305), and the +# address is not routable from the public internet - public exposure on this +# host is limited to 22/80/443, and 8369 is bound to the mesh address only. +# That is what protects the bearer token on the wire. +# +# --behind-tls-proxy affects exactly ONE thing: the startup bind check in +# refuseBind() (src/hub/server.ts), which otherwise refuses any non-loopback +# bind on the grounds that a bearer token would cross plaintext HTTP. It is +# read once at startup (src/hub/serve.ts) and never again. There is NO +# request-time behaviour attached to it: the server does not read +# X-Forwarded-For, X-Real-IP, or any other proxy header, and does not derive +# identity, address, or scheme from request headers. Passing the flag +# therefore cannot introduce a header-spoofing vector. +# +# THIS REASONING DEPENDS ON THE MESH. If NetBird is removed, or this service +# is rebound to a routable address, the justification is void and the flag +# MUST be removed (and the transport decision re-made) before that change +# ships. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# WHY --no-ui. +# +# `lg-hub serve` serves a web UI on the SAME ORIGIN as the JSON API by default +# (src/hub/serve.ts). That UI stores the member bearer token in localStorage +# (src/hub/ui.ts: TOKEN_KEY = "lg-hub-token") against an http:// origin, and +# the API it drives includes POST /v1/members, which mints a NEW member token +# for any caller holding an admin-scoped token (src/hub/handlers.ts). +# +# Persisting a bearer token in localStorage on a plaintext origin, next to a +# token-minting endpoint, is a materially larger blast radius than the JSON API +# alone - any XSS or hostile page reachable on that origin reads the token and +# can mint more. The hub has no browser-facing use case here, so the UI is +# switched off rather than accepted. Re-enabling it means re-doing this +# assessment, not just deleting a flag. +# --------------------------------------------------------------------------- + +# Invoke the lg-hub bin (dist/hub/cli.js), never dist/hub/serve.js directly: +# cli.js installs a process warning filter before the node:sqlite import graph +# is evaluated. Bypassing it reintroduces an ExperimentalWarning on every start. +# The host and port below are substituted by install-hub.sh, from +# LOOMGRAPH_HUB_IP and LOOMGRAPH_HUB_PORT in deploy/hub.env, when it renders this +# template into /etc/systemd/system/. Editing the installed unit by hand works, +# but the next install-hub.sh run overwrites it - change deploy/hub.env instead. +ExecStart=/usr/bin/lg-hub serve --host @MESH_IP@ --port @HUB_PORT@ --behind-tls-proxy --data-dir /var/lib/lghub --no-ui + +Restart=on-failure +RestartSec=5 + +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ReadWritePaths=/var/lib/lghub + +[Install] +WantedBy=multi-user.target diff --git a/deploy/netbird-acl.sh b/deploy/netbird-acl.sh new file mode 100755 index 0000000..7521272 --- /dev/null +++ b/deploy/netbird-acl.sh @@ -0,0 +1,1233 @@ +#!/usr/bin/env bash +# +# netbird-acl.sh - converge NetBird access control to the loomgraph hub model: +# members reach the hub on one TCP port and nothing else, the operator keeps +# mesh SSH, and the default All -> All policy is disabled (never deleted). +# +# Default mode is DRY-RUN: nothing is written without an explicit --apply. +# +# deploy/netbird-acl.sh dry-run, print every API call it would make +# deploy/netbird-acl.sh --apply converge (idempotent, re-runnable) +# deploy/netbird-acl.sh --verify read-only check of the converged state +# +# The management API token is read from the macOS Keychain (or an `nbtoken` +# helper) and is never printed, logged, or passed on a command line. +# +set -euo pipefail + +# ---------------------------------------------------------------- configuration + +# This script ships with NO deployment addresses baked in. Yours live in +# deploy/hub.env (gitignored); copy deploy/hub.env.example and fill it in. A +# script that falls back to someone else's mesh IP converges the wrong network, +# so the REQUIRED variables abort instead of defaulting. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${LOOMGRAPH_HUB_ENV:-$SCRIPT_DIR/hub.env}" +if [ -f "$ENV_FILE" ]; then + # `set -a` exports every assignment so the values reach this script's own + # parameter expansions below and any child process that wants them. + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a +fi + +# Named here rather than inlined at each use so the error says WHICH file to +# edit. `die` is defined further down; this runs after it, from main's preamble. +require_var() { + local name="$1" value="$2" what="$3" + if [ -z "$value" ]; then + die "$name is not set ($what). Set it in $ENV_FILE - copy deploy/hub.env.example to start." + fi +} + +API_BASE="${NETBIRD_API:-}" +KEYCHAIN_SERVICE="${NETBIRD_KEYCHAIN_SERVICE:-netbird-pat}" + +HUB_PEER_IP="${LOOMGRAPH_HUB_IP:-}" +HUB_PEER_NAME="${LOOMGRAPH_HUB_PEER_NAME:-the hub peer}" +SANDBOX_PEER_IP="${LOOMGRAPH_SANDBOX_IP:-}" +IFS=',' read -r -a MAC_PEER_IPS <<< "${LOOMGRAPH_MAC_IPS:-}" +HUB_PORT="${LOOMGRAPH_HUB_PORT:-8369}" +SSH_PORT="22" + +# The hub serves its web UI for ANY non-/v1 GET, so a status-code probe on a +# path like /healthz returns 200 with an HTML page even when the API is dead. +# Probe the real route and assert on the BODY. +HEALTH_PATH="${LOOMGRAPH_HEALTH_PATH:-/v1/health}" +HEALTH_EXPECT="${LOOMGRAPH_HEALTH_EXPECT:-}" +PROBE_CONNECT_TIMEOUT="${LOOMGRAPH_PROBE_CONNECT_TIMEOUT:-5}" +PROBE_TIMEOUT="${LOOMGRAPH_PROBE_TIMEOUT:-10}" +# Ports a colleague peer must NOT reach on the personal MacBooks. +BLOCKED_PROBE_PORTS=(22 80 443 "$HUB_PORT") + +GROUP_HUB="hub" +GROUP_MEMBERS="loomgraph-members" +GROUP_SANDBOX="sandbox" +GROUP_MACS="personal macbooks" +GROUP_CLIENTS="clients" +GROUP_ALL="All" + +POLICY_MEMBER_HUB="loomgraph-hub-access" +POLICY_OP_HUB="loomgraph-operator-hub-access" +POLICY_OP_HUB_SSH="loomgraph-operator-hub-ssh" +POLICY_OP_SANDBOX_SSH="loomgraph-operator-sandbox-ssh" +POLICY_DEFAULT="Default" +# Optional one-off cleanup. Empty means "no stale policy to delete" and step 5 +# becomes a no-op rather than matching a policy name that is not yours. +POLICY_DEAD="${NETBIRD_DEAD_POLICY:-}" + +# A route to the hub that does NOT depend on the mesh, printed in the lockout +# warning. Empty is a legitimate answer - and one worth seeing spelled out +# before the All -> All policy goes away. +PUBLIC_FALLBACK_SSH="${LOOMGRAPH_HUB_PUBLIC_SSH:-}" +SANDBOX_SSH="${LOOMGRAPH_SANDBOX_SSH:-}" +HUB_SSH="${LOOMGRAPH_HUB_SSH:-}" + +# ---------------------------------------------------------------- mode + output + +MODE="dry-run" +ASSUME_YES="no" +FROM_MEMBER="no" +DO_PROBE="yes" +FAIL_COUNT=0 +PASS_COUNT=0 + +usage() { + cat <<'USAGE' +Usage: netbird-acl.sh [--apply | --verify | --dry-run] [options] + + (no flag) DRY-RUN. Prints every API call that --apply would make. Default. + --apply Perform the changes. Requires the self-lockout guard to pass. + --verify Read-only. Checks the converged state against the acceptance + criteria, PASS/FAIL each. Exit 1 on any FAIL. + + --assume-yes Skip the interactive confirmation before disabling "Default". + Required when --apply runs without a TTY. + --from-member Run --verify from a loomgraph-members peer: adds the negative + reachability probes (the MacBooks must be UNREACHABLE from here). + Meaningless from the operator Mac - it would prove nothing. + --no-probe --verify checks the API state only; skip all network probes. + +Configuration: + Read from deploy/hub.env (override the path with LOOMGRAPH_HUB_ENV), then + from the environment. Copy deploy/hub.env.example and fill it in - nothing + deployment-specific is baked into this script. + + NETBIRD_API REQUIRED. Management API base URL, incl. /api + NETBIRD_TOKEN API token (else `nbtoken`, else macOS Keychain) + NETBIRD_KEYCHAIN_SERVICE Keychain service name, default netbird-pat + LOOMGRAPH_HUB_IP REQUIRED. Hub peer mesh IP + LOOMGRAPH_HUB_PEER_NAME display name for that peer in output + LOOMGRAPH_SANDBOX_IP second operator-SSH peer; empty skips it + LOOMGRAPH_MAC_IPS REQUIRED for --from-member. Comma-separated mesh + IPs that a member peer must NOT reach + LOOMGRAPH_HUB_PORT hub service port, default 8369 + LOOMGRAPH_HEALTH_PATH hub health route, default /v1/health + LOOMGRAPH_HEALTH_EXPECT regex the health body must match (overrides the + built-in healthy-body heuristic) + LOOMGRAPH_HUB_PUBLIC_SSH non-mesh fallback host:port for the lockout warning + NETBIRD_DEAD_POLICY exact name of a stale policy to delete; empty skips +USAGE +} + +log() { printf '%s\n' "$*"; } +info() { printf ' %s\n' "$*"; } +warn() { printf 'WARN %s\n' "$*" >&2; } +die() { printf 'ERROR %s\n' "$*" >&2; exit 1; } +step() { printf '\n== %s\n' "$*"; } +pass() { PASS_COUNT=$((PASS_COUNT + 1)); printf ' PASS %s\n' "$*"; } +fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); printf ' FAIL %s\n' "$*"; } + +while [ "$#" -gt 0 ]; do + case "$1" in + --apply) MODE="apply" ;; + --verify) MODE="verify" ;; + --dry-run) MODE="dry-run" ;; + --assume-yes) ASSUME_YES="yes" ;; + --from-member) FROM_MEMBER="yes" ;; + --no-probe) DO_PROBE="no" ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; die "unknown argument: $1" ;; + esac + shift +done + +# ---------------------------------------------------------------- prerequisites + +for required in curl python3; do + command -v "$required" >/dev/null 2>&1 || die "required command not found: $required" +done + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/netbird-acl.XXXXXX")" +chmod 700 "$WORKDIR" +cleanup() { + case "$WORKDIR" in + */netbird-acl.*) rm -r -f -- "$WORKDIR" ;; + esac +} +trap cleanup EXIT INT TERM + +GROUPS_JSON="$WORKDIR/groups.json" +POLICIES_JSON="$WORKDIR/policies.json" +PEERS_JSON="$WORKDIR/peers.json" +NBJSON="$WORKDIR/nbjson.py" + +# ---------------------------------------------------------------- json helper +# +# A single read-only python helper. It never touches the network and never sees +# the token; it only parses the JSON curl already fetched and builds request +# bodies with correct escaping. + +cat > "$NBJSON" <<'PY' +"""Read-only JSON helpers for netbird-acl.sh. No network, no secrets.""" +import json +import re +import sys + + +def load(path): + with open(path, encoding="utf-8") as handle: + return json.load(handle) + + +def group_by_name(groups, name): + for group in groups: + if group.get("name") == name: + return group + return None + + +def group_peer_ids(group): + ids = [] + for peer in group.get("peers") or []: + if isinstance(peer, dict): + ids.append(peer.get("id", "")) + else: + ids.append(str(peer)) + return [i for i in ids if i] + + +def rule_group_ids(rule, key): + ids = [] + for item in rule.get(key) or []: + if isinstance(item, dict): + ids.append(item.get("id", "")) + else: + ids.append(str(item)) + return [i for i in ids if i] + + +def policy_by_name(policies, name): + found = [p for p in policies if p.get("name") == name] + if len(found) > 1: + sys.stderr.write("WARN %d policies named %r; using the first\n" % (len(found), name)) + return found[0] if found else None + + +def cmd_group_id(argv): + group = group_by_name(load(argv[0]), argv[1]) + if not group: + return 1 + sys.stdout.write(group.get("id", "")) + return 0 + + +def cmd_group_peers(argv): + group = group_by_name(load(argv[0]), argv[1]) + if not group: + return 1 + for peer_id in group_peer_ids(group): + print(peer_id) + return 0 + + +def cmd_group_peer_count(argv): + group = group_by_name(load(argv[0]), argv[1]) + if not group: + return 1 + print(len(group_peer_ids(group))) + return 0 + + +def cmd_peer_id(argv): + for peer in load(argv[0]): + if peer.get("ip") == argv[1]: + sys.stdout.write(peer.get("id", "")) + return 0 + return 1 + + +def cmd_peer_name(argv): + for peer in load(argv[0]): + if peer.get("ip") == argv[1]: + sys.stdout.write(peer.get("name", "")) + return 0 + return 1 + + +def cmd_peer_ip(argv): + for peer in load(argv[0]): + if peer.get("id") == argv[1]: + sys.stdout.write(peer.get("ip", "")) + return 0 + return 1 + + +def cmd_groups_of_peer(argv): + for group in load(argv[0]): + if argv[1] in group_peer_ids(group): + print(group.get("name", "")) + return 0 + + +def cmd_policy_id(argv): + policy = policy_by_name(load(argv[0]), argv[1]) + if not policy: + return 1 + sys.stdout.write(policy.get("id", "")) + return 0 + + +def cmd_policy_enabled(argv): + policy = policy_by_name(load(argv[0]), argv[1]) + if not policy: + return 1 + return 0 if policy.get("enabled") else 2 + + +def cmd_policy_matches(argv): + """policy-matches + + Exit 0 only when the named policy is enabled AND carries an enabled accept + rule that is exactly src -> dst on that protocol/port set/direction. + """ + path, name, src, dst, proto, ports_csv, bidir = argv[:7] + policy = policy_by_name(load(path), name) + if not policy or not policy.get("enabled"): + return 1 + want_ports = sorted(p for p in ports_csv.split(",") if p) + want_bidir = bidir.lower() == "true" + for rule in policy.get("rules") or []: + if not rule.get("enabled"): + continue + if rule.get("action", "accept") != "accept": + continue + if rule_group_ids(rule, "sources") != [src]: + continue + if rule_group_ids(rule, "destinations") != [dst]: + continue + if rule.get("protocol") != proto: + continue + if sorted(str(p) for p in (rule.get("ports") or [])) != want_ports: + continue + if rule.get("port_ranges"): + continue + if bool(rule.get("bidirectional")) != want_bidir: + continue + return 0 + return 1 + + +def cmd_rules_referencing(argv): + """rules-referencing + + -> policy|rule|enabled|role|protocol|ports|direction, one line per rule. + """ + path, group_id = argv[:2] + for policy in load(path): + for rule in policy.get("rules") or []: + roles = [] + if group_id in rule_group_ids(rule, "sources"): + roles.append("source") + if group_id in rule_group_ids(rule, "destinations"): + roles.append("destination") + if not roles: + continue + enabled = bool(policy.get("enabled")) and bool(rule.get("enabled")) + print("|".join([ + policy.get("name", ""), + rule.get("name", ""), + "enabled" if enabled else "disabled", + "+".join(roles), + str(rule.get("protocol", "")), + ",".join(str(p) for p in (rule.get("ports") or [])), + "bidirectional" if rule.get("bidirectional") else "unidirectional", + ])) + return 0 + + +def cmd_dead_policy_safe(argv): + """Exit 0 only when every rule of the named policy has no sources and no destinations.""" + policy = policy_by_name(load(argv[0]), argv[1]) + if not policy: + return 1 + for rule in policy.get("rules") or []: + if rule_group_ids(rule, "sources") or rule_group_ids(rule, "destinations"): + return 2 + return 0 + + +def cmd_mk_group(argv): + print(json.dumps({"name": argv[0], "peers": [p for p in argv[1:] if p]}, sort_keys=True)) + return 0 + + +def cmd_mk_policy(argv): + """mk-policy """ + name, description, src, dst, proto, ports_csv, bidir = argv[:7] + rule = { + "name": name, + "description": description, + "enabled": True, + "action": "accept", + "bidirectional": bidir.lower() == "true", + "protocol": proto, + "sources": [src], + "destinations": [dst], + } + ports = [p for p in ports_csv.split(",") if p] + if ports: + rule["ports"] = ports + print(json.dumps({ + "name": name, + "description": description, + "enabled": True, + "sourcePostureChecks": [], + "rules": [rule], + }, sort_keys=True)) + return 0 + + +def cmd_mk_disable(argv): + """Build the PUT body that disables a policy, preserving every rule verbatim. + + GET returns sources/destinations as expanded group objects; PUT wants bare + group ids. Only the top-level `enabled` flag changes - same as the toggle in + the dashboard, so the policy can be re-enabled with one more PUT. + """ + policy = policy_by_name(load(argv[0]), argv[1]) + if not policy: + return 1 + rules = [] + for rule in policy.get("rules") or []: + new_rule = { + "id": rule.get("id"), + "name": rule.get("name", ""), + "description": rule.get("description", ""), + "enabled": bool(rule.get("enabled")), + "action": rule.get("action", "accept"), + "bidirectional": bool(rule.get("bidirectional")), + "protocol": rule.get("protocol", "all"), + "sources": rule_group_ids(rule, "sources"), + "destinations": rule_group_ids(rule, "destinations"), + } + if rule.get("ports"): + new_rule["ports"] = [str(p) for p in rule["ports"]] + if rule.get("port_ranges"): + new_rule["port_ranges"] = rule["port_ranges"] + rules.append({k: v for k, v in new_rule.items() if v is not None}) + print(json.dumps({ + "name": policy.get("name", ""), + "description": policy.get("description", ""), + "enabled": False, + "sourcePostureChecks": policy.get("source_posture_checks") or [], + "rules": rules, + }, sort_keys=True)) + return 0 + + +HEALTHY_TOKENS = {"ok", "up", "true", "pass", "passing", "healthy", "serving", "alive", "ready"} + + +def cmd_health_verdict(argv): + """health-verdict [expect-regex] -> VERDICT:detail on stdout. + + The hub serves its web UI for any non-/v1 GET, so an HTML body means the API + did not answer even when the status code was 200. Judge the body, never the + status code alone. + """ + with open(argv[0], "rb") as handle: + raw = handle.read() + text = raw.decode("utf-8", "replace").strip() + expect = argv[1] if len(argv) > 1 else "" + summary = " ".join(text.split())[:200] + + if not text: + print("EMPTY:no response body") + return 0 + lowered = text.lower() + if lowered.startswith(" [args...]\n") + sys.exit(64) + sys.exit(COMMANDS[sys.argv[1]](sys.argv[2:])) +PY + +nbj() { python3 "$NBJSON" "$@"; } + +# ---------------------------------------------------------------- api plumbing + +TOKEN="" +API_STATUS="" +API_BODY="" + +load_token() { + if [ -n "${NETBIRD_TOKEN:-}" ]; then + TOKEN="$NETBIRD_TOKEN" + elif command -v nbtoken >/dev/null 2>&1; then + TOKEN="$(nbtoken)" || die "the nbtoken helper failed; check it or set NETBIRD_TOKEN" + elif command -v security >/dev/null 2>&1; then + TOKEN="$(security find-generic-password -a "$USER" -s "$KEYCHAIN_SERVICE" -w 2>/dev/null)" || + die "no Keychain item for service \"$KEYCHAIN_SERVICE\" / account \"$USER\"; add it or set NETBIRD_TOKEN" + else + die "no token source: set NETBIRD_TOKEN, install nbtoken, or store the PAT in the Keychain" + fi + [ -n "$TOKEN" ] || die "empty NetBird token from the configured source" +} + +# api_call [body-file] -> response body on stdout, status in API_STATUS. +# +# The token reaches curl through a config file on stdin, so it never appears in +# the process table, in `ps`, or in any shell history. +api_call() { + local method="$1" path="$2" body_file="${3:-}" + local out status rc + out="$WORKDIR/response.json" + set +e + status="$( + { + printf 'url = "%s%s"\n' "$API_BASE" "$path" + printf 'request = "%s"\n' "$method" + printf 'header = "Authorization: Token %s"\n' "$TOKEN" + printf 'header = "Accept: application/json"\n' + printf 'silent\n' + printf 'show-error\n' + printf 'output = "%s"\n' "$out" + printf 'write-out = "%%{http_code}"\n' + if [ -n "$body_file" ]; then + printf 'header = "Content-Type: application/json"\n' + printf 'data-binary = "@%s"\n' "$body_file" + fi + } | curl --config - + )" + rc="$?" + set -e + [ "$rc" -eq 0 ] || die "curl failed (exit $rc) on $method $path" + API_STATUS="$status" + if [ -f "$out" ]; then + cat "$out" + rm -f -- "$out" + fi +} + +api_get() { + local path="$1" dest="$2" + api_call GET "$path" > "$dest" + case "$API_STATUS" in + 2*) ;; + *) die "GET $path returned HTTP $API_STATUS: $(head -c 400 "$dest")" ;; + esac +} + +# api_write +# +# Dry-run: prints the exact call and body, changes nothing. +# Apply: performs the call and leaves the response body in API_BODY. +api_write() { + local method="$1" path="$2" body_file="$3" description="$4" + API_BODY="" + if [ "$MODE" != "apply" ]; then + printf ' DRY-RUN %s %s%s\n' "$method" "$API_BASE" "$path" + printf ' %s\n' "$description" + if [ -n "$body_file" ]; then + printf ' body: %s\n' "$(nbj compact "$body_file")" + fi + return 0 + fi + API_BODY="$(api_call "$method" "$path" "$body_file")" + case "$API_STATUS" in + 2*) info "$method $path -> HTTP $API_STATUS ($description)" ;; + *) die "$method $path returned HTTP $API_STATUS: $(printf '%s' "$API_BODY" | head -c 400)" ;; + esac +} + +refresh_state() { + api_get "/groups" "$GROUPS_JSON" + api_get "/policies" "$POLICIES_JSON" + api_get "/peers" "$PEERS_JSON" +} + +# ---------------------------------------------------------------- lookups + +peer_id_for_ip() { + local ip="$1" id + if ! id="$(nbj peer-id "$PEERS_JSON" "$ip")"; then + die "no NetBird peer found with mesh IP $ip" + fi + printf '%s' "$id" +} + +group_id_or_empty() { + local name="$1" id + if id="$(nbj group-id "$GROUPS_JSON" "$name")"; then + printf '%s' "$id" + fi +} + +require_group_id() { + local name="$1" id + id="$(group_id_or_empty "$name")" + [ -n "$id" ] || die "expected group \"$name\" to exist but it does not" + printf '%s' "$id" +} + +slug() { printf '%s' "$1" | tr -c 'a-zA-Z0-9' '-'; } + +# write_body -> path of the file holding the generated JSON +write_body() { + local name="$1" + shift + local path="$WORKDIR/body-$name.json" + "$@" > "$path" + printf '%s' "$path" +} + +# ---------------------------------------------------------------- convergence + +# ensure_group [peer-id...] -> id in ENSURE_GROUP_ID +ENSURE_GROUP_ID="" +ensure_group() { + local name="$1" + shift + local existing body + existing="$(group_id_or_empty "$name")" + if [ -n "$existing" ]; then + info "group \"$name\" already exists ($existing) - no change" + ENSURE_GROUP_ID="$existing" + return 0 + fi + body="$(write_body "group-$(slug "$name")" nbj mk-group "$name" "$@")" + api_write POST "/groups" "$body" "create group \"$name\"" + if [ "$MODE" != "apply" ]; then + ENSURE_GROUP_ID="" + return 0 + fi + printf '%s' "$API_BODY" > "$WORKDIR/created-group.json" + ENSURE_GROUP_ID="$(nbj id-of "$WORKDIR/created-group.json")" + [ -n "$ENSURE_GROUP_ID" ] || die "group \"$name\" was created but the API returned no id" + info "group \"$name\" created ($ENSURE_GROUP_ID)" +} + +# ensure_policy +ensure_policy() { + local name="$1" description="$2" src="$3" dst="$4" proto="$5" ports="$6" bidir="$7" + local direction="unidirectional" + [ "$bidir" = "false" ] || direction="bidirectional" + if nbj policy-matches "$POLICIES_JSON" "$name" "$src" "$dst" "$proto" "$ports" "$bidir"; then + info "policy \"$name\" already matches and is enabled - no change" + return 0 + fi + if nbj policy-id "$POLICIES_JSON" "$name" > /dev/null; then + warn "policy \"$name\" exists but does not match the target rule." + die "refusing to rewrite \"$name\" automatically - inspect it in the dashboard, fix or delete it, then re-run" + fi + local body + body="$(write_body "policy-$(slug "$name")" \ + nbj mk-policy "$name" "$description" "$src" "$dst" "$proto" "$ports" "$bidir")" + api_write POST "/policies" "$body" "create policy \"$name\" ($proto/$ports, $direction)" +} + +# ---------------------------------------------------------------- lockout guard + +# Evaluated against the live policy list. Every path the operator needs after +# "Default" goes away must already exist and be enabled. +operator_path_ok() { + local macs="$1" hub="$2" sandbox="$3" ok=0 + if ! nbj policy-matches "$POLICIES_JSON" "$POLICY_OP_HUB" "$macs" "$hub" tcp "$HUB_PORT" false; then + warn "missing or disabled: \"$POLICY_OP_HUB\" ($GROUP_MACS -> $GROUP_HUB tcp/$HUB_PORT)" + ok=1 + fi + if ! nbj policy-matches "$POLICIES_JSON" "$POLICY_OP_HUB_SSH" "$macs" "$hub" tcp "$SSH_PORT" false; then + warn "missing or disabled: \"$POLICY_OP_HUB_SSH\" ($GROUP_MACS -> $GROUP_HUB tcp/$SSH_PORT)" + ok=1 + fi + if ! nbj policy-matches "$POLICIES_JSON" "$POLICY_OP_SANDBOX_SSH" "$macs" "$sandbox" tcp "$SSH_PORT" false; then + warn "missing or disabled: \"$POLICY_OP_SANDBOX_SSH\" ($GROUP_MACS -> $GROUP_SANDBOX tcp/$SSH_PORT)" + ok=1 + fi + return "$ok" +} + +lockout_warning() { + printf '\n' + printf ' ************************************************************\n' + printf ' * ABOUT TO DISABLE THE "Default" All -> All POLICY\n' + printf ' *\n' + printf ' * If the new policies are insufficient you WILL lose mesh\n' + printf ' * access to %s. Fallbacks, in order:\n' "$HUB_PEER_NAME" + printf ' *\n' + if [ -n "$PUBLIC_FALLBACK_SSH" ]; then + printf ' * 1. public SSH to %s, off the mesh\n' "$PUBLIC_FALLBACK_SSH" + else + printf ' * 1. NONE CONFIGURED - LOOMGRAPH_HUB_PUBLIC_SSH is empty, so\n' + printf ' * this script knows of no way back in without the mesh.\n' + fi + printf ' * 2. your provider console - know the login BEFORE continuing\n' + printf ' *\n' + printf ' * Run this from a shell that does NOT depend on the mesh.\n' + printf ' * "Default" is disabled, never deleted: re-enable it with a\n' + printf ' * single PUT if anything goes wrong.\n' + printf ' ************************************************************\n' + printf '\n' +} + +confirm_disable() { + if [ "$ASSUME_YES" = "yes" ]; then + info "confirmation skipped (--assume-yes)" + return 0 + fi + if [ ! -t 0 ]; then + die "refusing to disable \"$POLICY_DEFAULT\" without a TTY; re-run with --assume-yes if you are sure" + fi + local answer="" + printf 'Type DISABLE to continue, anything else aborts: ' + read -r answer + [ "$answer" = "DISABLE" ] || die "aborted by operator; nothing was disabled" +} + +# ---------------------------------------------------------------- converge flow + +# Refuse to proceed if a colleague peer also sits in "clients" (which carries +# the 0.0.0.0/0 exit node) or in "personal macbooks". Membership is fixed by +# hand, not silently rewritten here. +check_member_membership() { + local member_peer groups_of leaked=0 + if ! nbj group-id "$GROUPS_JSON" "$GROUP_MEMBERS" > /dev/null; then + return 0 + fi + while IFS= read -r member_peer; do + [ -n "$member_peer" ] || continue + groups_of="$(nbj groups-of-peer "$GROUPS_JSON" "$member_peer" | tr '\n' ' ')" + case " $groups_of " in + *" $GROUP_CLIENTS "*|*" $GROUP_MACS "*) + warn "peer $member_peer is in \"$GROUP_MEMBERS\" and also in: $groups_of" + leaked=1 + ;; + esac + done < <(nbj group-peers "$GROUPS_JSON" "$GROUP_MEMBERS") + [ "$leaked" -eq 0 ] || + die "a $GROUP_MEMBERS peer also belongs to $GROUP_CLIENTS or $GROUP_MACS; fix group membership first" +} + +converge() { + local hub_peer sandbox_peer + local hub_group members_group sandbox_group macs_group + + hub_peer="$(peer_id_for_ip "$HUB_PEER_IP")" + # The sandbox peer is OPTIONAL. Spec 00 listed only `hub` and + # `loomgraph-members`; the sandbox group exists because "operator retains mesh + # SSH to a second box" is unsatisfiable without a destination group holding + # it. A deployment with no such box sets LOOMGRAPH_SANDBOX_IP empty and gets + # neither the group nor its policy - not an empty group that matches nothing. + sandbox_peer="" + if [ -n "$SANDBOX_PEER_IP" ]; then + sandbox_peer="$(peer_id_for_ip "$SANDBOX_PEER_IP")" + fi + + step "Step 1 - groups" + ensure_group "$GROUP_HUB" "$hub_peer"; hub_group="$ENSURE_GROUP_ID" + ensure_group "$GROUP_MEMBERS"; members_group="$ENSURE_GROUP_ID" + sandbox_group="" + if [ -n "$sandbox_peer" ]; then + ensure_group "$GROUP_SANDBOX" "$sandbox_peer"; sandbox_group="$ENSURE_GROUP_ID" + else + info "LOOMGRAPH_SANDBOX_IP is empty - skipping the \"$GROUP_SANDBOX\" group" + fi + macs_group="$(require_group_id "$GROUP_MACS")" + check_member_membership + + step "Step 2 - member policy ($GROUP_MEMBERS -> $GROUP_HUB tcp/$HUB_PORT, unidirectional)" + ensure_policy "$POLICY_MEMBER_HUB" \ + "loomgraph colleagues reach the hub on tcp/$HUB_PORT and nothing else" \ + "$members_group" "$hub_group" tcp "$HUB_PORT" false + + step "Step 3 - operator policies (enumerated BEFORE Default is touched)" + ensure_policy "$POLICY_OP_HUB" \ + "operator Macs reach the loomgraph hub port" \ + "$macs_group" "$hub_group" tcp "$HUB_PORT" false + ensure_policy "$POLICY_OP_HUB_SSH" \ + "operator Macs administer the hub over the mesh" \ + "$macs_group" "$hub_group" tcp "$SSH_PORT" false + if [ -n "$sandbox_group" ]; then + ensure_policy "$POLICY_OP_SANDBOX_SSH" \ + "operator Macs keep mesh SSH to the sandbox peer" \ + "$macs_group" "$sandbox_group" tcp "$SSH_PORT" false + else + info "no sandbox peer configured - skipping \"$POLICY_OP_SANDBOX_SSH\"" + fi + + step "Step 4 - self-lockout guard, then DISABLE \"$POLICY_DEFAULT\"" + if [ "$MODE" = "apply" ]; then + # Re-read live state: the guard must judge what the API actually has, not + # what this run intended to create. + refresh_state + hub_group="$(require_group_id "$GROUP_HUB")" + sandbox_group="$(require_group_id "$GROUP_SANDBOX")" + macs_group="$(require_group_id "$GROUP_MACS")" + if ! operator_path_ok "$macs_group" "$hub_group" "$sandbox_group"; then + die "self-lockout guard FAILED: operator access policies are not in place. \"$POLICY_DEFAULT\" left ENABLED." + fi + info "self-lockout guard passed: operator keeps hub tcp/$HUB_PORT, hub tcp/$SSH_PORT, sandbox tcp/$SSH_PORT" + else + info "in --apply the guard re-reads live state here and refuses to disable" + info "\"$POLICY_DEFAULT\" unless the three operator policies above exist and are enabled." + fi + + local default_id="" + if default_id="$(nbj policy-id "$POLICIES_JSON" "$POLICY_DEFAULT")"; then + if nbj policy-enabled "$POLICIES_JSON" "$POLICY_DEFAULT"; then + lockout_warning + if [ "$MODE" = "apply" ]; then + confirm_disable + fi + local body + body="$(write_body "disable-default" nbj mk-disable "$POLICIES_JSON" "$POLICY_DEFAULT")" + api_write PUT "/policies/$default_id" "$body" \ + "DISABLE (never delete) policy \"$POLICY_DEFAULT\"" + info "rollback: PUT /policies/$default_id with the same body and enabled=true" + else + info "\"$POLICY_DEFAULT\" is already disabled - no change" + fi + else + warn "policy \"$POLICY_DEFAULT\" not found - it should exist, disabled, as a one-call rollback" + fi + + step "Step 5 - delete the stale auto-created policy, if one was named" + local dead_id="" + if [ -z "$POLICY_DEAD" ]; then + info "NETBIRD_DEAD_POLICY is empty - nothing to delete, skipping" + elif dead_id="$(nbj policy-id "$POLICIES_JSON" "$POLICY_DEAD")"; then + if nbj dead-policy-safe "$POLICIES_JSON" "$POLICY_DEAD"; then + api_write DELETE "/policies/$dead_id" "" "delete dead policy \"$POLICY_DEAD\"" + else + warn "\"$POLICY_DEAD\" has non-empty sources or destinations - NOT deleting it" + fi + else + info "dead policy \"$POLICY_DEAD\" not present - no change" + fi + info "the PEER behind that policy is left alone; delete it by hand if it is not in use" + + if [ "$MODE" = "apply" ]; then + printf '\nApply complete. Re-run with --verify to check the acceptance criteria.\n' + else + printf '\nDRY-RUN complete. Nothing was changed. Re-run with --apply to converge.\n' + fi +} + +# ---------------------------------------------------------------- verify flow + +# ---------------------------------------------------------------- probes +# +# Network probes, not NetBird API calls. They never carry the token. +# +# PROBE_RC curl exit code: 0 = an HTTP response came back, 7 = could not +# connect, 28 = timed out, 52/56 = TCP connected then died. +# PROBE_CODE HTTP status (0 when no response). +# PROBE_BODY first 2 KiB of the body. +PROBE_RC=0 +PROBE_CODE="" +PROBE_BODY_FILE="" + +http_probe() { + local host="$1" port="$2" path="$3" + PROBE_BODY_FILE="$WORKDIR/probe-body" + : > "$PROBE_BODY_FILE" + set +e + PROBE_CODE="$( + curl --silent \ + --connect-timeout "$PROBE_CONNECT_TIMEOUT" \ + --max-time "$PROBE_TIMEOUT" \ + --output "$PROBE_BODY_FILE" \ + --write-out '%{http_code}' \ + "http://$host:$port$path" 2>/dev/null + )" + PROBE_RC="$?" + set -e +} + +# A peer that should be blocked must give us no HTTP response at all. curl exit +# 7 (refused/unreachable) or 28 (timeout, the usual NetBird drop) is the pass +# signal; anything that completed a TCP handshake is a FAIL. +probe_is_blocked() { + case "$PROBE_RC" in + 7|28) return 0 ;; + *) return 1 ;; + esac +} + +probe_outcome_text() { + case "$PROBE_RC" in + 0) printf 'HTTP %s received' "$PROBE_CODE" ;; + 7) printf 'connection refused or filtered (curl 7)' ;; + 28) printf 'timed out (curl 28) - consistent with an ACL drop' ;; + 52) printf 'TCP CONNECTED then empty reply (curl 52)' ;; + 56) printf 'TCP CONNECTED then reset (curl 56)' ;; + 35) printf 'TCP CONNECTED, TLS handshake attempted (curl 35)' ;; + *) printf 'curl exit %s' "$PROBE_RC" ;; + esac +} + +# Positive probe: the hub API itself must answer on the real route, and the +# BODY must look healthy. A 200 with an HTML page is the web UI answering for a +# non-/v1 path - that is a dead API, not a pass. +verify_hub_health() { + local verdict kind detail + http_probe "$HUB_PEER_IP" "$HUB_PORT" "$HEALTH_PATH" + if [ "$PROBE_RC" -ne 0 ]; then + fail "hub $HUB_PEER_IP:$HUB_PORT$HEALTH_PATH did not answer: $(probe_outcome_text)" + return 0 + fi + verdict="$(nbj health-verdict "$PROBE_BODY_FILE" "$HEALTH_EXPECT")" + kind="${verdict%%:*}" + detail="${verdict#*:}" + case "$kind" in + HEALTHY) + pass "hub $HEALTH_PATH answered HTTP $PROBE_CODE and the body is healthy: $detail" + ;; + JSON_NO_STATUS) + # A JSON body proves the API answered, not the UI - but the schema is not + # one we recognise, so surface it instead of silently passing it as healthy. + pass "hub $HEALTH_PATH answered HTTP $PROBE_CODE with a non-HTML JSON body ($detail)" + info "check that body by eye, or pin it with LOOMGRAPH_HEALTH_EXPECT=" + ;; + HTML) + fail "hub $HEALTH_PATH returned HTTP $PROBE_CODE but the body is the WEB UI, not the API: $detail" + ;; + *) + fail "hub $HEALTH_PATH returned HTTP $PROBE_CODE with an unhealthy body [$kind]: $detail" + ;; + esac +} + +# Negative probes. Only meaningful when run FROM a loomgraph-members peer. +verify_member_isolation() { + local mac port blocked_all=0 + + http_probe "$HUB_PEER_IP" "$HUB_PORT" "$HEALTH_PATH" + if [ "$PROBE_RC" -eq 0 ]; then + pass "member peer reaches the hub on $HUB_PEER_IP:$HUB_PORT (HTTP $PROBE_CODE)" + else + fail "member peer cannot reach the hub on $HUB_PEER_IP:$HUB_PORT: $(probe_outcome_text)" + fi + + # Same host, a port the policy does not open: must be blocked. + http_probe "$HUB_PEER_IP" "$SSH_PORT" "/" + if probe_is_blocked; then + pass "member peer is blocked from $HUB_PEER_IP:$SSH_PORT ($(probe_outcome_text))" + else + fail "member peer REACHED $HUB_PEER_IP:$SSH_PORT - the policy is wider than tcp/$HUB_PORT ($(probe_outcome_text))" + fi + + for mac in "${MAC_PEER_IPS[@]}"; do + for port in "${BLOCKED_PROBE_PORTS[@]}"; do + http_probe "$mac" "$port" "/" + if probe_is_blocked; then + info "blocked as expected: $mac:$port ($(probe_outcome_text))" + else + fail "member peer REACHED MacBook $mac:$port - $(probe_outcome_text)" + blocked_all=1 + fi + done + done + if [ "$blocked_all" -eq 0 ]; then + pass "member peer cannot reach either MacBook on ports ${BLOCKED_PROBE_PORTS[*]}" + fi +} + +verify_members_reach_only_hub() { + local members_group="$1" extra=0 line policy_name enabled_flag + while IFS= read -r line; do + [ -n "$line" ] || continue + policy_name="${line%%|*}" + enabled_flag="$(printf '%s' "$line" | cut -d'|' -f3)" + if [ "$enabled_flag" != "enabled" ]; then + continue + fi + if [ "$policy_name" = "$POLICY_MEMBER_HUB" ]; then + continue + fi + fail "extra enabled rule touches $GROUP_MEMBERS: $line" + extra=1 + done < <(nbj rules-referencing "$POLICIES_JSON" "$members_group") + if [ "$extra" -eq 0 ]; then + pass "no other enabled rule references $GROUP_MEMBERS" + fi +} + +verify_members_group_hygiene() { + local stray=0 member_peer member_ip groups_of gname + while IFS= read -r member_peer; do + [ -n "$member_peer" ] || continue + member_ip="$(nbj peer-ip "$PEERS_JSON" "$member_peer" || printf 'unknown-ip')" + groups_of="$(nbj groups-of-peer "$GROUPS_JSON" "$member_peer")" + while IFS= read -r gname; do + [ -n "$gname" ] || continue + case "$gname" in + "$GROUP_ALL"|"$GROUP_MEMBERS") ;; + *) + fail "member peer $member_ip is also in group \"$gname\" (leak path around the ACL)" + stray=1 + ;; + esac + done <<< "$groups_of" + done < <(nbj group-peers "$GROUPS_JSON" "$GROUP_MEMBERS") + if [ "$stray" -eq 0 ]; then + pass "every $GROUP_MEMBERS peer sits only in $GROUP_ALL and $GROUP_MEMBERS" + fi +} + +verify() { + local hub_peer sandbox_peer + local hub_group members_group sandbox_group macs_group + + hub_peer="$(peer_id_for_ip "$HUB_PEER_IP")" + sandbox_peer="" + if [ -n "$SANDBOX_PEER_IP" ]; then + sandbox_peer="$(peer_id_for_ip "$SANDBOX_PEER_IP")" + fi + + step "Acceptance criteria" + + hub_group="$(group_id_or_empty "$GROUP_HUB")" + if [ -z "$hub_group" ]; then + fail "group \"$GROUP_HUB\" exists" + else + local hub_peers hub_count + hub_peers="$(nbj group-peers "$GROUPS_JSON" "$GROUP_HUB" | tr '\n' ' ')" + hub_count="$(nbj group-peer-count "$GROUPS_JSON" "$GROUP_HUB")" + if [ "$hub_count" = "1" ] && [ "$hub_peers" = "$hub_peer " ]; then + pass "group \"$GROUP_HUB\" holds exactly $HUB_PEER_NAME ($HUB_PEER_IP)" + else + fail "group \"$GROUP_HUB\" should hold only $HUB_PEER_NAME ($HUB_PEER_IP); holds $hub_count peer(s)" + fi + fi + + members_group="$(group_id_or_empty "$GROUP_MEMBERS")" + if [ -z "$members_group" ]; then + fail "group \"$GROUP_MEMBERS\" exists" + else + pass "group \"$GROUP_MEMBERS\" exists" + fi + + sandbox_group="" + if [ -z "$SANDBOX_PEER_IP" ]; then + info "LOOMGRAPH_SANDBOX_IP is empty - sandbox criteria not checked" + else + sandbox_group="$(group_id_or_empty "$GROUP_SANDBOX")" + if [ -z "$sandbox_group" ]; then + fail "group \"$GROUP_SANDBOX\" exists (required for operator mesh SSH to $SANDBOX_PEER_IP)" + else + local sandbox_peers + sandbox_peers="$(nbj group-peers "$GROUPS_JSON" "$GROUP_SANDBOX" | tr '\n' ' ')" + if [ "$sandbox_peers" = "$sandbox_peer " ]; then + pass "group \"$GROUP_SANDBOX\" holds exactly the sandbox peer ($SANDBOX_PEER_IP)" + else + fail "group \"$GROUP_SANDBOX\" should hold only the sandbox peer ($SANDBOX_PEER_IP)" + fi + fi + fi + + macs_group="$(group_id_or_empty "$GROUP_MACS")" + if [ -z "$macs_group" ]; then + fail "group \"$GROUP_MACS\" exists" + fi + + # A loomgraph-members peer reaches the hub IP on the hub port, and nothing else. + if [ -n "$members_group" ] && [ -n "$hub_group" ] && + nbj policy-matches "$POLICIES_JSON" "$POLICY_MEMBER_HUB" \ + "$members_group" "$hub_group" tcp "$HUB_PORT" false; then + pass "\"$POLICY_MEMBER_HUB\" enabled: $GROUP_MEMBERS -> $GROUP_HUB tcp/$HUB_PORT, unidirectional" + else + fail "\"$POLICY_MEMBER_HUB\" missing, disabled, or not exactly $GROUP_MEMBERS -> $GROUP_HUB tcp/$HUB_PORT unidirectional" + fi + + # A loomgraph-members peer reaches nothing else, including both MacBooks. + if [ -n "$members_group" ]; then + verify_members_reach_only_hub "$members_group" + verify_members_group_hygiene + fi + + # Operator retains mesh SSH to the hub (and the sandbox peer, when configured). + if [ -n "$macs_group" ] && [ -n "$hub_group" ] && [ -n "$sandbox_group" ] && + operator_path_ok "$macs_group" "$hub_group" "$sandbox_group"; then + pass "operator retains hub tcp/$HUB_PORT, hub tcp/$SSH_PORT and sandbox tcp/$SSH_PORT" + else + fail "operator access policies are incomplete (see warnings above)" + fi + + # Default is disabled, not deleted. + if nbj policy-id "$POLICIES_JSON" "$POLICY_DEFAULT" > /dev/null; then + if nbj policy-enabled "$POLICIES_JSON" "$POLICY_DEFAULT"; then + fail "\"$POLICY_DEFAULT\" is still ENABLED (All -> All, every port)" + else + pass "\"$POLICY_DEFAULT\" is present and disabled" + fi + else + fail "\"$POLICY_DEFAULT\" has been DELETED - it must remain, disabled, for one-call rollback" + fi + + # The stale auto-created policy is gone, when one was named. + if [ -z "$POLICY_DEAD" ]; then + info "NETBIRD_DEAD_POLICY is empty - stale-policy criterion not checked" + elif nbj policy-id "$POLICIES_JSON" "$POLICY_DEAD" > /dev/null; then + fail "stale policy \"$POLICY_DEAD\" still exists" + else + pass "stale policy \"$POLICY_DEAD\" is gone" + fi + + if [ "$DO_PROBE" = "yes" ]; then + step "Reachability probes" + if [ "$FROM_MEMBER" = "yes" ]; then + info "running from a $GROUP_MEMBERS peer: the MacBooks must be unreachable from here" + verify_member_isolation + else + verify_hub_health + info "the negative criteria (a member peer reaching nothing but the hub) cannot be" + info "proven from this machine. Re-run on a $GROUP_MEMBERS peer: netbird-acl.sh --verify --from-member" + fi + else + info "network probes skipped (--no-probe): the API state above is all that was checked" + fi + + printf '\n%s passed, %s failed\n' "$PASS_COUNT" "$FAIL_COUNT" + if [ "$FAIL_COUNT" -ne 0 ]; then + printf 'Acceptance: FAIL\n' + return 1 + fi + printf 'Acceptance (API side): PASS\n' + printf '\nStill to confirm by hand from the operator machine:\n' + printf ' netbird status --detail | grep -E "Peers count|Status:"\n' + if [ -n "$HUB_SSH" ]; then + printf ' ssh %s hostname # expect the hub host\n' "$HUB_SSH" + fi + if [ -n "$SANDBOX_SSH" ]; then + printf ' ssh %s hostname # expect the sandbox host\n' "$SANDBOX_SSH" + fi + printf ' curl -s http://%s:%s%s # read the BODY\n' "$HUB_PEER_IP" "$HUB_PORT" "$HEALTH_PATH" + printf 'And, from a %s peer once one has been enrolled:\n' "$GROUP_MEMBERS" + printf ' netbird-acl.sh --verify --from-member\n' +} + +# ---------------------------------------------------------------- main + +main() { + # Checked here, not at assignment, so --help still works without a hub.env and + # the message can name the file to edit. + require_var NETBIRD_API "$API_BASE" "NetBird management API base URL, including /api" + require_var LOOMGRAPH_HUB_IP "$HUB_PEER_IP" "mesh IP of the peer running lg-hub" + if [ "$FROM_MEMBER" = "yes" ] && [ -z "${MAC_PEER_IPS[0]:-}" ]; then + die "LOOMGRAPH_MAC_IPS is not set (the peers a member must NOT reach). --from-member proves nothing without them. Set it in $ENV_FILE." + fi + + case "$MODE" in + dry-run) log "MODE: DRY-RUN (no changes; pass --apply to converge)" ;; + apply) log "MODE: APPLY" ;; + verify) log "MODE: VERIFY (read-only)" ;; + esac + log "API: $API_BASE" + + load_token + refresh_state + + if [ "$MODE" = "verify" ]; then + verify + else + converge + fi +} + +main diff --git a/docs/hub-design.md b/docs/hub-design.md new file mode 100644 index 0000000..ba74fa0 --- /dev/null +++ b/docs/hub-design.md @@ -0,0 +1,1030 @@ +# The team hub — design and threat model + +Status: **proposal, not built.** Nothing in `src/hub/` exists yet. +Author decisions locked before this document was written are in [§1](#1-what-is-locked). +Open questions for the author: [§14](#14-open-questions-for-the-author). + +**Revision 2.** Two questions were put to an adversarial pair of reviews: should the hub +hold a database, and should it hold full conversations? The answers went in opposite +directions and both changed this document. Storage is now SQLite-as-truth +([§6](#6-hub-storage), decision D-3) — revision 1 was wrong, and the specific error is +recorded rather than quietly fixed. Central conversation storage is refused +([§18](#18-decision-record-d-3-and-d-4), decision D-4), on measured evidence from a real +transcript corpus. Three defects the pro-database review found in revision 1's briefs are +fixed: owner-curated brief depth ([§7.2](#72-brief-depth-is-the-owners-choice)), +redaction-on-read ([§7.3](#73-redaction-on-read)), and the admission that this design's +top-ranked threat is uninvestigable with the data it retains +([§9.6](#96-what-this-design-cannot-investigate)). Federated search replaces the central +corpus as the answer to deep search ([§17](#17-federated-search)). + +--- + +## 0. What stops being true + +Three sentences the project can say today, and will not be able to say after this ships: + +| Today | After the hub | +| --- | --- | +| "Not a workflow server. No daemon, no web UI, no cloud." | There is a daemon and a web UI. | +| "No signal bus, inbox, or daemon… an inbox that starts an agent on someone else's laptop is a different product with a much harder threat model." | This is that product. | +| Content leaves a machine only through a print-once, expiring link a human chose to send. | Content leaves on a schedule, to a box, and stays there. | + +That third one is the real change, and the second one is the warning being cashed. The +README already reasoned its way to *not* building this and named exactly why. Building it +is legitimate — it is the author's project and the author's call — but the "much harder +threat model" has to be actually built, not waived by the phrase "approval-gated." Most of +[§9](#9-threat-model) exists because approval-gating answers a question the attack does not ask. + +**What does not change:** loomgraph still makes zero model calls, still shells out to the +agent CLI you installed, and **the hub never runs an agent.** It stores and routes. Agents +run on member machines, under the member's own sandbox, started by the member. + +--- + +## 1. What is locked + +Decided by the author before design; not relitigated here. + +1. **The inbox is approval-gated.** An inbound message never executes anything. It queues. + A human runs `lg inbox accept ` for anything to happen. No auto-dispatch in v1. +2. **The hub ingests run events and distilled extractive briefs only.** No raw transcripts, + under any flag. `src/handoff/scan.ts` is a hard gate on ingest. +3. **One central hub** on a shared server, per-member bearer tokens, each member's CLI + talks to it. + +Locked decision 3 is the single largest risk multiplier in this design: it converts +per-laptop compromise into team-wide compromise, and creates aggregation leaks no +per-brief scanner can see ([§9.4](#94-what-the-scanner-stops-buying-under-retention)). It stays, +so the rest of the architecture compensates: hold as little as possible, never execute on +the hub, and put the real anti-injection defense on the *consuming* machine. + +--- + +## 2. Shape, in one paragraph + +The hub does to the team what `events.jsonl` already does to a run: a server-stamped, +append-only record of what happened, greppable via `lg-hub export --jsonl` and queryable via +SQL. The client's existing event log doubles as the push outbox, so there is no queue and no +spool directory. The existing pure-renderer pattern doubles as the web UI, so there is no +build step and no client JS. `src/handoff/scan.ts` sits in front of everything that ingests +text, and now in front of everything that serves it too ([§7.3](#73-redaction-on-read)). +Only the hub's own storage is a new idiom; the rest is the existing three pointed at a +network. + +--- + +## 3. Component boundaries + +**A third binary, `lg-hub`.** Not folded into `lg` — a long-running daemon would poison +`lg`'s "you run it and it exits" contract. Not folded into `lg-handoff` — that subtree's +import purity is load-bearing. + +``` +src/hub/ the lg-hub bin -> dist/hub/cli.js + cli.ts commander wiring; the only argv reader + server.ts node:http binding; deliberately dumb (see §12 on testing) + handlers.ts (WireRequest, deps) -> WireResponse; where all behavior lives + auth.ts token hashing, member resolution, revocation + storage.ts HubStore: node:sqlite, WAL, one transaction per batch (§6) + inbox.ts message lifecycle transitions + feed.ts feed partitioning and cursor logic + ui/*.ts pure (data) -> html renderers +src/team/ client side + transport.ts the injected Fetch seam (mirrors handoff's Exec seam) + sync.ts cursor logic over events.jsonl + fence.ts untrusted-content fencing <- security-critical, see §8.3 +src/commands/ new thin files: enroll.ts, sync.ts, inbox.ts, wired into src/cli.ts +``` + +**Import rule.** `src/hub/` may import `src/core/` and may import `src/handoff/scan.ts` +one-way. `src/handoff/` still imports nothing outward, so AGENTS.md's rule holds as +written. **Do not copy the scanner.** The `buildEnclavePushArgs`-exists-twice precedent is +for a 20-line argv builder; a security gate must never fork. AGENTS.md needs a line saying +which direction the new arrow points. + +Note what this costs: `src/handoff/types.ts:1-8` keeps the subtree extractable "once a team +fabric exists outside this repo." The fabric now exists *inside* it, so extraction later +means the hub depends on the extracted sibling. Acceptable, but the comment should be +updated rather than left to quietly become false. + +**Exit codes.** `lg`'s team verbs reuse its namespace — sync/inbox failure is `2`, never +`3` or `4`, which stay budget and paused. `lg-hub` gets its own small namespace, per the +`lg-handoff` precedent: `0` clean exit, `1` config or usage, `2` fatal runtime (port bind, +corrupt data dir). + +--- + +## 4. Wire protocol + +HTTP/1.1 + JSON on `node:http`. Client uses global `fetch` (Node ≥ 22) behind the seam. +Zero new dependencies. `Authorization: Bearer ` on everything except `/v1/health`. + +| Endpoint | Method | Notes | +| --- | --- | --- | +| `/v1/health` | GET | unauthenticated; `{ok, version}` | +| `/v1/events` | POST | `{runId, streamId, graphName, state, events[]}` — `state` is a **projection**, see [§4.1](#41-what-a-pushed-state-omits); `events[]` are raw JSONL lines, ordered by `seq` | +| `/v1/briefs` | POST | the four bundle files inline as strings | +| `/v1/feed?after=&limit=50` | GET | newest-first page + `nextCursor`; keyset, see [§6.2](#62-pagination-is-keyset-not-byte-offsets) | +| `/v1/runs/:member/:runId` | GET | stored state + events | +| `/v1/inbox` | POST | send; schema in [§8](#8-the-inbox) | +| `/v1/inbox?state=queued` | GET | addressee is always the authenticated member | +| `/v1/inbox/:id/transition` | POST | `{to, runId?}` | +| — | — | **there is no admin HTTP route.** Member management is CLI-only on the hub host ([§5](#5-identity-and-auth)) | + +### 4.1 What a pushed state omits + +`RunState` is not safe to push as-is, and revision 1 said otherwise by implication. Two of +its fields carry content, not status: + +- `vars: Record` — whatever was passed to `--var`, which is exactly where a + ticket id, an internal URL or a token ends up. +- `nodes[].output: unknown` — the agent CLI's raw stdout, verbatim. + +So [§11](#11-deletion--decision-d-2-revised)'s claim that events "carry status, cost and +timing — not content" is true of the event stream and **false of the checkpoint**. The client +therefore pushes a projection, built on the member's machine before anything leaves it: + +- `vars` becomes `varKeys: string[]` — the key names only. Not a map with the values nulled + out: a nulled slot is somewhere a later change can put a value back, and nothing would + fail when it did. A list of key names has nowhere to put a value at all. The key names are + the useful part for monitoring; the values are the risk, so the shape that can carry them + should not exist. +- `nodes[].output` is dropped entirely. Status, attempts, timing and cost stay; `error` + is kept but **masked and truncated** — see the paragraph below. +- `cwd` is rewritten through the existing `rewritePaths` before it is sent, so an absolute + home path does not become a team-readable field. + +`error` is kept but sanitised, not copied. The adapters build it from exactly the material this +section refuses to publish: `src/adapters/claude.ts:33` embeds 200 characters of raw stdout, +`claude.ts:113` and `codex.ts:120` append stderr, and `command.ts:40` falls back to the prompt +itself. So on projection an error has its paths rewritten, every scanner-known secret shape +replaced by a masked prefix, and its length capped at 200 characters. Dropping it outright would +cost real monitoring value; publishing it raw would undo the rest of this section. + +A batch names its run twice — once at the top level and once inside the projected state — +so the hub **rejects any batch where the two disagree** rather than picking a winner. Left +unchecked, a client could push `runId: "A"` carrying a state describing run `B`, and the hub +would store a row whose status, cost and node table belong to a different run entirely: a +wrong answer that never errors. Identity on the wire is single-sourced by refusal, not by +precedence. + +This is minimization at the source, the same principle as "the readers drop, they do not +carry." A hub that never receives the value cannot leak it, cannot be asked to mask it, and +cannot retain it by accident. If a run's output genuinely needs sharing, that is what a brief +is for — scanned, curated, and revocable. + +**Idempotency uses natural keys, not an `Idempotency-Key` header.** Events already carry +`(runId, seq)` from `src/core/events.ts`. The hub keys them `(member, streamId, runId, seq)` +where `member` comes from the token and **never** from the body. A per-run high-water mark +acks-and-drops anything at or below it. Same seq with different content is `409` plus a +visible feed item — silence about divergence is worse than noise. Briefs are keyed by +`sha256(handoff.md)`. Inbox messages carry a client `crypto.randomUUID()`. + +**Reserve `streamId` in phase 1 even though nothing reads it yet.** It is a random id +minted at `run_started`. Without it, `(runId, seq)` assumes one machine and one history per +run forever; a wiped `.loomgraph`, a copied repo directory, or any future multi-machine +resume produces same-key-different-content and the 409 policy fires noise exactly when the +user is already confused. Reserving the field now is free. Retrofitting it after real data +exists is not. + +**The outbox already exists — do not build one.** `.loomgraph/runs//events.jsonl` is +append-only and unbuffered by hard rule, which is the definition of a durable outbox. Sync +is a cursor over it: + +- `.loomgraph/sync/.cursor` holds the last acked seq, written temp-then-rename. +- `lg run` / `lg resume` hook the **existing** `onEvent` callback already threaded through + `EngineDeps` and used in `src/commands/run.ts`. Batch every 10 events or 5 s, 1500 ms + timeout, and **any failure is one line on stderr and nothing else.** A hub outage cannot + affect a run, its checkpoints, or its exit code. +- `lg sync [runId]` replays from the cursor. **This is the only path that must be correct.** + The live push is best-effort sugar over it. + +The cursor advances only on a 2xx naming `highWaterSeq`, so a cut connection just resends. + +**Ordering is by hub `receivedAt`, never by client `ts`.** Client clocks skew; the feed is +served from the hub's own arrival order. Client timestamps are displayed and labeled as +reported, and no cursor is ever derived from one ([§6.2](#62-pagination-is-keyset-not-byte-offsets)). + +--- + +## 5. Identity and auth + +Enrollment is admin-mediated and print-once, matching the enclave share-link aesthetic the +project already lives with: + +``` +# on the hub host +$ lg-hub member add alice +lgt_a1b2c3d4.<32 bytes base64url> # printed once, never recoverable + +# on alice's machine +$ lg enroll https://hub.internal lgt_a1b2c3d4.xxxx +wrote ~/.config/loomgraph/hub.json (0600) +``` + +The hub stores only `{member, keyId, tokenHash: sha256(secret), scopes, createdAt}` in the +`members` table ([§6.3](#63-schema)). Revision 1 said `members.jsonl`; the table supersedes +it, and there is no members file. `LOOMGRAPH_HUB_URL` / `LOOMGRAPH_HUB_TOKEN` override the file, the same +pattern as `ENCLAVE_TOKEN`. + +**Attribution is server-side, always.** Every stored record gets `member` stamped from the +token's keyId. `HandoffMeta.createdBy` — currently `userInfo().username` in +`src/handoff/commands.ts` — is displayed as *"claims created-by"* at most. Never trust a +client-supplied owner field; that is forgery vector A5. + +**Revocation** sets `members.revoked_at`, and a revoked token resolves to no member on the +next request — no restart, no SIGHUP, no replay. + +**Add a scan rule for the hub token shape before the first token is minted.** This is +non-optional and easy to forget. Members will paste tokens into shells and configs; agents +will read those shells; `lg-handoff pack` will faithfully distil a session quoting one. +The pipeline is *designed* to republish exactly this, and `SCAN_RULES` in +`src/handoff/scan.ts` has no rule for a shape that does not exist yet. Choose the `lgt_` +prefix, add the rule in the same commit. + +**Transport.** `lg-hub serve` refuses to bind a non-loopback address unless +`--behind-tls-proxy` is passed, and says why. Deploy behind Caddy, or on a WireGuard or +Tailscale interface. Bearer tokens over plaintext LAN HTTP are precisely the credential +class `scan.ts`'s `auth-header` rule exists to catch; the project should not ship the +vulnerability its own scanner names. + +--- + +## 6. Hub storage + +**SQLite (WAL, `node:sqlite`) is the hub's truth. JSONL is a derived export.** + +Revision 1 said the opposite, and was wrong in a way worth recording rather than silently +correcting. + +### 6.1 Why revision 1 was wrong + +Revision 1 did not choose JSONL *over* SQLite. It chose **both**: JSONL as truth, plus a +SQLite index, plus `lg-hub reindex`, plus a phase-4 test proving the rebuild was +byte-identical. That is two storage engines, two write paths, and a consistency proof +between them — assembled to avoid one engine that ships inside Node 22. Simplicity was the +stated goal and was not what the design delivered. + +Three specific errors: + +- **"Append-only is a physical property" was false.** Nothing physically prevents `sed -i` + on a `.jsonl` file. Append-only-ness of a file is discipline too. SQLite enforces it + *harder*, because the prohibition can be declared: + `CREATE TRIGGER … BEFORE UPDATE ON events BEGIN SELECT RAISE(ABORT,'append-only'); END;` +- **Tamper-evidence against the hub operator was zero in both designs.** An operator with + root rewrites a JSONL line as easily as a row. The real mechanism is a hash chain, which + revision 1 did not have; it is now a column. +- **`EventLog.read` skipping unparseable lines is correct on a laptop and wrong as server + truth.** `src/core/events.ts` states the intent plainly — "A torn or corrupt line must + not take down the audit trail" — which on a laptop is graceful degradation. As the + server's only copy it means a torn line silently deletes an event from history, and a + rebuild bakes the loss in. Loud corruption beats silent loss. + +**What does not change: the laptop.** `.loomgraph/runs//events.jsonl` stays exactly +as it is, unbuffered and append-only. AGENTS.md's invariant is about the run log, and the +run log is where the greppable-log property actually lives; revision 1 mistakenly read that +rule as binding on a component that did not exist when it was written. The hub is a +different component with a different job. **AGENTS.md needs one added line saying so**, and +saying that the hub's greppable artifact is a derived export, not its truth. + +### 6.2 Pagination is keyset, not byte offsets + +Revision 1's cursor was `base64({day, offset})` — a byte offset into a day-partitioned +file, handed to clients who hold it indefinitely. That is a public API made of the wrong +material: + +- `reindex`, the design's own recovery mechanism, invalidated every outstanding cursor + unless the rebuild was bit-perfect — which is precisely why that test had to exist. The + escape hatch and the pagination scheme were at war. +- Tombstoning ([§11](#11-deletion--decision-d-2-revised)) shifts offsets, and compaction was + rejected, so clients would page through tombstones forever. +- **A wrong byte offset is undetectable.** The client lands mid-line, or skips items, or + repeats them, and nothing errors. + +Cursors are now keyset over `(received_at, rowid)`, which survives rebuilds, retention +purges, schema evolution and reordering. + +### 6.3 Schema + +```sql +PRAGMA journal_mode=WAL; +PRAGMA foreign_keys=ON; +PRAGMA user_version=1; -- bumped by any later column addition; phases 2-4 add tables + +-- the verbatim client line is kept in `json`, so the export in §6.4 is lossless +CREATE TABLE events ( + member TEXT NOT NULL, stream_id TEXT NOT NULL, run_id TEXT NOT NULL, seq INTEGER NOT NULL, + received_at TEXT NOT NULL, kind TEXT NOT NULL, node_id TEXT, + json TEXT NOT NULL CHECK (json_valid(json)), -- the client's line, byte-for-byte + prev_hash BLOB, row_hash BLOB NOT NULL, + UNIQUE (member, stream_id, run_id, seq) +); -- an ordinary rowid table: §6.2's cursor needs rowid to exist +CREATE INDEX events_feed ON events(received_at); -- see the note below on rowid + +-- the hash chain is global and single-writer; the head is updated inside the ingest +-- transaction so it can never disagree with the rows +CREATE TABLE chain_head (id INTEGER PRIMARY KEY CHECK (id = 1), head BLOB NOT NULL); + +CREATE TABLE runs ( + member TEXT NOT NULL, run_id TEXT NOT NULL, stream_id TEXT NOT NULL, + graph_name TEXT, state_json TEXT, high_water_seq INTEGER NOT NULL, updated_at TEXT NOT NULL, + PRIMARY KEY (member, run_id)); + +CREATE TABLE briefs ( + brief_id TEXT PRIMARY KEY, member TEXT NOT NULL, sha256 TEXT UNIQUE NOT NULL, + received_at TEXT NOT NULL, expires_at TEXT, revoked_at TEXT, + key_id TEXT REFERENCES item_keys(key_id), -- null only if encryption is off + handoff_md BLOB, meta_json BLOB, html BLOB); +CREATE TABLE brief_files (brief_id TEXT, path TEXT, PRIMARY KEY (brief_id, path)); +CREATE TABLE brief_shares (brief_id TEXT, grantee TEXT, granted_at TEXT, revoked_at TEXT, + PRIMARY KEY (brief_id, grantee)); + +CREATE TABLE inbox ( + id TEXT PRIMARY KEY, from_member TEXT NOT NULL, to_member TEXT NOT NULL, + subject TEXT, body TEXT, re_json TEXT, proposed_graph TEXT, + state TEXT NOT NULL, created_at TEXT NOT NULL); +CREATE TABLE inbox_history (id TEXT, to_state TEXT, ts TEXT, by TEXT, run_id TEXT); + +CREATE TABLE members ( + key_id TEXT PRIMARY KEY, member TEXT NOT NULL, token_hash TEXT NOT NULL, + scopes TEXT NOT NULL, created_at TEXT NOT NULL, revoked_at TEXT); +CREATE TABLE sessions (sid TEXT PRIMARY KEY, member TEXT, expires_at TEXT); +CREATE TABLE read_marks (member TEXT, kind TEXT, ref TEXT, read_at TEXT, + PRIMARY KEY (member, kind, ref)); +CREATE TABLE access_log (ts TEXT, member TEXT, action TEXT, ref TEXT); +CREATE TABLE item_keys (key_id TEXT PRIMARY KEY, wrapped_key BLOB NOT NULL); + +CREATE VIRTUAL TABLE search USING fts5(member, kind, ref, text); + +CREATE TRIGGER events_no_update BEFORE UPDATE ON events + BEGIN SELECT RAISE(ABORT, 'events is append-only'); END; +CREATE TRIGGER events_no_delete BEFORE DELETE ON events + BEGIN SELECT RAISE(ABORT, 'events is append-only'); END; +``` + +Everything revision 1 hand-rolled becomes a declared constraint: the high-water mark is a +primary key, 409-on-divergence is `INSERT OR IGNORE` plus a compare-on-conflict, the feed +is an index, receipts are columns, the members-file replay is a table, the §11 tombstone is +`revoked_at`. One ingest batch is one transaction — it happened or it did not — replacing +revision 1's four-file interleaving that had to be reasoned about by hand. + +`row_hash = sha256(prev_hash || json)` where `prev_hash` is the value in `chain_head`, +genesis being 32 zero bytes; the head advances in the same transaction as the insert. The +head is published to members periodically. This is the tamper-evidence revision 1 claimed +from file semantics and did not actually have. + +**`events` is written with `INSERT … ON CONFLICT DO NOTHING`, never `INSERT OR REPLACE`.** +`OR REPLACE` is a delete followed by an insert, so it fires the append-only delete trigger +and aborts the transaction. The shortest path to a green test at that point is deleting the +trigger, which is why this is written down here rather than left to be discovered. + +`WITHOUT ROWID` was in revision 1 and is removed: SQLite gives such tables no `rowid` +column, which the keyset cursor in [§6.2](#62-pagination-is-keyset-not-byte-offsets) +requires. Verified against this machine's `node:sqlite` — `SELECT rowid` on a +`WITHOUT ROWID` table fails with `no such column: rowid`. + +The index on `events` is `(received_at)` alone, for the same reason and verified the same way: +`rowid` is not referenceable inside an index expression, so +`CREATE INDEX ... ON events(received_at, rowid)` fails with `no such column: rowid` on SQLite +3.51.3. It is also unnecessary - every SQLite index on a rowid table implicitly ends in `rowid`, +so `(received_at)` already gives the `(received_at, rowid)` ordering +[§6.2](#62-pagination-is-keyset-not-byte-offsets) needs. Two forms of the same mistake: do not +write `rowid` into a `WITHOUT ROWID` table, and do not write it into an index. + +### 6.4 The greppable artifact survives as an export + +`lg-hub export --jsonl` emits exactly revision 1's directory layout — `events.jsonl` per +run, one JSON object per line — reconstructed from the `json` column, which holds the +verbatim client line. The grep audience loses nothing; the query audience gains +`lg metrics`, full-text search, unread state, threading, revocable share grants and +multi-day range queries, none of which are reachable by walking files. + +The export has two modes and neither re-encodes a line. `--out ` writes the layout above, +`runs///events.jsonl`, so identity lives in the path rather than in an envelope. +`--jsonl` writes the raw lines to stdout for grepping, where identity is simply not +representable - a flat stream cannot carry it without corrupting the line, and corrupting the +line is the one thing this export exists not to do. + +**The law for AGENTS.md, inverted from revision 1:** the hub's database is truth; JSONL is +a rebuildable export. The laptop's `events.jsonl` is untouched and remains append-only. + +### 6.5 Operations + +- **Backup** is `VACUUM INTO 'snap.db'` — one statement, consistent. Revision 1's + "`rsync` the data directory" was a live copy of dozens of files mid-write. +- **Recovery** at 2am: `PRAGMA integrity_check`, then restore the last snapshot and replay + from members' local cursors, which are the real durable outbox ([§4](#4-wire-protocol)) + and are unaffected by hub state. +- **Migrations** are cheap because the event payload is an opaque verbatim `json` column; + new client fields need no `ALTER TABLE`. Only hub-side projections migrate. +- **`engines.node` must be `>=22.13`.** `node:sqlite` is behind `--experimental-sqlite` + before 22.13.0, so the current `>=22` would let `lg-hub` crash at import on 22.0–22.12. + It also emits an `ExperimentalWarning` on import at every version tested; the `lg-hub` + bin filters that one warning before importing the store, which requires the import to be + dynamic. Stdout is unaffected either way — the warning goes to stderr. +- **Postgres is not warranted.** One process, ten members at most, embedded synchronous + access, zero-dependency ethos. Revisit at multiple hub nodes or roughly fifty members; + arguing for it now would only discredit the SQLite case. + +## 7. Visibility — decision D-1 + +**The two memos disagreed here, and this is the resolution.** + +The architecture memo said every member reads everything; right-sized for a small team. +The threat memo said private-by-default with explicit scoped sharing, because +`lg-handoff` is private-only and *refuses* `--visibility org`, and because a flat pool +means one leaked token or one XSS drains the whole team's briefs. + +**Resolution: the push is the sharing decision.** + +- Nothing reaches the hub that a member did not push. Sync is **opt-in per repository** + (`lg sync --enable` writes `.loomgraph/hub.json`), never on by default, never global. + A member who never enables sync is invisible to the hub, and that must stay true. +- **Run events, once pushed, are team-readable.** This is the monitoring feature the author + asked for, and enabling sync on a repo is the consent act. Making pushed runs private + would make the feature pointless. +- **Briefs are private to the sender until explicitly shared** to named members, revocably. + A brief is quoted session content — a different asset class from a status table. +- **An inbox message is readable only by its sender and its addressee.** No broadcast. + +So the threat memo wins on briefs and inboxes; the architecture memo wins on run events; +and the granularity that makes both defensible is per-repo opt-in rather than per-item +prompting. What was given up: a member cannot enable sync on a repo and then hide one +embarrassing run in it. Retraction ([§11](#11-deletion--decision-d-2-revised)) is the answer to +that, not per-run visibility flags. + +**Token scopes** (`ingest` / `read` / `admin`) are separate from this and should land by +phase 3. A CI token that pushes events should not be able to read everyone's briefs or send +inbox messages. + +### 7.1 What "conversations" means here — decision D-4 + +The author asked whether the hub should hold a database to share **all conversations** +between members. The database half is [§6](#6-hub-storage); this half is refused, and the +evidence is in [§18](#18-decision-record-d-3-and-d-4). The short form: on the author's own +machine, 61% of real agent transcripts contain a credential shape the existing scanner +already recognises, which is a floor rather than an estimate. Centralising transcripts +means roughly six in ten uploads carrying a known credential shape, permanently, on one +shared box, readable by everyone with a token. + +The counter-design — per-session opt-in, encryption at rest, short retention, access +logging, redaction-on-read — was argued well and defeated by its own requirement: server-side +search, redaction and rendering all need the hub to hold decryptable plaintext, so it +mitigates every threat except A4 while materially raising A4's payoff. Its author's summary +of the position was "I chose the honeypot." On a shared server, that is the wrong choice. + +**What is kept from that argument is [§7.2](#72-brief-depth-is-the-owners-choice), +[§7.3](#73-redaction-on-read), [§9.6](#96-what-this-design-cannot-investigate) and +[§17](#17-federated-search)** — because the objection that briefs are too thin was correct +even though the proposed remedy was not. + +### 7.2 Brief depth is the owner's choice + +Revision 1 inherited `lg-handoff`'s fixed extraction: `firstTurn(user)`, `lastTurn(assistant)`, +`lastTurn(user)`, plus a file list. A sixty-turn session becomes three quoted turns, and the +load-bearing one — Done — is **the agent's summary of its own work**, which the README's own +failure-mode section teaches you to distrust: Claude Code returns `subtype: "success"` with +`is_error: true` on a lapsed session, and a sandboxed verifier can report PASS having read +nothing. The brief keeps the claim and discards the evidence, then tells the reader to +verify every claim against the repo. + +That makes a fixed-shape brief the worst point on the curve: most of the retention risk, +little of the value. The fix is not more content by default — it is letting the person who +was there choose: + +```bash +lg-handoff pack claude --turns 12-31,44 --include-tool-result 27 --session-file +``` + +Still extractive, still no model call, still scanned, still owner-curated. What changes is +that the dead end at turn 23 — "we tried patching `disburse.ts` first and it broke +reconciliation" — can be carried, because that sentence is worth more to the next person +than the summary is. `--turns` without an explicit list keeps today's default. + +The turns and tool-result blocks a reader can request are bounded by what the reader +already extracts; **this does not loosen "the readers drop, they do not carry."** Adding a +field to `DistilledSession` still means deciding it is safe to publish. + +### 7.3 Redaction-on-read + +Scanning only at ingest means a rule added later protects nothing already stored. The +`lgt_` token rule from [§5](#5-identity-and-auth) is the worked example: any token that +leaked before that rule existed is exposed for as long as the store keeps it. + +So stored content is also served through `scanText` + `rewritePaths` masking **at egress**, +on every read path — API, web UI and export. Consequences worth stating: every rule added +in future retroactively protects all history; a finding at read time is logged and surfaced +to the owner rather than silently masked; and the ingest gate stays exactly as it is, since +egress masking is a second layer and not a replacement for refusing to store a secret. + +Cost: reads are no longer a straight file copy. At this data volume that is not a +performance question. + +--- + +## 8. The inbox + +### 8.1 Message schema + +``` +{ v: 1, id: uuid, from: , to: { member: "bob" }, + subject: string, body: string, + re: { member, runId } | { briefId } | null, + proposedGraph: { source: , vars: {...} } | null, + createdAt, state, history: [{to, ts, by, runId?}] } +``` + +**Addressing is person-only.** A repo has no owner who can approve; a run has no inbox. +Both exist only as the optional `re:` context reference. Repo-addressing is the feature +that quietly turns this into a dispatch system, so it is deliberately absent. + +**Ingest gates**, in this order, fail-closed, mirroring `pushCommand`: `scanText` over +`subject`, `body`, `proposedGraph.source` and every var value — reject with masked findings +on any hit; then `parseGraph` on any `proposedGraph`, rejecting invalid graphs at send time +so an acceptor never receives an unrunnable request. Validation stays loud, per AGENTS.md. + +**Lifecycle:** `queued → seen → accepted | declined | expired`, then +`accepted → done | failed`, reported by the acceptor's own sync. Only the addressee's token +may transition its own messages. + +### 8.2 What `accept` actually does — decision D-1b + +The memos disagreed here too. The architecture memo had `accept` write the sender's graph +via `saveGraphSource` and enter the normal `runCommand` path. The threat memo said a +message must never be able to name the task, because that is attack A2 with a green light. + +**Resolution: the acceptor names the graph. The message is only ever data inside it.** + +``` +$ lg inbox show 7f2a # mandatory reading step; see §8.3 +$ lg inbox accept 7f2a --graph ./graphs/triage.yaml +``` + +`--graph` points at a **local file the acceptor already has and trusts.** The message body +is exposed to that graph only as `{{inbox.body}}`, which is materialized pre-fenced +([§8.3](#83-fencing-is-the-load-bearing-control)). The sender's `proposedGraph` is inert by +default; running it requires `--use-proposed-graph`, which prints the full YAML plus +`renderPlan` and requires typing the message id to confirm. There is no flag that skips +`show`, and **there must never be an `--auto-accept`, a trusted-sender bypass, or an +accept triggered by an event.** + +`--cwd` is always the acceptor's. A message may name a repo *remote* as a suggestion and +can never name a local path. + +Inbox-sourced runs default to the most restricted sandbox available and never inherit +`workspace-write` or `bypass`. A message cannot name its own execution mode — sender-supplied +capability is the whole attack with permission attached. + +Progress flows back to the sender through ordinary event sync. No new mechanism. + +### 8.3 Fencing is the load-bearing control + +**This is the most important section in this document.** + +An inbox message is untrusted input authored by someone else's agent, which may itself have +been steered by a web page, a dependency README, or a PR body it read. Approval-gating is a +boolean on *ingestion*; the exploit is in *interpretation*. A human clicking accept is +saying "this looks like real work from a colleague," not auditing an instruction set they +were never shown as an instruction set. Habits decay into muscle memory within a week. + +So `src/team/fence.ts` wraps every inbox-sourced value before any agent CLI sees it: + +- an explicit, un-spoofable delimiter, with delimiter-lookalikes in the body neutralized; +- a preamble stating the content is untrusted third-party data and instructions inside it + are not to be followed; +- every line prefixed, reusing the discipline in `src/handoff/render.ts` — whose `quote()` + exists so "no line of transcript can break out of the quote," and which deliberately + ships **no markdown engine** because an inline-link parser is a way to smuggle + `javascript:` into a page. The same reasoning applies verbatim to inbox content. + +`lg inbox show` renders with the same fence: sender, source run, timestamp, and the entire +body — untruncated — inside a visible quarantine frame labeled *"untrusted message from +<member>; loomgraph did not write this and cannot vouch for it."* No link activation, +no markdown, escape everything. + +**Say plainly, in the README and in `show`'s own output, that fencing is mitigation and not +proof.** Nothing at the prompt layer is a hard boundary against a determined injection. +Fencing lowers the odds, the restricted sandbox bounds the blast radius, and the human is +the last check — the same posture the scanner section already takes. + +--- + +## 9. Threat model + +### 9.1 New trust boundaries + +Today there are two: transcript → readers (the narrowing boundary in +`src/handoff/readers/*.ts`), and bundle → enclave (scan, then constraints, then spawn, in +`pushCommand`). The hub adds: + +- **B1 member → hub.** Every run now has a network side effect, on a schedule, not per + human decision. +- **B2 hub → member.** Entirely new direction. `lg-handoff` has "No pull" as a design + point; this deletes it. +- **B3 member ↔ member, transitively.** Any teammate can author input to my machine. Since + teammates run agents, this is really: **anything any teammate's agent ever read** can + author input to my machine. +- **B4 browser ↔ hub.** A client class with cookies, a DOM, and adversarial text to render. +- **B5 storage at rest.** Aggregated team content, long-lived, on one box. A new asset class. +- **B6 hub operator and co-tenants.** Rooting my laptop gets you my sessions. Rooting the + hub gets you the team's. +- **B7 token custody.** A new secret on N machines — and one the handoff pipeline is built + to accidentally republish ([§5](#5-identity-and-auth)). + +### 9.2 Attack paths, ranked + +**A1 — prompt-injected teammate → my inbox → my agent. (High × Critical; not close.)** +Teammate's agent reads a poisoned page, is instructed to send a hub message, the message +queues, I accept because it reads like plausible colleague work, my agent consumes it as +instructions. Sender authenticated, transport intact, human approved: **every planned +control passes and the attack still lands.** Mitigated only by [§8.3](#83-fencing-is-the-load-bearing-control) +fencing + acceptor-named graph + restricted sandbox. This is what the README's warning was about. + +**A2 — malicious accept. (High × Critical.)** A1's mechanism restated, because +approval-gating is designed as the defense against it and does not defend against it. +Accept gates whether a message enters the workflow; it says nothing about what the message +says once in. Approval authorizes the topic; the payload is in the details. + +**A3 — compromised member token. (Medium × High.)** Possession equals identity, replayable +until noticed. Grants reads per [§7](#7-visibility--decision-d-1) plus forged messages to +every other member — feeding A1 from an authenticated sender, which clears reputation +checks. Uniquely here, the token can leak *through loomgraph's own handoff pipeline*. + +**A4 — compromised hub. (Low × Catastrophic.)** Read everything ingested, impersonate +anyone, inject into every inbox with no injection needed, rewrite the log. The mitigation is +not "trust the hub" — it is that the hub holds as little as possible and cannot itself +execute, which is why A1's real defense lives on the consuming machine. + +**A5 — replay or forgery of events. (Medium × Medium-High.)** Forged `run_finished`, +resurrected states, spoofed run ids. Corrupts the shared record and anything keyed off it. +Countered by server-side attribution and the natural-key high-water mark. + +**A6 — XSS from brief content. (Medium-High × High.)** Briefs are arbitrary quoted model +and user text by construction, rendered in an authenticated origin holding the team's data. + +**A7 — scanner miss, retained forever.** See below. + +### 9.3 Why the scanner still earns its place + +Keep it as a mandatory, **server-side, non-bypassable** ingest gate. It genuinely catches +URL-embedded credentials, `Authorization` headers, vendor-prefixed keys, JWTs, and +`TOKEN=`-style assignments, and — the part that matters most — `scanBundleDir` **fails +closed** via `UNREADABLE_RULE`, so "clean" means "looked and found nothing." Re-run it on +the server even when the client claims clean. + +### 9.4 What the scanner stops buying under retention + +Retention inverts the cost of a false negative. Under handoff a miss sat behind a link that +expired in 7 days and could be revoked. On the hub it sits indefinitely, readable by +everyone [§7](#7-visibility--decision-d-1) admits. Every named gap — AWS secret access +keys, header-less PEM bodies, hex client secrets, non-home absolute paths — becomes +permanent team-wide exposure instead of a week-long single-recipient one. + +And aggregation creates leak shapes that are in no single brief, which a line-oriented +single-file scanner structurally cannot see: + +- **The hub token itself**, until the rule from [§5](#5-identity-and-auth) exists. +- **Cross-brief correlation** — a hostname here, a username there, a ticket scheme in a + third. Individually beneath notice; together, a map of the team's infrastructure. +- **The org graph.** `meta.json` carries `createdBy`, `createdAt`, and + `repo.remote/sha/branch`. Across a team that is an accurate timestamped record of who + touched what in which private repo. No rule flags it, and it is exactly what a departing + employee or an attacker wants. +- **`files.txt` as a source-tree map.** The union of `filesTouched` sketches private + codebases' structure. + +Mitigation is minimization, not more rules: ingest the least identity that works, make +`repo.remote` and `files.txt` opt-in, and be able to actually delete. + +### 9.5 Web UI surface + +Requirements, all v1: contextual escaping on every interpolated value (the existing +`escapeHtml` in both renderers); **no markdown engine**, matching +`src/handoff/render.ts`'s stated rationale; no `innerHTML` on any brief-derived value; +strict CSP `default-src 'none'` with no inline script, so a missed escape cannot execute; +``, already present in the handoff page; +`X-Frame-Options: DENY` and `frame-ancestors 'none'` against clickjacking. + +Note `svg` is in `ENCLAVE_ALLOWED_EXTENSIONS` and SVG is an XSS vector (inline ` + + +`; diff --git a/src/hub/wire.test.ts b/src/hub/wire.test.ts new file mode 100644 index 0000000..b4759f6 --- /dev/null +++ b/src/hub/wire.test.ts @@ -0,0 +1,737 @@ +import { describe, expect, it } from "vitest"; +import { + MAX_BODY_BYTES, + NO_EVENTS_YET, + eventBatchSchema, + type EventBatch, + type IngestConflict, + type IngestResult, + type ProjectedNode, + type ProjectedState, + type RunRow, +} from "./wire.js"; + +const rawLine = + '{"ts":"2026-08-25T00:00:00.000Z","runId":"run-1","seq":0,"kind":"run_started","data":{"graph":"g"}}'; + +function baseState(): ProjectedState { + return { + runId: "run-1", + graphName: "g", + status: "running", + createdAt: "2026-08-25T00:00:00.000Z", + updatedAt: "2026-08-25T00:00:01.000Z", + cwd: "/work", + varKeys: [], + budget: { maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 10 }, + spent: { usd: 0, wallClockSec: 0, nodeRuns: 0 }, + nodes: {}, + completed: [], + seq: 1, + }; +} + +function baseBatch(overrides: Partial = {}): EventBatch { + return { + runId: "run-1", + streamId: "s-1", + graphName: "g", + state: baseState(), + events: [rawLine], + ...overrides, + }; +} + +describe("eventBatchSchema", () => { + it("rejects a batch whose events elements are objects rather than strings", () => { + const batch = baseBatch({ + events: [JSON.parse(rawLine) as unknown as string], + }); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + + it("rejects an events element that is not valid JSON", () => { + const batch = baseBatch({ events: ["this is {not json"] }); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + + it("accepts a realistic raw line unchanged and projects its seq", () => { + const batch = baseBatch(); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(true); + if (!result.success) return; + const [line] = result.data.events; + expect(line).toBe(rawLine); + expect(JSON.parse(line ?? "").seq).toBe(0); + }); + + it("accepts a line carrying an unknown top-level key and returns it byte-identically", () => { + const line = rawLine.slice(0, -1) + ',"futureKey":{"x":1}}'; + const batch = baseBatch({ events: [line] }); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.events[0]).toBe(line); + }); + + it("rejects a line that is missing seq", () => { + const line = rawLine.replace(',"seq":0', ""); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(false); + }); + + it("rejects a line whose kind is not one of the known kinds", () => { + const line = rawLine.replace('"kind":"run_started"', '"kind":"run_exploded"'); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(false); + }); + + it("rejects a state carrying an unknown extra key", () => { + const state = { ...baseState(), vars: {} }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects a ProjectedNode carrying an output key", () => { + const state = baseState(); + state.nodes = { + n1: { + nodeId: "n1", + status: "succeeded", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: "2026-08-25T00:00:01.000Z", + attempts: 1, + error: null, + costUsd: 0.1, + output: "should not be here", + } as ProjectedNode & { output: string }, + }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects a batch whose state.runId differs from its top-level runId", () => { + const batch = baseBatch({ state: { ...baseState(), runId: "run-B" } }); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.runId")).toBe(true); + }); + + it("rejects a batch whose state.graphName differs from its top-level graphName", () => { + const batch = baseBatch({ state: { ...baseState(), graphName: "other" } }); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.graphName")).toBe( + true, + ); + }); + + it("reports both problems when runId and graphName both disagree", () => { + const batch = baseBatch({ + runId: "run-A", + graphName: "gA", + state: { ...baseState(), runId: "run-B", graphName: "gB" }, + }); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(false); + if (result.success) return; + const paths = result.error.issues.map((i) => i.path.join(".")); + expect(paths).toContain("state.runId"); + expect(paths).toContain("state.graphName"); + }); + + it("rejects a batch whose nodes map key differs from the node's nodeId", () => { + const state = baseState(); + state.nodes = { + alpha: { + nodeId: "BETA", + status: "running", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: null, + attempts: 1, + error: null, + costUsd: 0.1, + }, + }; + const result = eventBatchSchema.safeParse(baseBatch({ state })); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.nodes.alpha.nodeId")).toBe( + true, + ); + }); + + it("reports every mismatched nodes entry, not just the first", () => { + const state = baseState(); + state.nodes = { + alpha: { + nodeId: "BETA", + status: "running", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: null, + attempts: 1, + error: null, + costUsd: 0.1, + }, + gamma: { + nodeId: "delta", + status: "succeeded", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: "2026-08-25T00:00:01.000Z", + attempts: 1, + error: null, + costUsd: 0.1, + }, + }; + const result = eventBatchSchema.safeParse(baseBatch({ state })); + expect(result.success).toBe(false); + if (result.success) return; + const paths = result.error.issues.map((i) => i.path.join(".")); + expect(paths).toContain("state.nodes.alpha.nodeId"); + expect(paths).toContain("state.nodes.gamma.nodeId"); + }); + + it("tells a not-JSON line apart from a JSON-but-invalid-event line", () => { + const notJson = eventBatchSchema.safeParse(baseBatch({ events: ["this is {not json"] })); + expect(notJson.success).toBe(false); + if (notJson.success) return; + const notJsonMessages = notJson.error.issues.map((i) => i.message); + expect(notJsonMessages.some((m) => m !== "Invalid input")).toBe(true); + expect(notJsonMessages.some((m) => m.includes("JSON"))).toBe(true); + + const badEvent = eventBatchSchema.safeParse( + baseBatch({ events: [rawLine.replace(',"seq":0', "")] }), + ); + expect(badEvent.success).toBe(false); + if (badEvent.success) return; + const badEventMessages = badEvent.error.issues.map((i) => i.message); + expect(badEventMessages.some((m) => m !== "Invalid input")).toBe(true); + expect(badEventMessages.some((m) => m.includes("seq"))).toBe(true); + }); + + it("accepts a fully valid batch", () => { + expect(eventBatchSchema.safeParse(baseBatch()).success).toBe(true); + }); + + it("rejects an event line whose seq is negative", () => { + const line = rawLine.replace('"seq":0', '"seq":-1'); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(false); + }); + + it("rejects an event line whose seq is not an integer", () => { + const line = rawLine.replace('"seq":0', '"seq":1.5'); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(false); + }); + + it("rejects a state whose seq is negative", () => { + const state = { ...baseState(), seq: -1 }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects a state whose seq is not an integer", () => { + const state = { ...baseState(), seq: 1.5 }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects an empty identity string on the batch or the state", () => { + expect(eventBatchSchema.safeParse(baseBatch({ runId: "" })).success).toBe(false); + expect(eventBatchSchema.safeParse(baseBatch({ streamId: "" })).success).toBe(false); + expect(eventBatchSchema.safeParse(baseBatch({ graphName: "" })).success).toBe(false); + expect( + eventBatchSchema.safeParse(baseBatch({ state: { ...baseState(), runId: "" } })).success, + ).toBe(false); + expect( + eventBatchSchema.safeParse(baseBatch({ state: { ...baseState(), graphName: "" } })).success, + ).toBe(false); + }); + + it("rejects a node with negative costUsd", () => { + const state = baseState(); + state.nodes = { + n1: { + nodeId: "n1", + status: "succeeded", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: "2026-08-25T00:00:01.000Z", + attempts: 1, + error: null, + costUsd: -0.1, + }, + }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects a node whose attempts is negative or not an integer", () => { + const withAttempts = (attempts: number) => { + const state = baseState(); + state.nodes = { + n1: { + nodeId: "n1", + status: "running", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: null, + attempts, + error: null, + costUsd: 0.1, + }, + }; + return baseBatch({ state }); + }; + expect(eventBatchSchema.safeParse(withAttempts(-1)).success).toBe(false); + expect(eventBatchSchema.safeParse(withAttempts(1.5)).success).toBe(false); + }); + + it("rejects negative or non-integer spent values", () => { + const withSpent = (spent: { usd: number; wallClockSec: number; nodeRuns: number }) => + baseBatch({ state: { ...baseState(), spent } }); + expect(eventBatchSchema.safeParse(withSpent({ usd: -0.1, wallClockSec: 0, nodeRuns: 0 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withSpent({ usd: 0, wallClockSec: -1, nodeRuns: 0 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withSpent({ usd: 0, wallClockSec: 0, nodeRuns: -1 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withSpent({ usd: 0, wallClockSec: 0, nodeRuns: 1.5 })).success).toBe(false); + }); + + it("rejects a budget the engine's own graph parser would reject", () => { + const withBudget = (budget: { maxUsd: number; maxWallClockSec: number; maxNodeRuns: number }) => + baseBatch({ state: { ...baseState(), budget } }); + expect(eventBatchSchema.safeParse(withBudget({ maxUsd: 0, maxWallClockSec: 60, maxNodeRuns: 10 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withBudget({ maxUsd: 1, maxWallClockSec: 0, maxNodeRuns: 10 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withBudget({ maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 0 })).success).toBe(false); + expect(eventBatchSchema.safeParse(withBudget({ maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 1.5 })).success).toBe(false); + }); + + it("rejects a varKeys array with duplicate entries", () => { + const state = { ...baseState(), varKeys: ["a", "b", "a"] }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects a batch whose event seqs are not strictly increasing", () => { + const events = [2, 0, 1].map((s) => rawLine.replace('"seq":0', `"seq":${s}`)); + const result = eventBatchSchema.safeParse(baseBatch({ events })); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "events.1")).toBe(true); + }); + + it("rejects intra-batch duplicate seqs", () => { + const events = [rawLine, rawLine.replace('"seq":0', '"seq":0')]; + const result = eventBatchSchema.safeParse(baseBatch({ events })); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "events.1")).toBe(true); + }); + + it("rejects an event line whose runId differs from the batch runId", () => { + const line = rawLine.replace('"runId":"run-1"', '"runId":"run-other"'); + const result = eventBatchSchema.safeParse(baseBatch({ events: [line] })); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "events.0")).toBe(true); + }); + + it("rejects a completed id that is absent from nodes", () => { + const state = baseState(); + state.nodes = { + n1: { + nodeId: "n1", + status: "succeeded", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: "2026-08-25T00:00:01.000Z", + attempts: 1, + error: null, + costUsd: 0.1, + }, + }; + state.completed = ["n1", "ghost"]; + const result = eventBatchSchema.safeParse(baseBatch({ state })); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.completed")).toBe(true); + }); + + it("still accepts a batch with no events", () => { + expect(eventBatchSchema.safeParse(baseBatch({ events: [] })).success).toBe(true); + }); + + const node = (overrides: Partial = {}): ProjectedNode => ({ + nodeId: "n1", + status: "succeeded", + startedAt: "2026-08-25T00:00:00.000Z", + endedAt: "2026-08-25T00:00:01.000Z", + attempts: 1, + error: null, + costUsd: 0.1, + ...overrides, + }); + + const withNode = (n: ProjectedNode) => { + const state = baseState(); + state.nodes = { [n.nodeId]: n }; + return baseBatch({ state }); + }; + + describe("CLASS 1: timestamps must round-trip through toISOString", () => { + const bad = ["", " ", "banana", "\u0001"]; + + it.each(bad)("rejects ts %j on an event line", (ts) => { + const line = rawLine.replace('"ts":"2026-08-25T00:00:00.000Z"', `"ts":${JSON.stringify(ts)}`); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(false); + }); + + it.each(bad)("rejects state.createdAt %j", (createdAt) => { + const state = { ...baseState(), createdAt } as ProjectedState; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it.each(bad)("rejects state.updatedAt %j", (updatedAt) => { + const state = { ...baseState(), updatedAt } as ProjectedState; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it.each(bad)("rejects node.startedAt %j", (startedAt) => { + const batch = withNode(node({ startedAt })); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + + it.each(bad)("rejects a non-null node.endedAt %j", (endedAt) => { + const batch = withNode(node({ status: "succeeded", endedAt })); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + }); + + describe("CLASS 2: identity strings", () => { + const bad = [" ", "a\nb"]; + + it.each(bad)("rejects eventBatchSchema.runId %j", (runId) => { + expect(eventBatchSchema.safeParse(baseBatch({ runId })).success).toBe(false); + }); + it.each(bad)("rejects eventBatchSchema.streamId %j", (streamId) => { + expect(eventBatchSchema.safeParse(baseBatch({ streamId })).success).toBe(false); + }); + it.each(bad)("rejects eventBatchSchema.graphName %j", (graphName) => { + expect(eventBatchSchema.safeParse(baseBatch({ graphName })).success).toBe(false); + }); + it.each(bad)("rejects state.runId %j", (runId) => { + const state = { ...baseState(), runId } as ProjectedState; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + it.each(bad)("rejects state.graphName %j", (graphName) => { + const state = { ...baseState(), graphName } as ProjectedState; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + it.each(bad)("rejects state.cwd %j", (cwd) => { + const state = { ...baseState(), cwd } as ProjectedState; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + }); + + describe("CLASS 3: node identity", () => { + const bad = ["", "a b", "x".repeat(70), "END"]; + + it.each(bad)("rejects a nodes map key %j", (key) => { + const state = baseState(); + state.nodes = { [key]: node({ nodeId: key }) }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it.each(bad)("rejects a nodeId field %j", (nodeId) => { + expect(eventBatchSchema.safeParse(withNode(node({ nodeId }))).success).toBe(false); + }); + }); + + describe("CLASS 4: status and timestamp coherence", () => { + it("rejects a succeeded node with a null endedAt", () => { + const batch = withNode(node({ status: "succeeded", endedAt: null })); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.nodes.n1.endedAt")).toBe( + true, + ); + }); + + it("rejects a failed node with a null endedAt", () => { + const batch = withNode(node({ status: "failed", endedAt: null, error: "boom" })); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + + it("rejects a skipped node with a null endedAt", () => { + const batch = withNode(node({ status: "skipped", endedAt: null })); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }); + + it("accepts a pending or running node with a null endedAt", () => { + expect(eventBatchSchema.safeParse(withNode(node({ status: "pending", endedAt: null }))).success).toBe(true); + expect(eventBatchSchema.safeParse(withNode(node({ status: "running", endedAt: null }))).success).toBe(true); + }); + + it("rejects an endedAt earlier than its startedAt", () => { + const batch = withNode( + node({ startedAt: "2026-08-25T00:00:02.000Z", endedAt: "2026-08-25T00:00:01.000Z" }), + ); + const result = eventBatchSchema.safeParse(batch); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues.some((i) => i.path.join(".") === "state.nodes.n1.endedAt")).toBe( + true, + ); + }); + + it("accepts endedAt equal to startedAt", () => { + const batch = withNode( + node({ startedAt: "2026-08-25T00:00:00.000Z", endedAt: "2026-08-25T00:00:00.000Z" }), + ); + expect(eventBatchSchema.safeParse(batch).success).toBe(true); + }); + }); + + describe("CLASS 5: attempts", () => { + it("rejects attempts of 0", () => { + expect(eventBatchSchema.safeParse(withNode(node({ attempts: 0 }))).success).toBe(false); + }); + + it("rejects attempts above MAX_ATTEMPTS", () => { + expect(eventBatchSchema.safeParse(withNode(node({ attempts: 1001 }))).success).toBe(false); + }); + }); + + describe("CLASS 6: cardinality", () => { + it("rejects varKeys with more than MAX_VAR_KEYS entries", () => { + const state = { ...baseState(), varKeys: Array.from({ length: 257 }, (_, i) => `k${i}`) }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects nodes with more than MAX_NODES entries", () => { + const state = baseState(); + for (let i = 0; i < 1001; i++) { + const id = `n${i}`; + state.nodes[id] = node({ nodeId: id }); + } + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + + it("rejects completed with more than MAX_NODES entries", () => { + const state = baseState(); + for (let i = 0; i < 1001; i++) { + const id = `n${i}`; + state.nodes[id] = node({ nodeId: id }); + } + state.completed = Array.from({ length: 1001 }, (_, i) => `n${i}`); + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + }); + + describe("CLASS 7: completed duplicates", () => { + it("rejects a completed list with duplicate ids", () => { + const state = baseState(); + state.nodes = { + n1: node(), + n2: node({ nodeId: "n2" }), + }; + state.completed = ["n1", "n1"]; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(false); + }); + }); + + describe("CLASS 8: empty error string", () => { + it("rejects a node error that is an empty string", () => { + expect(eventBatchSchema.safeParse(withNode(node({ status: "failed", error: "" }))).success).toBe(false); + }); + + it("rejects a node error that is whitespace only", () => { + expect(eventBatchSchema.safeParse(withNode(node({ status: "failed", error: " " }))).success).toBe(false); + }); + + it("accepts a null error", () => { + expect(eventBatchSchema.safeParse(withNode(node({ error: null }))).success).toBe(true); + }); + }); + + /** + * A node error is routinely a stack trace or a multi-line stderr dump. Refusing the + * whitespace controls inside one 400s the WHOLE batch, and because the sync cursor only + * advances on a 2xx the same batch is retried forever - that run can never sync again. + * Tab, newline and carriage return must therefore pass; everything the check exists to + * stop must still be stopped. + */ + describe("CLASS 9: control characters in a node error", () => { + const stackTrace = + "Error: boom\n at run (/app/src/core/engine.ts:12:5)\n at main (/app/src/cli.ts:3:1)"; + + it("accepts a node error carrying a multi-line stack trace", () => { + const result = eventBatchSchema.safeParse( + withNode(node({ status: "failed", error: stackTrace })), + ); + expect(result.success).toBe(true); + }); + + it("preserves the newlines in an accepted multi-line error byte-for-byte", () => { + const result = eventBatchSchema.safeParse( + withNode(node({ status: "failed", error: stackTrace })), + ); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.state.nodes["n1"]?.error).toBe(stackTrace); + }); + + it.each([ + ["tab", "command failed:\texit status 1"], + ["carriage return", "downloading...\rdownload failed"], + ["CRLF", "line one\r\nline two"], + ])("accepts a node error containing a %s", (_label, error) => { + expect( + eventBatchSchema.safeParse(withNode(node({ status: "failed", error }))).success, + ).toBe(true); + }); + + it.each([ + ["NUL", "boom\u0000truncated"], + ["ESC / ANSI colour sequence", "\u001b[31mboom\u001b[0m"], + ["ESC / OSC terminal title sequence", "boom\u001b]0;pwned\u0007"], + ["BEL", "boom\u0007"], + ["SOH", "boom\u0001"], + ["vertical tab", "boom\u000bmore"], + ["form feed", "boom\u000cmore"], + ["DEL", "boom\u007f"], + ])("still rejects a node error containing %s", (_label, error) => { + expect( + eventBatchSchema.safeParse(withNode(node({ status: "failed", error }))).success, + ).toBe(false); + }); + + it("still rejects an error made only of newlines, which is empty after trimming", () => { + expect( + eventBatchSchema.safeParse(withNode(node({ status: "failed", error: "\n\n" }))).success, + ).toBe(false); + }); + + /** + * The widening is scoped to `error` alone. Identity strings keep the original refusal - + * a newline or tab in a run id, graph name or cwd is still a delimiter-injection shape + * and must stay a 400. + */ + it.each(["runId", "graphName", "cwd"] as const)( + "still rejects a newline in state.%s", + (field) => { + const state = baseState(); + const batch = baseBatch({ state: { ...state, [field]: `${state[field]}\nx` } }); + expect(eventBatchSchema.safeParse(batch).success).toBe(false); + }, + ); + + it("still rejects a tab in the top-level streamId", () => { + expect(eventBatchSchema.safeParse(baseBatch({ streamId: "s\t1" })).success).toBe(false); + }); + }); + + describe("must remain accepted", () => { + it("accepts an event seq gap, e.g. [0,5]", () => { + const events = [0, 5].map((s) => rawLine.replace('"seq":0', `"seq":${s}`)); + expect(eventBatchSchema.safeParse(baseBatch({ events })).success).toBe(true); + }); + + it("accepts a line with empty data", () => { + const line = rawLine.replace('"data":{"graph":"g"}', '"data":{}'); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(true); + }); + + it("accepts an unknown top-level key and returns it byte-identically", () => { + const line = rawLine.slice(0, -1) + ',"futureKey":{"x":1}}'; + const result = eventBatchSchema.safeParse(baseBatch({ events: [line] })); + expect(result.success).toBe(true); + if (!result.success) return; + expect(result.data.events[0]).toBe(line); + }); + + it("accepts an empty nodes map alongside a run_started line", () => { + const state = { ...baseState(), nodes: {} }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(true); + }); + + it("accepts a seq of Number.MAX_SAFE_INTEGER on a line", () => { + const line = rawLine.replace('"seq":0', `"seq":${Number.MAX_SAFE_INTEGER}`); + expect(eventBatchSchema.safeParse(baseBatch({ events: [line] })).success).toBe(true); + }); + + it("accepts spent greater than budget", () => { + const state = { + ...baseState(), + spent: { usd: 5, wallClockSec: 0, nodeRuns: 20 }, + }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(true); + }); + + it("accepts a budget.maxUsd of 1e-300", () => { + const state = { ...baseState(), budget: { maxUsd: 1e-300, maxWallClockSec: 60, maxNodeRuns: 10 } }; + expect(eventBatchSchema.safeParse(baseBatch({ state })).success).toBe(true); + }); + }); +}); + +describe("MAX_BODY_BYTES", () => { + it("is 5 MiB", () => { + expect(MAX_BODY_BYTES).toBe(5 * 1024 * 1024); + }); +}); + +describe("NO_EVENTS_YET", () => { + it("is -1 and cannot collide with a real zero-based seq", () => { + expect(NO_EVENTS_YET).toBe(-1); + expect(NO_EVENTS_YET).toBeLessThan(0); + }); +}); + +describe("ingest result union", () => { + function ingestTag(result: IngestResult | IngestConflict): number { + if (result.conflict) { + return result.seq; + } + return result.highWaterSeq; + } + + it("routes both arms of the conflict-tagged union to distinguishable values", () => { + const success: IngestResult = { + conflict: false, + highWaterSeq: 7, + accepted: 2, + duplicates: 1, + }; + const conflict: IngestConflict = { + conflict: true, + runId: "run-1", + seq: 3, + }; + expect(ingestTag(success)).toBe(7); + expect(ingestTag(conflict)).toBe(3); + }); +}); + +describe("RunRow status", () => { + function runRowStatus(row: RunRow): number { + switch (row.status) { + case "pending": + return 0; + case "running": + return 1; + case "paused": + return 2; + case "succeeded": + return 3; + case "failed": + return 4; + } + } + + it("switches exhaustively over the status union, like the ingest union test", () => { + const base: RunRow = { + member: "m", + runId: "run-1", + streamId: "s-1", + graphName: "g", + status: "running", + updatedAt: "2026-08-25T00:00:00.000Z", + receivedAt: "2026-08-25T00:00:00.000Z", + }; + expect(runRowStatus({ ...base, status: "pending" })).toBe(0); + expect(runRowStatus(base)).toBe(1); + expect(runRowStatus({ ...base, status: "succeeded" })).toBe(3); + expect(runRowStatus({ ...base, status: "failed" })).toBe(4); + }); +}); diff --git a/src/hub/wire.ts b/src/hub/wire.ts new file mode 100644 index 0000000..b124642 --- /dev/null +++ b/src/hub/wire.ts @@ -0,0 +1,388 @@ +import { z } from "zod"; +import type { Budget, BudgetSpent, NodeStatus, RunStatus } from "../core/types.js"; + +/** One push. `events` are RAW LINES from events.jsonl, never re-serialized objects. */ +export interface EventBatch { + runId: string; + streamId: string; + graphName: string; + state: ProjectedState; + events: string[]; // each element is one verbatim line, ordered by seq +} + +/** A node result with its content removed. No `output` field exists, by construction. */ +export interface ProjectedNode { + nodeId: string; + status: NodeStatus; + startedAt: string; + endedAt: string | null; + attempts: number; + error: string | null; + costUsd: number; +} + +/** + * `RunState` with every content-carrying field structurally absent. Declared HERE, not in + * `src/team/project.ts`, because it is wire vocabulary; commit 1.10 supplies the function + * that produces it. Note there is no `streamId` - `EventBatch` carries that at top level, + * which is also why this type is buildable before commit 1.9 exists. + */ +export interface ProjectedState { + runId: string; + graphName: string; + status: RunStatus; + createdAt: string; + updatedAt: string; + cwd: string; // rewritten by projectState() before it gets here + varKeys: string[]; // KEY NAMES ONLY - there is nowhere to put a value + budget: Budget; + spent: BudgetSpent; + nodes: Record; + completed: string[]; + seq: number; +} + +/** + * The two types below form a discriminated union tagged on `conflict`. `IngestResult` + * carries `conflict: false` explicitly so a caller cannot narrow by accident or forget to + * check. Commit 1.7's handler must switch on `.conflict` to produce a 409; a caller that + * treats the union as a bare success object is a bug. Never widen this back to a union whose + * success arm lacks the tag - forgetting the check must stay a type error. + */ +export interface IngestResult { + conflict: false; + highWaterSeq: number; + accepted: number; + duplicates: number; +} + +export interface IngestConflict { + conflict: true; + runId: string; + seq: number; +} + +export interface FeedItem { + ts: string; + member: string; + kind: FeedKind; + ref: string; +} + +// phases 2-3 use the later members +export type FeedKind = + | "run_started" + | "run_finished" + | "brief_published" + | "inbox_sent" + | "inbox_accepted"; + +export interface RunRow { + member: string; + runId: string; + streamId: string; + graphName: string; + // The same union `ProjectedState.status` and the engine's `RunStatus` use, so a `switch` + // over it stays exhaustive and the hub cannot surface a status the engine never produces. + status: RunStatus; + updatedAt: string; + receivedAt: string; +} + +export const MAX_BODY_BYTES = 5 * 1024 * 1024; + +// Bounds on projected-state sizes. The engine can never exceed these: a graph is +// validated before it runs and a run has one row per node, so each ceiling is well above +// what a real RunState produces while still bounding what a single push can carry. +export const MAX_VAR_KEYS = 256; // at most this many distinct var names in one state +export const MAX_NODES = 1000; // at most this many nodes in one run +export const MAX_ATTEMPTS = 1000; // at most this many attempts of a single node + +/** + * A timestamp the engine itself can write. Every `ts`, `createdAt`, `updatedAt`, + * `startedAt` and `endedAt` the engine produces comes from `new Date().toISOString()`, + * so requiring a value that round-trips through `toISOString()` is the real contract, + * not a tightening. Rejects "", whitespace, "banana" and control characters. + */ +const isoInstant = z + .string() + .refine((s) => { + const d = new Date(s); + return Number.isNaN(d.getTime()) ? false : d.toISOString() === s; + }, { + message: "must be an ISO-8601 instant that round-trips through toISOString()", + }); + +/** + * An identity string the engine produces: non-empty after trimming and free of control + * characters (delimiters never belong in a run id, stream id, graph name or cwd). + */ +const identityString = z + .string() + .min(1) + .refine((s) => s.trim() !== "" && !rejectControl(s), { + message: "must be non-empty after trimming and contain no control characters", + }); + +function rejectControl(s: string): boolean { + return /[\u0000-\u001f\u007f]/.test(s); +} + +/** + * `rejectControl` minus the three whitespace controls that legitimately occur INSIDE error + * text: tab (U+0009), line feed (U+000A) and carriage return (U+000D). + * + * A node error is routinely a stack trace or a multi-line stderr dump. Refusing a newline + * there 400s the WHOLE batch, and the sync cursor only advances on a 2xx + * (`src/team/sync.ts` - `syncRun` returns `{ok:false}` without writing the cursor), so the + * same batch is retried forever and that run can never sync again. Ordinary error output + * must not be a denial-of-sync. + * + * Everything the original check exists to stop is still stopped: NUL, BEL, ESC - so ANSI + * colour and OSC terminal-title sequences cannot ride in on an error string - vertical tab, + * form feed, every other C0 code point, and DEL. Identity strings (run id, stream id, graph + * name, cwd) keep using `rejectControl` unchanged: a tab or newline there is still a + * delimiter-injection shape with no legitimate producer. Do not point identityString at + * this function. + */ +function rejectControlInText(s: string): boolean { + return /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(s); +} + +/** A node id the engine's graph parser (src/core/graph.ts) would accept. */ +const nodeIdString = z + .string() + .refine((id) => id !== "END" && /^[A-Za-z0-9_-]{1,64}$/.test(id), { + message: "must match [A-Za-z0-9_-] and be 1-64 characters, and may not be END", + }); + +/** + * The value of `highWaterSeq` when no event has ever been ingested for a + * (member, streamId, runId). It is -1 rather than 0 or null because `seq` is + * zero-based: `EventLog.append` starts at `read(runId).length`, so seq 0 is a + * real ingested event and cannot double as "nothing yet". `highWater()` returns + * this instead of null so the type is a plain number and no caller can forget a + * null check. + */ +export const NO_EVENTS_YET = -1; + +const projectedNodeSchema = z + .object({ + nodeId: nodeIdString, + status: z.enum(["pending", "running", "succeeded", "failed", "skipped"]), + startedAt: isoInstant, + endedAt: isoInstant.nullable(), + attempts: z.number().int().min(1).max(MAX_ATTEMPTS), + // `rejectControlInText`, not `rejectControl`: see its comment. An error of only + // whitespace still fails, because `trim()` strips the tabs and newlines now allowed. + error: z + .string() + .nullable() + .refine((s) => s === null || (s.trim() !== "" && !rejectControlInText(s)), { + message: + "error must be null or a non-empty string whose only control characters are tab, newline or carriage return", + }), + costUsd: z.number().nonnegative(), + }) + .strict(); + +const projectedStateSchema = z + .object({ + runId: identityString, + graphName: identityString, + status: z.enum(["pending", "running", "paused", "succeeded", "failed"]), + createdAt: isoInstant, + updatedAt: isoInstant, + cwd: identityString, + varKeys: z + .array(z.string()) + .max(MAX_VAR_KEYS) + .refine((keys) => new Set(keys).size === keys.length, { + message: "varKeys must not contain duplicate entries", + }), + budget: z + .object({ + maxUsd: z.number().positive(), + maxWallClockSec: z.number().positive(), + maxNodeRuns: z.number().int().positive(), + }) + .strict(), + spent: z + .object({ + usd: z.number().nonnegative(), + wallClockSec: z.number().nonnegative(), + nodeRuns: z.number().int().nonnegative(), + }) + .strict(), + nodes: z + .record(nodeIdString, projectedNodeSchema) + .refine((n) => Object.keys(n).length <= MAX_NODES, { + message: `nodes must have at most ${MAX_NODES} entries`, + }), + completed: z + .array(z.string()) + .max(MAX_NODES) + .refine((ids) => new Set(ids).size === ids.length, { + message: "completed must not contain duplicate entries", + }), + seq: z.number().int().nonnegative(), + }) + .strict(); + +/** + * Deliberately NOT strict, unlike the projected schemas above. The hub stores each line + * verbatim and projects only `(seq, kind, nodeId)` out of it, so an unknown top-level key is + * inert data - refusing it buys no safety and would break forward compatibility (a member on + * a newer `lg` than the hub could not sync at all, failing as a 400 on a whole batch). + * ProjectedState and ProjectedNode ARE strict because an unknown key there is an + * unclassified field that might carry content. Do not add `.strict()` here. + */ +const eventSchema = z + .object({ + ts: isoInstant, + runId: z.string(), + seq: z.number().int().nonnegative(), + kind: z.enum([ + "run_started", + "node_started", + "node_finished", + "edge_crossed", + "budget_checked", + "budget_exceeded", + "human_requested", + "human_resolved", + "run_finished", + ]), + nodeId: z.string().optional(), + data: z.record(z.string(), z.unknown()), + }); + +/** + * The verbatim-line rule is load-bearing. The server zod-parses each line to validate it + * and to project `(seq, kind, node_id)` into columns, then stores the original string. + * Re-stringifying breaks 6.4's lossless export with no failing test, so the schema accepts + * an array of strings and does not transform them into parsed objects. + */ +export const eventBatchSchema = z + .object({ + runId: identityString, + streamId: identityString, + graphName: identityString, + state: projectedStateSchema, + events: z.array( + // each element must parse to a valid LgEvent but stays a string on the wire + z.string().superRefine((line, ctx) => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "line is not valid JSON", + }); + return; + } + const result = eventSchema.safeParse(parsed); + if (!result.success) { + const detail = result.error.issues + .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("; "); + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `line is valid JSON but not a valid event: ${detail}`, + }); + } + }), + ), + }) + .strict() + .superRefine((batch, ctx) => { + if (batch.state.runId !== batch.runId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "runId"], + message: `state.runId (${batch.state.runId}) does not match top-level runId (${batch.runId})`, + }); + } + if (batch.state.graphName !== batch.graphName) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "graphName"], + message: `state.graphName (${batch.state.graphName}) does not match top-level graphName (${batch.graphName})`, + }); + } + for (const [key, value] of Object.entries(batch.state.nodes)) { + if (key !== value.nodeId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "nodes", key, "nodeId"], + message: `state.nodes.${key}.nodeId (${value.nodeId}) does not match its map key (${key})`, + }); + } + if (value.endedAt === null && value.status !== "pending" && value.status !== "running") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "nodes", key, "endedAt"], + message: `state.nodes.${key} is ${value.status} but has a null endedAt`, + }); + } + if (value.endedAt !== null && value.endedAt < value.startedAt) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "nodes", key, "endedAt"], + message: `state.nodes.${key} endedAt (${value.endedAt}) is earlier than its startedAt (${value.startedAt})`, + }); + } + } + + const parsedSeqs = batch.events.map((line) => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + return null; + } + const seq = (parsed as { seq?: unknown }).seq; + return typeof seq === "number" ? seq : null; + }); + for (let i = 1; i < parsedSeqs.length; i++) { + const prev = parsedSeqs[i - 1]!; + const cur = parsedSeqs[i]!; + if (prev === null || cur === null) continue; + if (cur <= prev) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["events", i], + message: `event line ${i} has seq ${cur}, which is not greater than the previous line's seq ${prev}`, + }); + break; + } + } + + for (let i = 0; i < batch.events.length; i++) { + let parsed: unknown; + try { + parsed = JSON.parse(batch.events[i]!); + } catch { + continue; + } + const runId = (parsed as { runId?: unknown }).runId; + if (typeof runId === "string" && runId !== batch.runId) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["events", i], + message: `event line ${i} carries runId (${runId}), which does not match the top-level runId (${batch.runId})`, + }); + } + } + + const nodeKeys = new Set(Object.keys(batch.state.nodes)); + const unknownCompleted = batch.state.completed.filter((id) => !nodeKeys.has(id)); + if (unknownCompleted.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["state", "completed"], + message: `completed references ids not present in nodes: ${unknownCompleted.join(", ")}`, + }); + } + }); diff --git a/src/team/batch.test.ts b/src/team/batch.test.ts new file mode 100644 index 0000000..e341a06 --- /dev/null +++ b/src/team/batch.test.ts @@ -0,0 +1,562 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execute, newRunState } from "../core/engine.js"; +import { EventLog, type LgEvent } from "../core/events.js"; +import { parseGraph } from "../core/graph.js"; +import { CheckpointStore } from "../core/store.js"; +import type { RunState } from "../core/types.js"; +import type { Adapter, AdapterInput, AdapterOutput } from "../adapters/types.js"; +import type { EventBatch } from "../hub/wire.js"; +import { formatEventLine } from "../commands/context.js"; +import { exitCodeFor } from "../commands/render.js"; +import { makeBatcher, makeRunBatcher, type BatchCtx, type Batcher } from "./batch.js"; +import type { Fetch, HubConfig } from "./transport.js"; + +const CFG: HubConfig = { url: "http://hub.test", token: "lgt_00000000.FAKEfake0000FAKEfake0000" }; + +const SUCCEED_SRC = ` +name: batch-succeed +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + +const FAIL_SRC = ` +name: batch-fail +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "boom" } +edges: + - { from: a, to: END } +`; + +/** The timing keys a run's wall clock varies in; normalizing them is the plan's mechanism. */ +const TIMING_KEYS = new Set(["ts", "createdAt", "updatedAt", "startedAt", "endedAt", "wallClockSec"]); + +function normalize(v: unknown): unknown { + if (Array.isArray(v)) return v.map(normalize); + if (v !== null && typeof v === "object") { + const out: Record = {}; + for (const [k, val] of Object.entries(v as Record)) { + out[k] = TIMING_KEYS.has(k) ? 0 : normalize(val); + } + return out; + } + return v; +} + +function ev(seq: number): LgEvent { + return { + ts: "2026-08-25T00:00:00.000Z", + runId: "fixed-run", + seq, + kind: "node_started", + nodeId: `n${seq}`, + data: { seq }, + }; +} + +function okOutput(text: string): AdapterOutput { + return { ok: true, text, costUsd: 0, raw: null, error: null }; +} +function badOutput(error: string): AdapterOutput { + return { ok: false, text: "", costUsd: 0, raw: null, error }; +} +function stub(name: string, fn: (i: AdapterInput) => AdapterOutput): Adapter { + return { name, run: async (i) => fn(i) }; +} + +const okRegistry: Record = { command: stub("command", (i) => okOutput(i.prompt)) }; +const failRegistry: Record = { command: stub("command", () => badOutput("boom")) }; + +function seedRun(dir: string, runId: string): { store: CheckpointStore; log: EventLog } { + const store = new CheckpointStore(dir); + const log = new EventLog(dir); + const state: RunState = { + runId, + streamId: "11111111-2222-3333-4444-555555555555", + graphName: "g", + status: "running", + createdAt: "2026-08-25T00:00:00.000Z", + updatedAt: "2026-08-25T00:00:01.000Z", + cwd: dir, + vars: {}, + budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 }, + spent: { usd: 0, wallClockSec: 0, nodeRuns: 0 }, + nodes: {}, + completed: [], + seq: 0, + }; + store.save(state); + return { store, log }; +} + +function makeCtx(store: CheckpointStore, runId = "fixed-run"): BatchCtx { + return { + runId, + store, + opts: { + home: "/home/alice", + username: "alice", + repoRoot: "/repo", + hostname: "alice-laptop.local", + }, + }; +} + +function okFetchCalls(): { f: Fetch; calls: { count: number; batches: EventBatch[] } } { + const calls = { count: 0, batches: [] as EventBatch[] }; + const f: Fetch = async (_url, init) => { + calls.count += 1; + const batch = JSON.parse(init.body ?? "{}") as EventBatch; + calls.batches.push(batch); + return { status: 200, json: async () => ({ highWaterSeq: batch.events.length - 1 }) }; + }; + return { f, calls }; +} + +const rejectingFetch: Fetch = async () => { + const err = new Error("connect ECONNREFUSED 127.0.0.1:8080"); + (err as NodeJS.ErrnoException).code = "ECONNREFUSED"; + throw err; +}; + +const neverSettling: Fetch = () => + new Promise<{ status: number; json(): Promise }>(() => {}); + +function captureConsole(): { outs: string[]; errs: string[] } { + const outs: string[] = []; + const errs: string[] = []; + vi.spyOn(console, "log").mockImplementation((...a: unknown[]) => { + outs.push(a.join(" ")); + }); + vi.spyOn(console, "error").mockImplementation((...a: unknown[]) => { + errs.push(a.join(" ")); + }); + return { outs, errs }; +} + +const FIXED_STREAM_ID = "11111111-2222-3333-4444-555555555555"; + +interface RunRequest { + dir: string; + src: string; + registry: Record; + makeBatcherFor: ((store: CheckpointStore) => Batcher) | null; + printed: string[]; + events: LgEvent[]; + /** State.cwd; defaults to `dir`. Fixed across a compared pair so the runs' state is comparable. */ + cwd?: string; +} + +async function runOnce(request: RunRequest): Promise<{ final: RunState; code: number }> { + const store = new CheckpointStore(request.dir); + const log = new EventLog(request.dir); + const runId = "fixed-run"; + const graph = parseGraph(request.src); + const state = newRunState(graph, { runId, cwd: request.cwd ?? request.dir, vars: {} }); + state.streamId = FIXED_STREAM_ID; + const batcher = request.makeBatcherFor === null ? null : request.makeBatcherFor(store); + const final = await execute(graph, state, { + store, + log, + registry: request.registry, + sleep: async () => {}, + onEvent: (event) => { + const line = formatEventLine(event); + if (line) { + request.printed.push(line); + console.log(line); + } + request.events.push(event); + batcher?.onEvent(event); + }, + }); + await batcher?.flush(); + batcher?.stop(); + return { final, code: exitCodeFor(final, log.read(runId)) }; +} + +function expectIdentical( + a: { final: RunState; code: number }, + evA: LgEvent[], + b: { final: RunState; code: number }, + evB: LgEvent[], +): void { + const seqA = evA.map((e) => `${e.kind}:${e.seq}`); + const seqB = evB.map((e) => `${e.kind}:${e.seq}`); + expect(seqA).toEqual(seqB); + expect(normalize(evA)).toEqual(normalize(evB)); + expect(normalize(a.final)).toEqual(normalize(b.final)); + expect(a.code).toBe(b.code); +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("a hub outage cannot change a run", () => { + it("succeeding run: outcome identical with a rejecting hub, one stderr line, no pending timer", async () => { + vi.useFakeTimers(); + try { + const { errs } = captureConsole(); + const dirA = mkdtempSync(join(tmpdir(), "lg-batch-ok-a-")); + const dirB = mkdtempSync(join(tmpdir(), "lg-batch-ok-b-")); + try { + const evA: LgEvent[] = []; + const evB: LgEvent[] = []; + const pA: string[] = []; + const pB: string[] = []; + const withB = await runOnce({ + dir: dirA, + src: SUCCEED_SRC, + registry: okRegistry, + makeBatcherFor: (store) => makeBatcher(CFG, rejectingFetch, makeCtx(store)), + printed: pA, + events: evA, + cwd: "/fixed-cwd", + }); + expect(errs.filter((l) => l.includes("hub sync"))).toHaveLength(1); + + errs.length = 0; + const withoutB = await runOnce({ + dir: dirB, + src: SUCCEED_SRC, + registry: okRegistry, + makeBatcherFor: null, + printed: pB, + events: evB, + cwd: "/fixed-cwd", + }); + expect(errs).toHaveLength(0); + + expectIdentical(withB, evA, withoutB, evB); + expect(pA).toEqual(pB); + expect(vi.getTimerCount()).toBe(0); + } finally { + rmSync(dirA, { recursive: true, force: true }); + rmSync(dirB, { recursive: true, force: true }); + } + } finally { + vi.useRealTimers(); + } + }); + + it("failing run: outcome identical with a rejecting hub, one stderr line, no pending timer", async () => { + vi.useFakeTimers(); + try { + const { errs } = captureConsole(); + const dirA = mkdtempSync(join(tmpdir(), "lg-batch-fail-a-")); + const dirB = mkdtempSync(join(tmpdir(), "lg-batch-fail-b-")); + try { + const evA: LgEvent[] = []; + const evB: LgEvent[] = []; + const pA: string[] = []; + const pB: string[] = []; + const withB = await runOnce({ + dir: dirA, + src: FAIL_SRC, + registry: failRegistry, + makeBatcherFor: (store) => makeBatcher(CFG, rejectingFetch, makeCtx(store)), + printed: pA, + events: evA, + cwd: "/fixed-cwd", + }); + expect(errs.filter((l) => l.includes("hub sync"))).toHaveLength(1); + + errs.length = 0; + const withoutB = await runOnce({ + dir: dirB, + src: FAIL_SRC, + registry: failRegistry, + makeBatcherFor: null, + printed: pB, + events: evB, + cwd: "/fixed-cwd", + }); + expect(errs).toHaveLength(0); + + expectIdentical(withB, evA, withoutB, evB); + expect(pA).toEqual(pB); + expect(vi.getTimerCount()).toBe(0); + } finally { + rmSync(dirA, { recursive: true, force: true }); + rmSync(dirB, { recursive: true, force: true }); + } + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("batcher buffering and timing", () => { + it("1. nine events dispatch nothing; the tenth triggers one flush carrying all ten", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-buffer-")); + try { + const { store } = seedRun(dir, "fixed-run"); + const { f, calls } = okFetchCalls(); + const batcher = makeBatcher(CFG, f, makeCtx(store)); + for (let i = 0; i < 9; i++) batcher.onEvent(ev(i)); + expect(calls.count).toBe(0); + + batcher.onEvent(ev(9)); + await batcher.flush(); + + expect(calls.count).toBe(1); + expect(calls.batches[0]!.events).toHaveLength(10); + batcher.stop(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("2. the 5-second periodic timer flushes a partial buffer", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-periodic-")); + try { + const { store } = seedRun(dir, "fixed-run"); + const { f, calls } = okFetchCalls(); + vi.useFakeTimers(); + try { + const batcher = makeBatcher(CFG, f, makeCtx(store)); + for (let i = 0; i < 3; i++) batcher.onEvent(ev(i)); + expect(calls.count).toBe(0); + + await vi.advanceTimersByTimeAsync(5000); + + expect(calls.count).toBe(1); + expect(calls.batches[0]!.events).toHaveLength(3); + batcher.stop(); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("3. the periodic timer is unref'd and stop() clears it", () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-unref-")); + try { + const { store } = seedRun(dir, "fixed-run"); + const { f } = okFetchCalls(); + + // Real timers: the interval makeBatcher created carries no ref back. + let captured: ReturnType | undefined; + const originalSetInterval = globalThis.setInterval.bind(globalThis); + const spy = vi + .spyOn(globalThis, "setInterval") + .mockImplementation( + ((handler: (...args: unknown[]) => void, timeout?: number, ...args: unknown[]) => { + const t = originalSetInterval(handler, timeout, ...args); + captured = t; + return t; + }) as typeof globalThis.setInterval, + ); + const batcher = makeBatcher(CFG, f, makeCtx(store)); + expect(captured).toBeDefined(); + expect(captured!.hasRef()).toBe(false); + spy.mockRestore(); + batcher.stop(); + + // Fake timers: stop() removes the only pending periodic timer. + vi.useFakeTimers(); + try { + const b2 = makeBatcher(CFG, f, makeCtx(store)); + expect(vi.getTimerCount()).toBe(1); + b2.stop(); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("4. flush() resolves inside its ceiling when the transport never settles and ignores the abort signal", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-ceiling-")); + try { + const { store } = seedRun(dir, "fixed-run"); + const batcher = makeBatcher(CFG, neverSettling, { + ...makeCtx(store), + flushCeilingMs: 30, + timeoutMs: 50, + }); + for (let i = 0; i < 5; i++) batcher.onEvent(ev(i)); + + const started = Date.now(); + await batcher.flush(); + const elapsed = Date.now() - started; + + expect(elapsed).toBeLessThan(1000); + batcher.stop(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("5. flush() never rejects: a rejecting transport, a non-2xx response, and a never-settling transport", async () => { + const non2xx: Fetch = async () => ({ status: 500, json: async () => ({ error: "boom" }) }); + const cases: Array<{ f: Fetch; ceilingMs: number }> = [ + { f: rejectingFetch, ceilingMs: 200 }, + { f: non2xx, ceilingMs: 200 }, + { f: neverSettling, ceilingMs: 30 }, + ]; + for (const [i, c] of cases.entries()) { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-neverreject-")); + try { + const { store } = seedRun(dir, `run-${i}`); + const batcher = makeBatcher(CFG, c.f, { + ...makeCtx(store, `run-${i}`), + flushCeilingMs: c.ceilingMs, + timeoutMs: 50, + }); + batcher.onEvent(ev(0)); + await expect(batcher.flush()).resolves.toBeUndefined(); + batcher.stop(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + } + }); +}); + +describe("no-op gate and failure accounting", () => { + it("6. no-op batcher when loadHubConfig is null or repoSyncEnabled is false; the Fetch is never called", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-gates-")); + try { + // repo opted in, but no hub config + const optIn = join(dir, "optin"); + mkdirSync(join(optIn, ".loomgraph"), { recursive: true }); + writeFileSync(join(optIn, ".loomgraph", "hub.json"), '{"sync":true}\n', "utf8"); + const { store: storeA } = seedRun(join(dir, "store-a"), "fixed-run"); + const callsA = { count: 0 }; + const fA: Fetch = async () => { + callsA.count += 1; + return { status: 200, json: async () => ({ highWaterSeq: 0 }) }; + }; + const b1 = makeRunBatcher({ cfg: null, cwd: optIn, f: fA, ctx: makeCtx(storeA) }); + for (let i = 0; i < 25; i++) b1.onEvent(ev(i)); + await b1.flush(); + b1.stop(); + expect(callsA.count).toBe(0); + + // hub configured, but the repo has not opted in + const noOptIn = join(dir, "nooptin"); + mkdirSync(noOptIn, { recursive: true }); + const { store: storeB } = seedRun(join(dir, "store-b"), "fixed-run"); + const callsB = { count: 0 }; + const fB: Fetch = async () => { + callsB.count += 1; + return { status: 200, json: async () => ({ highWaterSeq: 0 }) }; + }; + const b2 = makeRunBatcher({ cfg: CFG, cwd: noOptIn, f: fB, ctx: makeCtx(storeB) }); + for (let i = 0; i < 25; i++) b2.onEvent(ev(i)); + await b2.flush(); + b2.stop(); + expect(callsB.count).toBe(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("7. every dispatched promise is caught: a process-level unhandledRejection listener never fires", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-rej-")); + try { + const rejections: unknown[] = []; + const onUnhandled = (reason: unknown): void => { + rejections.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + try { + const { store } = seedRun(dir, "fixed-run"); + const b1 = makeBatcher(CFG, rejectingFetch, { + ...makeCtx(store), + flushCeilingMs: 25, + timeoutMs: 25, + }); + for (let i = 0; i < 28; i++) b1.onEvent(ev(i)); + await b1.flush(); + b1.stop(); + + const b2 = makeBatcher(CFG, neverSettling, { + ...makeCtx(store), + flushCeilingMs: 25, + timeoutMs: 25, + }); + b2.onEvent(ev(0)); + await b2.flush(); + b2.stop(); + + // Give any dangling postEvents chain time to settle and surface a rejection. + await new Promise((r) => setTimeout(r, 60)); + expect(rejections).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandled); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("8. five separate failing flushes print exactly one stderr line", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-oneline-")); + try { + const { errs } = captureConsole(); + const { store } = seedRun(dir, "fixed-run"); + const batcher = makeBatcher(CFG, rejectingFetch, makeCtx(store)); + for (let round = 0; round < 5; round++) { + batcher.onEvent(ev(round)); + await batcher.flush(); + } + batcher.stop(); + expect(errs).toHaveLength(1); + expect(errs[0]).toContain("hub sync unavailable"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("9. live console progress output is unchanged when the batcher is active", async () => { + const dir = mkdtempSync(join(tmpdir(), "lg-batch-progress-")); + try { + const { outs } = captureConsole(); + const printedWith: string[] = []; + const printedWithout: string[] = []; + await runOnce({ + dir: join(dir, "a"), + src: SUCCEED_SRC, + registry: okRegistry, + makeBatcherFor: (store) => makeBatcher(CFG, rejectingFetch, makeCtx(store)), + printed: printedWith, + events: [], + cwd: "/fixed-cwd", + }); + const withLines = outs.slice(); + outs.length = 0; + + await runOnce({ + dir: join(dir, "b"), + src: SUCCEED_SRC, + registry: okRegistry, + makeBatcherFor: null, + printed: printedWithout, + events: [], + cwd: "/fixed-cwd", + }); + + expect(printedWith).toEqual(printedWithout); + expect(outs).toEqual(withLines); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); \ No newline at end of file diff --git a/src/team/batch.ts b/src/team/batch.ts new file mode 100644 index 0000000..ad51559 --- /dev/null +++ b/src/team/batch.ts @@ -0,0 +1,189 @@ +import type { LgEvent } from "../core/events.js"; +import type { CheckpointStore } from "../core/store.js"; +import { buildBatch, type ProjectionOpts } from "./sync.js"; +import { postEvents, repoSyncEnabled, type Fetch, type HubConfig } from "./transport.js"; + +/** The live-push seam a command drives alongside its console progress output. */ +export interface Batcher { + /** Synchronous and fire-and-forget; called from inside the engine's commit(). */ + onEvent(event: LgEvent): void; + /** Push whatever is buffered, bounded by a ceiling, resolving within it. */ + flush(): Promise; + /** Clear the periodic timer; the last chance to emit the single failure line. */ + stop(): void; +} + +/** + * Everything the batcher needs to assemble and send a push. `store` is loaded + * per batch (never cached) so the projected state tracks whatever checkpoint + * the engine most recently saved, exactly as `lg sync` recomputes it per batch. + */ +export interface BatchCtx { + runId: string; + store: CheckpointStore; + opts: ProjectionOpts; + /** Per-request bound handed to postEvents (default 10s). */ + timeoutMs?: number; + /** flush()'s own ceiling (default 10s); kept small in tests. */ + flushCeilingMs?: number; +} + +/** Dispatch once a batch reaches this many buffered events. */ +const FLUSH_EVENT_COUNT = 10; +/** ...or on this cadence, whichever comes first. */ +const PERIOD_MS = 5_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 10_000; +const DEFAULT_FLUSH_CEILING_MS = 10_000; + +function noopBatcher(): Batcher { + return { + onEvent: () => {}, + flush: async () => {}, + stop: () => {}, + }; +} + +/** + * Assumption A1 is enforced HERE or it is not enforced anywhere: a live batcher + * exists only when the hub is configured (loadHubConfig is non-null) AND the + * repo has opted in (repoSyncEnabled is true). Either gate failing yields a + * no-op batcher that never touches the Fetch, so a run on an unenrolled machine + * cannot hang on a hub address that does not exist. + */ +export function makeRunBatcher(opts: { + cfg: HubConfig | null; + cwd: string; + f: Fetch; + ctx: BatchCtx; +}): Batcher { + if (opts.cfg === null || !repoSyncEnabled(opts.cwd)) return noopBatcher(); + return makeBatcher(opts.cfg, opts.f, opts.ctx); +} + +export function makeBatcher(cfg: HubConfig, f: Fetch, ctx: BatchCtx): Batcher { + return new LiveBatcher(cfg, f, ctx); +} + +class LiveBatcher implements Batcher { + private buffer: string[] = []; + private flushChain: Promise = Promise.resolve(); + private failures = 0; + private failureReported = false; + private periodicTimer: ReturnType | undefined; + + constructor( + private readonly cfg: HubConfig, + private readonly f: Fetch, + private readonly ctx: BatchCtx, + ) { + // THE PERIODIC TIMER IS unref()d AND CLEARED IN stop(). Without the unref, + // `lg run` would appear to hang for up to 5 seconds after every run: the + // run's real work ends, nothing else keeps the loop alive, but the interval + // would. stop() also must clear it so the timer never fires after its run + // is over. + this.periodicTimer = setInterval(() => this.scheduleDispatch(), PERIOD_MS); + this.periodicTimer.unref(); + } + + onEvent(event: LgEvent): void { + // SYNCHRONOUS and fire-and-forget. This runs inside the engine's commit(), + // on the checkpoint path, so no `await` may precede the dispatch - putting + // network latency here would put it on every edge crossing. + this.buffer.push(JSON.stringify(event)); + if (this.buffer.length >= FLUSH_EVENT_COUNT) this.scheduleDispatch(); + } + + private scheduleDispatch(): void { + // Every dispatched promise carries its own .catch: an unhandled rejection + // changes the process exit path and vitest results. pushBuffered never + // rejects, so the catch is belt-and-braces that must stay. + this.flushChain = this.flushChain.then(() => this.pushBuffered()).catch(() => {}); + } + + flush(): Promise { + // flush()'s OWN CEILING TIMER IS DELIBERATELY NOT unref()d - unlike the + // periodic timer above and unlike postEvents' internal request timeout. + // postEvents bounds a hung transport by racing it against an unref()d + // timer, which means when this flush is the only pending work in the + // process (exactly the state at end of run) Node is free to exit before + // that timer fires and the transport promise never settles. So flush() + // imposes its own ceiling with a timer that DOES keep the loop alive: it + // always fires, flush() always resolves, never rejects and never hangs. + // Hitting the ceiling is a failure, like any other failed push. + return new Promise((resolve) => { + const ceilingMs = this.ctx.flushCeilingMs ?? DEFAULT_FLUSH_CEILING_MS; + const ceiling = setTimeout(() => { + this.failures += 1; + this.emitFailureOnce(); + resolve(); + }, ceilingMs); + this.flushChain + .then(() => this.pushBuffered()) + .then( + () => { + clearTimeout(ceiling); + this.emitFailureOnce(); + resolve(); + }, + () => { + this.failures += 1; + clearTimeout(ceiling); + this.emitFailureOnce(); + resolve(); + }, + ); + }); + } + + stop(): void { + if (this.periodicTimer !== undefined) { + clearInterval(this.periodicTimer); + this.periodicTimer = undefined; + } + this.emitFailureOnce(); + } + + private async pushBuffered(): Promise { + const lines = this.buffer; + this.buffer = []; + if (lines.length === 0) return; + + const state = this.ctx.store.load(this.ctx.runId); + // The engine checkpoints before the first event, so a missing state here + // means the run never really started - count it as an unpushable batch. + if (state === null) { + this.failures += 1; + return; + } + + // The try wraps ONLY the push pipeline - the batch build and the fetch + // promise chain. The engine, the checkpoint and the console printing are + // all untouched, so a hub failure can surface here and nowhere else. + try { + const batch = buildBatch(state, this.ctx.opts, lines); + const result = await postEvents( + this.f, + this.cfg, + batch, + this.ctx.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS, + ); + if (!result.ok) this.failures += 1; + } catch { + this.failures += 1; + } + // A failed batch is dropped, not re-buffered: the events are already + // durable in events.jsonl, so the live stream losing them costs nothing + // and `lg sync` re-pushes anything it missed once the hub is back. + } + + /** At most one stderr line for the whole run, however many batches fail. */ + private emitFailureOnce(): void { + if (this.failures > 0 && !this.failureReported) { + this.failureReported = true; + const unit = this.failures === 1 ? "batch" : "batches"; + console.error( + `hub sync unavailable: ${this.failures} ${unit} not pushed (run ${this.ctx.runId})`, + ); + } + } +} \ No newline at end of file diff --git a/src/team/project.test.ts b/src/team/project.test.ts new file mode 100644 index 0000000..bd0eea1 --- /dev/null +++ b/src/team/project.test.ts @@ -0,0 +1,530 @@ +// Every credential in this file is fabricated: fixed "FAKE"/"fake" filler in +// the shape of the real thing. Nothing here is or ever was a live secret. +// +// Secret-shaped fixtures are assembled from split parts at runtime. A provider's +// secret scanner matches on shape, not validity, so a complete literal in the +// source trips GitHub secret scanning and files an alert against this repo. +// Splitting the prefix from the body keeps the assertion honest without handing +// a detector anything to match. +const shaped = (prefix: string, body: string): string => prefix + body; + +import { describe, expect, it } from "vitest"; +import type { NodeResult, RunState } from "../core/types.js"; +import { eventBatchSchema, type EventBatch } from "../hub/wire.js"; +import { projectState, safeText } from "./project.js"; + +const OPTS = { + home: "/home/alice", + username: "alice", + repoRoot: "/home/alice/work/repo", + hostname: "alice-laptop.local", +}; + +function baseState(): RunState { + return { + runId: "run-1", + streamId: "11111111-2222-3333-4444-555555555555", + graphName: "g", + status: "running", + createdAt: "2026-08-25T00:00:00.000Z", + updatedAt: "2026-08-25T00:00:01.000Z", + cwd: "/home/alice/work/repo", + vars: {}, + budget: { maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 10 }, + spent: { usd: 0, wallClockSec: 0, nodeRuns: 0 }, + nodes: {}, + completed: [], + seq: 1, + }; +} + +function node( + nodeId: string, + status: NodeResult["status"], + overrides: Partial = {}, +): NodeResult { + const terminal = status === "succeeded" || status === "failed" || status === "skipped"; + return { + nodeId, + status, + startedAt: "2026-08-25T00:00:02.000Z", + endedAt: terminal ? "2026-08-25T00:00:03.000Z" : null, + attempts: 1, + output: null, + error: null, + costUsd: 0, + ...overrides, + }; +} + +describe("projectState", () => { + it("the whole serialized projection contains neither a vars value nor a node output", () => { + const varsSecret = shaped("sk-ant-", "api03-VARSSECRET000000FAKEfake0000"); + const outputSecret = shaped("ghp", "_OUTPUTSECRET0000FAKEfake0000"); + const state = baseState(); + state.vars = { ticketId: "PLAT-4711", apiToken: varsSecret }; + state.nodes.a = node("a", "succeeded", { output: `pushed ${outputSecret}` }); + + const projected = projectState(state, OPTS); + const serialized = JSON.stringify(projected); + + expect(serialized).not.toContain(varsSecret); + expect(serialized).not.toContain(outputSecret); + expect(serialized).toContain("apiToken"); + expect(serialized).toContain("ticketId"); + }); + + it("a vars value containing a shaped secret is absent while its key survives", () => { + const secret = shaped("AIza", "FAKESECRET0000FAKEfake0000FAKEfake0000"); + const state = baseState(); + state.vars = { deployToken: secret }; + + const projected = projectState(state, OPTS); + const serialized = JSON.stringify(projected); + + expect(serialized).not.toContain(secret); + expect(projected.varKeys).toContain("deployToken"); + }); + + it("a node output containing a shaped secret is absent", () => { + const secret = shaped("github", "_pat_11FAKEfake0000FAKEfake0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { output: `stdout:\n${secret}` }); + + const projected = projectState(state, OPTS); + const serialized = JSON.stringify(projected); + + expect(serialized).not.toContain(secret); + }); + + it("costUsd, status and attempts survive on each projected node", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { attempts: 3, costUsd: 1.25, error: "boom" }); + state.nodes.b = node("b", "running", { attempts: 2, costUsd: 0.5 }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a).toEqual({ + nodeId: "a", + status: "failed", + startedAt: "2026-08-25T00:00:02.000Z", + endedAt: "2026-08-25T00:00:03.000Z", + attempts: 3, + error: "boom", + costUsd: 1.25, + }); + expect(projected.nodes.b!.status).toBe("running"); + expect(projected.nodes.b!.attempts).toBe(2); + expect(projected.nodes.b!.costUsd).toBe(0.5); + }); + + it("startedAt, endedAt and error survive the node projection", () => { + const state = baseState(); + state.nodes.a = node("a", "succeeded", { + startedAt: "2026-08-25T01:00:00.000Z", + endedAt: "2026-08-25T01:02:00.000Z", + error: "retried once, then passed", + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.startedAt).toBe("2026-08-25T01:00:00.000Z"); + expect(projected.nodes.a!.endedAt).toBe("2026-08-25T01:02:00.000Z"); + expect(projected.nodes.a!.error).toBe("retried once, then passed"); + }); + + it("rewrites a cwd at the repo root to the repoRoot placeholder", () => { + const state = baseState(); + state.cwd = "/home/alice/work/repo"; + + const projected = projectState(state, OPTS); + + expect(projected.cwd).toBe("${REPO_ROOT}"); + }); + + it("rewrites a cwd under home but outside the repo to the home placeholder", () => { + const state = baseState(); + state.cwd = "/home/alice/downloads"; + + const projected = projectState(state, OPTS); + + expect(projected.cwd).toBe("${HOME}/downloads"); + }); + + it("rewrites a username-shaped path, proving username is threaded through", () => { + const state = baseState(); + state.cwd = "/Users/alice/notes.md"; + + const projected = projectState(state, OPTS); + + expect(projected.cwd).toBe("${HOME}/notes.md"); + }); + + it("rewrites a standalone username token only at path boundaries", () => { + const state = baseState(); + state.cwd = "alice/config/main.yaml"; + + const projected = projectState(state, OPTS); + + expect(projected.cwd).toBe("user/config/main.yaml"); + }); + + it("varKeys equals the vars keys exactly, including order", () => { + const state = baseState(); + state.vars = { + zeta: 1, + alpha: "x", + token: shaped("sk-", "Fakevalue0000"), + }; + + const projected = projectState(state, OPTS); + + expect(projected.varKeys).toEqual(["zeta", "alpha", "token"]); + expect("vars" in projected).toBe(false); + }); + + it("empty vars gives an empty varKeys", () => { + const projected = projectState(baseState(), OPTS); + expect(projected.varKeys).toEqual([]); + }); + + it("the projection carries no streamId - EventBatch owns it at top level", () => { + const projected = projectState(baseState(), OPTS); + expect("streamId" in projected).toBe(false); + }); + + it("no projected node has an output property", () => { + const state = baseState(); + state.nodes.a = node("a", "succeeded", { output: "irrelevant" }); + state.nodes.b = node("b", "running", { output: shaped("glpat", "Z-0000-not-real") }); + + const projected = projectState(state, OPTS); + + for (const n of Object.values(projected.nodes)) { + expect("output" in n).toBe(false); + } + }); + + it("classifies every RunState key before anything leaves the machine", () => { + // Adding a field to `RunState` fails typecheck here until someone classifies it: + // `Record` is exhaustive by construction. + // A classified "dropped" field appearing on the projection fails at runtime, and a + // "projected" field going missing fails the key-set equality below. This is plan + // §1.10's "a future field must be explicitly classified before it can be pushed". + const classification: Record = { + runId: "projected", + streamId: "dropped", // carried on EventBatch, never on ProjectedState + graphName: "projected", + status: "projected", + createdAt: "projected", + updatedAt: "projected", + cwd: "projected", + vars: "dropped", // the values are secrets; only key names survive as varKeys + budget: "projected", + spent: "projected", + nodes: "projected", // per-node output is dropped inside the projection + completed: "projected", + seq: "projected", + }; + + const state = baseState(); + state.vars = { a: 1, b: 2 }; + state.nodes.c = node("c", "succeeded"); + const projection = projectState(state, OPTS); + + const expected = new Set([ + ...Object.entries(classification) + .filter(([, kind]) => kind === "projected") + .map(([key]) => key), + // derived from the dropped `vars` field - the only derived key there is + "varKeys", + ]); + expect(new Set(Object.keys(projection))).toEqual(expected); + }); + + it("round-trips through eventBatchSchema inside a valid EventBatch", () => { + const state = baseState(); + state.nodes.a = node("a", "succeeded", { costUsd: 0.5 }); + state.completed = ["a"]; + state.status = "succeeded"; + + const projected = projectState(state, OPTS); + const batch: EventBatch = { + runId: projected.runId, + streamId: state.streamId, + graphName: projected.graphName, + state: projected, + events: [], + }; + + const parsed = eventBatchSchema.safeParse(batch); + expect(parsed.success).toBe(true); + }); + + it("a shaped secret planted in a node error is absent while the message survives", () => { + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { error: `could not reach the agent: ${secret}` }); + + const projected = projectState(state, OPTS); + const serialized = JSON.stringify(projected); + + expect(serialized).not.toContain(secret); + expect(serialized).toContain("could not reach the agent"); + }); + + it("a masked secret appears as the first 4 characters of the match followed by ...", () => { + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { error: `could not reach the agent: ${secret}` }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toContain("sk-a..."); + expect(projected.nodes.a!.error).not.toContain("api03"); + }); + + it("an absolute home path in a node error is rewritten to the HOME placeholder", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: "missing /home/alice/.config/loomgraph/hub.json", + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("missing ${HOME}/.config/loomgraph/hub.json"); + }); + + it("an ANSI-coloured error is stripped clean and passes the hub's wire schema", () => { + // Many CLIs colour stderr by default. The hub still rejects ESC (U+001B) in + // a node error - deliberately, so a colour code or an OSC terminal-title + // sequence cannot ride in - so an uncleaned error would 400 the whole batch + // and wedge that run's sync forever, exactly as a newline used to. + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: "\u001b[31mbuild failed\u001b[0m in \u001b]0;title\u0007module", + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("build failed in module"); + + const batch: EventBatch = { + runId: projected.runId, + streamId: state.streamId, + graphName: projected.graphName, + state: projected, + events: [], + }; + expect(eventBatchSchema.safeParse(batch).success).toBe(true); + }); + + it("a multi-line stack trace survives intact - tab, newline and carriage return are kept", () => { + const trace = "Error: boom\n\tat run (/home/alice/work/repo/src/a.ts:1:1)\r\n\tat main"; + const state = baseState(); + state.nodes.a = node("a", "failed", { error: trace }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe( + "Error: boom\n\tat run (${REPO_ROOT}/src/a.ts:1:1)\r\n\tat main", + ); + + const batch: EventBatch = { + runId: projected.runId, + streamId: state.streamId, + graphName: projected.graphName, + state: projected, + events: [], + }; + expect(eventBatchSchema.safeParse(batch).success).toBe(true); + }); + + it("a NUL, a bell, a DEL and a vertical tab are removed while the text around them survives", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { error: "a\u0000b\u0007c\u007fd\u000b" }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("abcd"); + }); + + it("control characters are stripped BEFORE masking, so an escape spliced into a secret cannot evade the masker", () => { + // A colouriser can emit an escape in the middle of a token. Stripping after + // masking would hand the wire a reassembled, UNMASKED secret; stripping + // first means the masker sees the contiguous token it has a rule for. + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const spliced = `${secret.slice(0, 12)}\u001b[0m${secret.slice(12)}`; + const state = baseState(); + state.nodes.a = node("a", "failed", { error: `key ${spliced}` }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("key sk-a..."); + expect(projected.nodes.a!.error).not.toContain("api03"); + }); + + it("the full chain runs in order: rewrite, then mask, then cap", () => { + // ORDER REGRESSION, pinned as one assertion because control stripping now + // sits in this chain and makes it easy to disturb. Capping before masking + // would truncate the key below its rule's `{16,}` tail and publish real + // characters; masking before rewriting would leave the home path intact. + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const filler = "y".repeat(190); + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: `\u001b[31m/home/alice/work/repo/a.ts ${secret} ${filler}\u001b[0m`, + }); + + const projected = projectState(state, OPTS); + const error = projected.nodes.a!.error!; + + expect(error).not.toContain("\u001b"); + // rewrite ran: the repo root became a placeholder + expect(error).toContain("${REPO_ROOT}/a.ts"); + // mask ran, and ran before the cap: the key is 7 characters, not a fragment + expect(error).toContain("sk-a..."); + expect(error).not.toContain("api03"); + // cap ran last, on the already-rewritten, already-masked, already-stripped text + expect(error).toHaveLength(201); + expect(error.endsWith("…")).toBe(true); + }); + + it("masking runs BEFORE capping, so a secret straddling the 200-char cap cannot be published as a fragment", () => { + // ORDER REGRESSION. Capping first would truncate this key below its rule's + // `{16,}` tail, the mask would then fail to match, and the surviving + // prefix would publish real key material. Do not reorder rewrite/mask/cap. + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { error: `${"x".repeat(190)} ${secret}` }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toContain("sk-a..."); + expect(projected.nodes.a!.error).not.toContain("api03"); + expect(projected.nodes.a!.error).not.toContain(secret.slice(0, 20)); + }); + + it("d. the machine hostname is rewritten out of a projected node error", () => { + // BUG 1: `rewritePaths` has always had a hostname rule, but `ProjectionOpts` + // had no `hostname` field, so no caller could supply one and the rule never + // fired - a leak in the one channel that IS an allowlist. + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: "ssh alice-laptop.local: connection refused (short form alice-laptop too)", + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).not.toContain("alice-laptop"); + expect(projected.nodes.a!.error).toContain("${HOSTNAME}"); + }); + + it("a repo-root path in a node error is rewritten to the REPO_ROOT placeholder", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: "script at /home/alice/work/repo/scripts/deploy.sh blew up", + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("script at ${REPO_ROOT}/scripts/deploy.sh blew up"); + }); + + it("a node error longer than 200 characters is truncated to 200 plus a single ellipsis", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: `node failed after exhaustive retries: ${"retry-".repeat(50)}`, + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error!.length).toBeLessThanOrEqual(201); + expect(projected.nodes.a!.error!.endsWith("…")).toBe(true); + expect(projected.nodes.a!.error!.startsWith("node failed after exhaustive retries")).toBe(true); + }); + + it("a null node error stays exactly null - never an empty string", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { error: null }); + + const projected = projectState(state, OPTS); + const serialized = JSON.stringify(projected); + + expect(projected.nodes.a!.error).toBeNull(); + expect(serialized).toContain('"error":null'); + expect(serialized).not.toContain('"error":""'); + }); + + it("a command-shaped error carrying a path and a secret comes out with both sanitised", () => { + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { + error: `command exited with code 1: /home/alice/work/repo/run.sh: error: token ${secret} rejected`, + }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toContain("${REPO_ROOT}/run.sh"); + expect(projected.nodes.a!.error).toContain("token sk-a... rejected"); + expect(projected.nodes.a!.error).not.toContain(secret); + expect(projected.nodes.a!.error).not.toContain("/home/alice"); + }); + + it("a short node error with no secret or path passes through unchanged", () => { + const state = baseState(); + state.nodes.a = node("a", "failed", { error: "boom" }); + + const projected = projectState(state, OPTS); + + expect(projected.nodes.a!.error).toBe("boom"); + }); + + it("a projection with a masked error still passes eventBatchSchema", () => { + const secret = shaped("sk-ant-", "api03-0000VERYFAKE0000VERYFAKE0000"); + const state = baseState(); + state.nodes.a = node("a", "failed", { error: `could not reach the agent: ${secret}` }); + + const projected = projectState(state, OPTS); + const batch: EventBatch = { + runId: projected.runId, + streamId: state.streamId, + graphName: projected.graphName, + state: projected, + events: [], + }; + + const parsed = eventBatchSchema.safeParse(batch); + expect(parsed.success).toBe(true); + }); +}); +describe("safeText strips control characters the wire schema refuses", () => { + // The hub accepts \t \n \r inside a node error (multi-line stack traces are + // legitimate) but still refuses the rest of C0, DEL and ESC, so ANSI colour + // and OSC terminal-title sequences cannot ride in. Many CLI tools colour + // their stderr by default, so an unsanitised ESC would 400 the batch and + // wedge that run's sync permanently - the same failure mode as the newline + // bug, reached by a different route. Stripping belongs here, producer-side. + it("removes ANSI colour codes so the result passes the wire schema", () => { + const out = safeText("\u001b[31mbuild failed\u001b[0m", OPTS); + expect(out).not.toMatch(/\u001b/); + expect(out).toContain("build failed"); + }); + + it("removes OSC, BEL, NUL, VT, FF and DEL", () => { + const out = safeText("a\u0000b\u0007c\u000bd\u000ce\u007ff", OPTS); + expect(out).toBe("abcdef"); + }); + + it("PRESERVES tab, newline and carriage return", () => { + const out = safeText("line1\nline2\tcol\r\nline3", OPTS); + expect(out).toBe("line1\nline2\tcol\r\nline3"); + }); + + it("strips before capping, so the cap still bounds the final text", () => { + const noisy = `${"\u001b[31m".repeat(200)}${"x".repeat(300)}`; + const out = safeText(noisy, OPTS); + expect(out).not.toMatch(/\u001b/); + expect((out ?? "").length).toBeLessThanOrEqual(201); + }); +}); diff --git a/src/team/project.ts b/src/team/project.ts new file mode 100644 index 0000000..3929f47 --- /dev/null +++ b/src/team/project.ts @@ -0,0 +1,158 @@ +import type { RunState } from "../core/types.js"; +import type { ProjectedState, ProjectedNode } from "../hub/wire.js"; +import { SCAN_RULES, rewritePaths } from "../handoff/scan.js"; + +/** + * The machine facts every published string is rewritten against. Mirrors + * `rewritePaths`' own opts (`src/handoff/scan.ts`) rather than a narrower + * `(state, home, repoRoot)` form, because `rewritePaths` skips a protection + * whenever the field it needs is empty - a narrower signature invites a caller + * to pass `""` and silently disable one. `hostname` is REQUIRED for exactly + * that reason: it was optional on `rewritePaths`, no caller on the sync path + * ever supplied it, and the machine hostname published unrewritten for the + * whole of phase 1. Do not make it optional again. + */ +export interface ProjectionIdentity { + home: string; + username: string; + repoRoot: string; + hostname: string; +} + +/** + * Ceiling on a published node error. 200 is the number `claude.ts:33` already + * truncates stdout to, so it matches the largest thing the adapters + * deliberately allow through; a multi-kilobyte stderr dump must not ride along. + */ +const MAX_ERROR_LENGTH = 200; + +const MASK_PREFIX_LENGTH = 4; + +/** + * Mask every secret shape `SCAN_RULES` recognises, in the same at-most- + * four-characters-plus-ellipsis shape as the private `mask()` in + * scan.ts:158-164, with matches of 4 characters or fewer left as-is. Only the + * PRESENTATION is written here; the RULES are imported, never copied, so this + * is not a fork of the scanner - a rule added there starts masking here + * without any edit to this file. + */ +function maskSecrets(text: string): string { + let out = text; + for (const rule of SCAN_RULES) { + // Patterns ship without the `g` flag (scan.ts:26-27); clone with it so the + // replace visits every match, not just the first. + const global = new RegExp(rule.pattern.source, `${rule.pattern.flags}g`); + out = out.replace(global, (match) => { + if (match.length <= MASK_PREFIX_LENGTH) return match; + return `${match.slice(0, MASK_PREFIX_LENGTH)}...`; + }); + } + return out; +} + +/** + * Remove control characters the hub's wire schema refuses, keeping the three + * that legitimately appear in error text. + * + * `projectedNodeSchema.error` permits only TAB, LF and CR; the rest of C0, DEL + * and ESC stay refused so ANSI colour and OSC terminal-title sequences cannot + * ride in. Many CLI tools colour their stderr by default, so an unsanitised ESC + * would 400 the batch and wedge that run's sync PERMANENTLY - the same failure + * as the newline bug, reached by a different route. The producer strips, and the + * hub keeps refusing: validation must never refuse a shape the engine can + * legitimately produce, and the engine must not produce one it refuses. + * + * ORDER MATTERS - this runs FIRST, before rewrite and mask. An ESC spliced into + * a secret or a path defeats their patterns, and stripping afterwards would + * reassemble the original in clear. Full order: strip -> rewrite -> mask -> cap. + * Do not reorder: a masked token is already `first4 + "..."`, so capping cannot + * reveal a fragment, but any other arrangement is exploitable. + */ +function stripControl(s: string): string { + return ( + s + // OSC: ESC ] ... terminated by BEL or ST. Must run before CSI so a title + // sequence is consumed whole rather than leaving its payload behind. + .replace(/\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)?/g, "") + // CSI: ESC [ params intermediates final. Removing only the ESC byte would + // leave "[31m" in the text - and, worse, leave a spliced secret still + // unmatchable by the masker. + .replace(/\u001b\[[0-9;?]*[ -/]*[@-~]?/g, "") + // Any other two-byte escape. + .replace(/\u001b[@-_]?/g, "") + // Remaining C0 and DEL, keeping TAB, LF and CR. + .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, "") + ); +} + +/** + * Sanitise a published string: paths rewritten, secrets masked, length capped. + * + * ORDER IS LOAD-BEARING - rewrite, then mask, then cap. Capping first would let + * a secret straddling the 200th character be truncated below its rule's + * `{16,}` tail, so the mask would no longer match and the surviving prefix + * would publish real key material. Do not reorder these three lines. + * + * Exported because `buildBatch` sanitises event `data` with the SAME function. + * A second implementation over there would drift from this one; there must be + * exactly one definition of "safe to publish" on the sync path. + */ +export function safeText( + error: string | null, + opts: ProjectionIdentity, +): string | null { + if (error === null) return null; + + // Strip FIRST. An ESC spliced into the middle of a secret or an absolute path + // breaks the masker's and the rewriter's patterns; stripping afterwards would + // then reassemble the original in clear. Removing the noise before either one + // runs is what makes them see the real shape. + let out = stripControl(error); + out = rewritePaths(out, opts); + out = maskSecrets(out); + + if (out.length > MAX_ERROR_LENGTH) { + out = `${out.slice(0, MAX_ERROR_LENGTH)}…`; + } + return out; +} + +/** + * Build the wire projection of a run's state. This is where content stops being pushed: + * `vars` VALUES and node `output` are structurally unpublishable, because `ProjectedState` + * has no field that can carry them. The mapping is hand-written field by field - never a + * type-level subtraction over `RunState`, never a mapped type, never an object spread + * followed by deletes - so a future content-carrying field added to `RunState` cannot + * silently start publishing itself. + * + * `opts` is `ProjectionIdentity` - see its doc comment for why every field is required. + */ +export function projectState(state: RunState, opts: ProjectionIdentity): ProjectedState { + const nodes: Record = {}; + for (const [id, node] of Object.entries(state.nodes)) { + nodes[id] = { + nodeId: node.nodeId, + status: node.status, + startedAt: node.startedAt, + endedAt: node.endedAt, + attempts: node.attempts, + error: safeText(node.error, opts), + costUsd: node.costUsd, + }; + } + + return { + runId: state.runId, + graphName: state.graphName, + status: state.status, + createdAt: state.createdAt, + updatedAt: state.updatedAt, + cwd: rewritePaths(state.cwd, opts), + varKeys: Object.keys(state.vars), + budget: state.budget, + spent: state.spent, + nodes, + completed: state.completed, + seq: state.seq, + }; +} \ No newline at end of file diff --git a/src/team/sync-redaction.test.ts b/src/team/sync-redaction.test.ts new file mode 100644 index 0000000..8d95ab8 --- /dev/null +++ b/src/team/sync-redaction.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { EventLog } from "../core/events.js"; +import { CheckpointStore } from "../core/store.js"; +import { execute, newRunState } from "../core/engine.js"; +import { parseGraph } from "../core/graph.js"; +import type { Adapter, AdapterInput, AdapterOutput } from "../adapters/types.js"; +import type { RunState } from "../core/types.js"; +import { HubStore } from "../hub/storage.js"; +import { handle, type HandlerDeps, type WireRequest } from "../hub/handlers.js"; +import type { EventBatch } from "../hub/wire.js"; +import { buildBatch, sanitizeEventLine, syncRun, type ProjectionOpts } from "./sync.js"; +import type { Fetch, HubConfig } from "./transport.js"; + +/** + * THE EVENT STREAM IS A PUBLISHED CHANNEL. `src/team/project.test.ts` proves the + * *projection* is an allowlist; nothing proved anything about the raw event + * lines that ride alongside it in the same push, which is exactly how they came + * to carry unmasked errors, un-rewritten paths and interpolated var values. + * These tests pin the push, not the projection: the local log stays raw and the + * wire does not. + */ + +const FROZEN = "2026-08-25T00:00:00.000Z"; + +/** An anthropic-key shape, so `SCAN_RULES` has a rule that must fire on it. */ +const SECRET = "sk-ant-api03-LEAKLEAKLEAKLEAK1234"; +const HOME = "/home/alice"; +const HOME_PATH = `${HOME}/.config/loomgraph/hub.json`; +const HOSTNAME = "alice-laptop.local"; + +let tmp: string; +let cwd: string; +let eventRoot: string; +let store: CheckpointStore; +let log: EventLog; +let opts: ProjectionOpts; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "loomgraph-redact-")); + cwd = join(tmp, "repo"); + eventRoot = join(cwd, ".loomgraph", "runs"); + store = new CheckpointStore(eventRoot); + log = new EventLog(eventRoot); + opts = { home: HOME, username: "alice", repoRoot: cwd, hostname: HOSTNAME }; +}); + +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +function stub(name: string, out: AdapterOutput): Adapter { + return { name, run: async (_input: AdapterInput) => out }; +} + +/** The real ingest handler, so a sanitized line still has to pass the wire schema. */ +function hub(): { f: Fetch; cfg: HubConfig; pushed: EventBatch[]; statuses: number[] } { + const hubStore = HubStore.open(":memory:", { now: () => FROZEN }); + const token = hubStore.addMember("alice", ["ingest"]).token; + const deps: HandlerDeps = { store: hubStore, now: () => FROZEN, version: "test-v" }; + const pushed: EventBatch[] = []; + const statuses: number[] = []; + const f: Fetch = async (url, init) => { + const body = JSON.parse(init.body ?? "null") as EventBatch; + const req: WireRequest = { + method: init.method, + path: new URL(url).pathname, + query: {}, + headers: { authorization: init.headers.authorization }, + body, + }; + const res = handle(req, deps); + pushed.push(body); + statuses.push(res.status); + return { status: res.status, json: async () => res.body }; + }; + return { f, cfg: { url: "http://hub.test", token }, pushed, statuses }; +} + +async function push(runId: string, state: RunState): Promise<{ wire: string; statuses: number[] }> { + const { f, cfg, pushed, statuses } = hub(); + const result = await syncRun({ f, cfg, cwd, eventRoot, runId, state, opts, timeoutMs: 5000 }); + expect(result.ok).toBe(true); + return { wire: JSON.stringify(pushed), statuses }; +} + +function localLog(runId: string): string { + return readFileSync(join(eventRoot, runId, "events.jsonl"), "utf8"); +} + +const FAILING_GRAPH = ` +name: leaky +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + boom: { type: command, run: "true" } +edges: + - { from: boom, to: END } +`; + +const HUMAN_GRAPH = ` +name: ask +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + ask: { type: human, question: "ship with {{vars.token}}?" } +edges: + - { from: ask, to: END } +`; + +describe("what a sync publishes", () => { + it("a. a node error carrying a secret and an absolute home path reaches the hub with neither", async () => { + const runId = "run-leak"; + const graph = parseGraph(FAILING_GRAPH); + const state = newRunState(graph, { runId, cwd }); + const failed = await execute(graph, state, { + store, + log, + registry: { + command: stub("command", { + ok: false, + text: "", + costUsd: 0, + raw: null, + error: `command exited with code 1: could not read ${HOME_PATH}, key ${SECRET}`, + }), + }, + sleep: async () => {}, + }); + expect(failed.status).toBe("failed"); + + const { wire, statuses } = await push(runId, failed); + + expect(statuses).toEqual([200]); + expect(wire).not.toContain(SECRET); + expect(wire).not.toContain("sk-ant-api03-LEAK"); + expect(wire).not.toContain(HOME_PATH); + expect(wire).not.toContain("/home/alice"); + // The event is still published - sanitized, not dropped. + expect(wire).toContain("node_finished"); + expect(wire).toContain("could not read ${HOME}/.config/loomgraph/hub.json"); + expect(wire).toContain("sk-a..."); + }); + + it("b. a human question that interpolated a secret var does not reach the hub", async () => { + const runId = "run-ask"; + const graph = parseGraph(HUMAN_GRAPH); + const state = newRunState(graph, { runId, cwd, vars: { token: SECRET } }); + const paused = await execute(graph, state, { store, log, registry: {}, sleep: async () => {} }); + expect(paused.status).toBe("paused"); + + const answered = await execute(graph, paused, { + store, + log, + registry: {}, + sleep: async () => {}, + humanAnswers: { ask: `approved, reused ${SECRET}` }, + }); + + const { wire, statuses } = await push(runId, answered); + + expect(statuses).toEqual([200]); + expect(wire).not.toContain(SECRET); + expect(wire).toContain("human_requested"); + expect(wire).toContain("human_resolved"); + // Both the interpolated question and the typed answer survive, masked. + expect(wire).toContain("ship with sk-a...?"); + expect(wire).toContain("approved, reused sk-a..."); + }); + + it("c. the local events.jsonl keeps the raw values - sanitising is a push-time transform only", async () => { + const runId = "run-local"; + const graph = parseGraph(FAILING_GRAPH); + const state = newRunState(graph, { runId, cwd }); + const failed = await execute(graph, state, { + store, + log, + registry: { + command: stub("command", { + ok: false, + text: "", + costUsd: 0, + raw: null, + error: `could not read ${HOME_PATH}, key ${SECRET}`, + }), + }, + sleep: async () => {}, + }); + + const before = localLog(runId); + expect(before).toContain(SECRET); + expect(before).toContain(HOME_PATH); + + await push(runId, failed); + + // Byte-identical after the push: sync writes only the cursor. + expect(localLog(runId)).toBe(before); + expect(localLog(runId)).toContain(SECRET); + expect(localLog(runId)).toContain(HOME_PATH); + }); +}); + +describe("sanitizeEventLine", () => { + const line = (kind: string, data: Record, nodeId?: string): string => + JSON.stringify({ + ts: "2026-08-25T00:00:00.000Z", + runId: "r", + seq: 0, + kind, + ...(nodeId === undefined ? {} : { nodeId }), + data, + }); + + it("rewrites the raw cwd run_started publishes, which the projection rewrote and this did not", () => { + const out = sanitizeEventLine( + line("run_started", { graph: "g", resumed: false, cwd: `${HOME}/work/repo`, streamId: "s" }), + { ...opts, repoRoot: "/nowhere" }, + ); + const data = (JSON.parse(out!) as { data: Record }).data; + expect(data.cwd).toBe("${HOME}/work/repo"); + expect(data).toEqual({ graph: "g", resumed: false, cwd: "${HOME}/work/repo", streamId: "s" }); + }); + + it("drops a data field nobody classified, rather than publishing it", () => { + // The whole point of the allowlist: a field added to an event's `data` + // upstream must NOT start publishing itself just because it exists. + const out = sanitizeEventLine( + line("node_finished", { status: "failed", attempts: 1, costUsd: 0, error: null, stdout: SECRET }), + opts, + ); + expect(out).not.toContain(SECRET); + expect(JSON.parse(out!)).toEqual({ + ts: "2026-08-25T00:00:00.000Z", + runId: "r", + seq: 0, + kind: "node_finished", + data: { status: "failed", attempts: 1, costUsd: 0, error: null }, + }); + }); + + it("drops an unclassifiable line entirely: bad JSON, a non-object, or an unknown kind", () => { + expect(sanitizeEventLine("not json", opts)).toBeNull(); + expect(sanitizeEventLine("[1,2,3]", opts)).toBeNull(); + expect(sanitizeEventLine("null", opts)).toBeNull(); + expect(sanitizeEventLine(line("teleported", { secret: SECRET }), opts)).toBeNull(); + }); + + it("drops a text field whose type changed under us instead of publishing it unsanitised", () => { + const out = sanitizeEventLine(line("human_requested", { question: { raw: SECRET } }, "ask"), opts); + expect(out).not.toContain(SECRET); + expect((JSON.parse(out!) as { data: Record }).data).toEqual({}); + }); + + it("leaves a line with nothing to sanitise byte-identical to its local log line", () => { + const source = line("node_started", { attempt: 1, type: "command" }, "boom"); + expect(sanitizeEventLine(source, opts)).toBe(source); + }); + + it("buildBatch is the choke point: the lines it emits are the sanitised ones", () => { + const graph = parseGraph(FAILING_GRAPH); + const state = newRunState(graph, { runId: "run-b", cwd }); + const batch = buildBatch(state, opts, [ + line("node_finished", { status: "failed", attempts: 1, costUsd: 0, error: SECRET }, "boom"), + "this line cannot be classified", + ]); + expect(batch.events).toHaveLength(1); + expect(batch.events[0]).not.toContain(SECRET); + expect(batch.events[0]).toContain("sk-a..."); + }); +}); + +describe("control characters on the event path", () => { + it("an ANSI-coloured node error goes out clean on the EVENT line too, and the hub accepts the batch", async () => { + // The event path reuses `safeText`, so it inherits control stripping. This + // pins that: an ESC on an event line is not schema-checked by the hub (the + // event `data` record is deliberately permissive), so nothing else would + // catch a regression here - the colour codes would just silently publish. + const runId = "run-ansi"; + const graph = parseGraph(FAILING_GRAPH); + const state = newRunState(graph, { runId, cwd }); + const failed = await execute(graph, state, { + store, + log, + registry: { + command: stub("command", { + ok: false, + text: "", + costUsd: 0, + raw: null, + error: `\u001b[31mbuild failed\u001b[0m reading ${HOME_PATH}\n\tkey ${SECRET}`, + }), + }, + sleep: async () => {}, + }); + + const { wire, statuses } = await push(runId, failed); + + expect(statuses).toEqual([200]); + // Both forms: a raw ESC, and the `\\u001b` text JSON.stringify would escape + // it to. Asserting only the raw char would pass vacuously. + expect(wire).not.toContain("\u001b"); + expect(wire).not.toContain("\\u001b"); + expect(wire).not.toContain("[31m"); + expect(wire).not.toContain(SECRET); + expect(wire).not.toContain(HOME_PATH); + // The newline is legitimate and survives, JSON-escaped, on the event line. + expect(wire).toContain("build failed reading ${HOME}/.config/loomgraph/hub.json"); + expect(wire).toContain("sk-a..."); + + // ...and the escape is still in the local log, where JSON.stringify wrote + // it as the six characters `\\u001b`. + expect(localLog(runId)).toContain("\\u001b[31m"); + expect(localLog(runId)).toContain(SECRET); + }); +}); diff --git a/src/team/sync.test.ts b/src/team/sync.test.ts new file mode 100644 index 0000000..2a6c48a --- /dev/null +++ b/src/team/sync.test.ts @@ -0,0 +1,595 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { EventLog } from "../core/events.js"; +import type { RunState } from "../core/types.js"; +import { HubStore } from "../hub/storage.js"; +import { handle, type HandlerDeps, type WireRequest } from "../hub/handlers.js"; +import { + pendingLines, + readCursor, + syncRun, + writeCursor, + type ProjectionOpts, +} from "./sync.js"; +import type { EventBatch } from "../hub/wire.js"; +import type { Fetch, HubConfig } from "./transport.js"; + +const FROZEN = "2026-08-25T00:00:00.000Z"; + +const OPTS: ProjectionOpts = { + home: "/home/alice", + username: "alice", + repoRoot: "/work/repo", + hostname: "alice-laptop.local", +}; + +const CFG: HubConfig = { url: "http://hub.test", token: "lgt_00000000.FAKEfake0000FAKEfake0000" }; + +let tmp: string; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "loomgraph-sync-")); +}); + +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +function makeState(runId: string): RunState { + return { + runId, + streamId: "11111111-2222-3333-4444-555555555555", + graphName: "g", + status: "running", + createdAt: "2026-08-25T00:00:00.000Z", + updatedAt: "2026-08-25T00:00:01.000Z", + cwd: "/work/repo", + vars: {}, + budget: { maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 10 }, + spent: { usd: 0, wallClockSec: 0, nodeRuns: 0 }, + nodes: {}, + completed: [], + seq: 1, + }; +} + +function cursorFile(cwd: string, runId: string): string { + return join(cwd, ".loomgraph", "sync", `${runId}.cursor`); +} + +function writeEvents(eventRoot: string, runId: string, count: number): void { + const log = new EventLog(eventRoot); + for (let i = 0; i < count; i++) { + log.append(runId, { kind: "run_started", data: {} }); + } +} + +function setupRun(count: number, acked?: number): { cwd: string; eventRoot: string; runId: string } { + const cwd = join(tmp, "repo"); + const eventRoot = join(cwd, ".loomgraph", "runs"); + const runId = "run-sync"; + writeEvents(eventRoot, runId, count); + if (acked !== undefined) writeCursor(cwd, runId, acked); + return { cwd, eventRoot, runId }; +} + +function hubSeam( + store: HubStore, + token: string, + onRequest?: (n: number) => void, +): { fetch: Fetch; requests: Array<{ body: EventBatch; status: number }> } { + const deps: HandlerDeps = { store, now: () => FROZEN, version: "test-v" }; + const requests: Array<{ body: EventBatch; status: number }> = []; + const fetch: Fetch = async (url, init) => { + onRequest?.(requests.length); + const body = JSON.parse(init.body ?? "null") as EventBatch; + const req: WireRequest = { + method: init.method, + path: new URL(url).pathname, + query: {}, + headers: { authorization: init.headers.authorization }, + body, + }; + const res = handle(req, deps); + requests.push({ body, status: res.status }); + return { status: res.status, json: async () => res.body }; + }; + return { fetch, requests }; +} + +function realHub(): { + fetch: Fetch; + cfg: HubConfig; + store: HubStore; + requests: Array<{ body: EventBatch; status: number }>; +} { + const store = HubStore.open(":memory:", { now: () => FROZEN }); + const token = store.addMember("alice", ["ingest"]).token; + const { fetch, requests } = hubSeam(store, token); + return { fetch, cfg: { url: "http://hub.test", token }, store, requests }; +} + +describe("cursor", () => { + it("9. writeCursor then readCursor round-trips", () => { + const cwd = join(tmp, "repo"); + writeCursor(cwd, "run-1", 42); + expect(readCursor(cwd, "run-1")).toEqual({ ackedSeq: 42 }); + writeCursor(cwd, "run-1", 0); + expect(readCursor(cwd, "run-1")).toEqual({ ackedSeq: 0 }); + }); + + it("10. a corrupt cursor reads as null, never throws", () => { + const cwd = join(tmp, "repo"); + const path = cursorFile(cwd, "run-1"); + const corruptions = [ + '{"ackedSe', // truncated mid-JSON + "", // empty file + "not json at all", // plain text + '{"ackedSeq":"seven"}', // string value, not a number + '{"ackedSeq":-1}', // negative : a real ack is never negative + '{"ackedSeq":2.5}', // non-integer + '{"ackedSeq":null}', // null value + "[1,2,3]", // valid JSON, not the object we expect + ]; + for (const content of corruptions) { + mkdirSync(join(cwd, ".loomgraph", "sync"), { recursive: true }); + writeFileSync(path, content, "utf8"); + expect(readCursor(cwd, "run-1")).toBeNull(); + } + expect(() => readCursor(cwd, "run-1")).not.toThrow(); + }); +}); + +describe("pendingLines", () => { + it("11. returns only lines whose seq is above ackedSeq, preserving order", () => { + const lines = [0, 1, 2, 3, 4].map((seq) => + JSON.stringify({ ts: "2026-08-25T00:00:00.000Z", runId: "r", seq, kind: "run_started", data: {} }), + ); + expect(pendingLines(lines, 2)).toEqual([lines[3], lines[4]]); + expect(pendingLines(lines, -1)).toEqual(lines); + expect(pendingLines(lines, 4)).toEqual([]); + const late = JSON.stringify({ + ts: "2026-08-25T00:01:00.000Z", + runId: "r", + seq: 7, + kind: "run_started", + data: {}, + }); + expect(pendingLines([...lines, late], 5)).toEqual([late]); + }); + + it("an unreadable line is never treated as pending", () => { + expect(pendingLines(["this is not json"], 0)).toEqual([]); + }); +}); + +describe("syncRun", () => { + it("the cursor never passes the last ack: every call fails and the cursor sits exactly where it started", async () => { + const { cwd, eventRoot, runId } = setupRun(8, 5); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = async () => { + throw new Error("connect ECONNRESET"); + }; + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 5 }); + }); + + it("12. a successful sync advances the cursor to the returned highWaterSeq", async () => { + const { cwd, eventRoot, runId } = setupRun(4); + const fetch: Fetch = async () => ({ status: 200, json: async () => ({ highWaterSeq: 3 }) }); + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result).toEqual({ ok: true, ackedSeq: 3 }); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 3 }); + expect(readFileSync(cursorFile(cwd, runId), "utf8")).toBe('{"ackedSeq":3}\n'); + }); + + it("13. a failed sync leaves the cursor byte-identical", async () => { + const { cwd, eventRoot, runId } = setupRun(6, 1); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = async () => ({ status: 500, json: async () => ({ error: "boom" }) }); + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 1 }); + }); + + it("a timeout leaves the cursor unchanged", async () => { + const { cwd, eventRoot, runId } = setupRun(6, 1); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = (_url, init) => + new Promise<{ status: number; json(): Promise }>((_resolve, reject) => { + init.signal?.addEventListener("abort", () => { + const err = new Error("signal aborted"); + err.name = "AbortError"; + reject(err); + }); + }); + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 25, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("a malformed 2xx body leaves the cursor unchanged", async () => { + const { cwd, eventRoot, runId } = setupRun(6, 1); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = async () => ({ status: 200, json: async () => ({ nope: 1 }) }); + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("json() rejecting leaves the cursor unchanged", async () => { + const { cwd, eventRoot, runId } = setupRun(6, 1); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = async () => ({ + status: 200, + json: async () => { + throw new Error("bad json"); + }, + }); + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + }); + + it("14. windowing: first batch carries 500, the cursor advances, and a mid-run second batch is accepted by the real handler", async () => { + const { cwd, eventRoot, runId } = setupRun(600); + const store = HubStore.open(":memory:", { now: () => FROZEN }); + const token = store.addMember("alice", ["ingest"]).token; + const { fetch, requests } = hubSeam(store, token, (n) => { + if (n === 1) { + // Between batch 1 and batch 2 the cursor must already name the first high-water mark. + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 499 }); + } + }); + const cfg: HubConfig = { url: "http://hub.test", token }; + + const result = await syncRun({ + f: fetch, + cfg, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + + expect(result.ok).toBe(true); + expect(requests).toHaveLength(2); + + const first = requests[0]!.body; + expect(first.events).toHaveLength(500); + expect((JSON.parse(first.events[0]!) as { seq: number }).seq).toBe(0); + expect(requests[0]!.status).toBe(200); + + const second = requests[1]!.body; + expect(second.events).toHaveLength(100); + expect((JSON.parse(second.events[0]!) as { seq: number }).seq).toBe(500); + expect(requests[1]!.status).toBe(200); + + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 599 }); + const stored = store.events("alice", makeState(runId).streamId, runId); + expect(stored).toHaveLength(600); + expect(store.listRuns("alice")).toHaveLength(1); + }); + + it("15. sync writes nothing under runs/", async () => { + const cwd = join(tmp, "repo"); + const eventRoot = join(cwd, ".loomgraph", "runs"); + const runId = "run-ro"; + writeEvents(eventRoot, runId, 4); + + const snapshot = (): Array<{ path: string; mtimeMs: number; bytes: number }> => { + const out: Array<{ path: string; mtimeMs: number; bytes: number }> = []; + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, entry.name); + if (entry.isDirectory()) walk(p); + else out.push({ path: p, mtimeMs: statSync(p).mtimeMs, bytes: statSync(p).size }); + } + }; + walk(eventRoot); + return out.sort((a, b) => a.path.localeCompare(b.path)); + }; + + const before = snapshot(); + const { fetch, cfg } = realHub(); + const result = await syncRun({ + f: fetch, + cfg, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + expect(result.ok).toBe(true); + expect(snapshot()).toEqual(before); + }); + + it("16. a cut mid-batch resends the identical events array", async () => { + const { cwd, eventRoot, runId } = setupRun(6); + const store = HubStore.open(":memory:", { now: () => FROZEN }); + const token = store.addMember("alice", ["ingest"]).token; + const deps: HandlerDeps = { store, now: () => FROZEN, version: "test-v" }; + const bodies: string[] = []; + let calls = 0; + const fetch: Fetch = async (url, init) => { + bodies.push(init.body ?? ""); + calls += 1; + if (calls === 1) { + // The hub is off: connection refused, so this attempt stores nothing. + const err = new Error("connect ECONNREFUSED 127.0.0.1:8080"); + (err as NodeJS.ErrnoException).code = "ECONNREFUSED"; + throw err; + } + const body = JSON.parse(init.body ?? "null") as EventBatch; + const res = handle( + { + method: init.method, + path: new URL(url).pathname, + query: {}, + headers: { authorization: init.headers.authorization }, + body, + }, + deps, + ); + return { status: res.status, json: async () => res.body }; + }; + const cfg: HubConfig = { url: "http://hub.test", token }; + + const firstAttempt = await syncRun({ + f: fetch, + cfg, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + expect(firstAttempt.ok).toBe(false); + + const secondAttempt = await syncRun({ + f: fetch, + cfg, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + expect(secondAttempt.ok).toBe(true); + + expect(bodies).toHaveLength(2); + const firstBatch = JSON.parse(bodies[0]!) as EventBatch; + const secondBatch = JSON.parse(bodies[1]!) as EventBatch; + expect(firstBatch.events).toHaveLength(6); + expect(secondBatch.events).toEqual(firstBatch.events); + expect(secondBatch.runId).toBe(runId); + }); + + it("17. a kill between windows leaves the cursor on the first window's ack, and the retry resends from exactly that point", async () => { + const { cwd, eventRoot, runId } = setupRun(600); + let calls = 0; + const failingFetch: Fetch = async (_url, _init) => { + calls += 1; + if (calls === 1) return { status: 200, json: async () => ({ highWaterSeq: 499 }) }; + throw new Error("socket hang up mid-sync"); + }; + const first = await syncRun({ + f: failingFetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + expect(first.ok).toBe(false); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 499 }); + expect(readFileSync(cursorFile(cwd, runId), "utf8")).toBe('{"ackedSeq":499}\n'); + + let resend: EventBatch | null = null; + const workingFetch: Fetch = async (_url, init) => { + resend = JSON.parse(init.body ?? "null") as EventBatch; + return { status: 200, json: async () => ({ highWaterSeq: 599 }) }; + }; + const second = await syncRun({ + f: workingFetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 5000, + }); + expect(second).toEqual({ ok: true, ackedSeq: 599 }); + + const seqs = resend!.events.map((line) => (JSON.parse(line) as { seq: number }).seq); + expect(seqs).toEqual(Array.from({ length: 100 }, (_, i) => 500 + i)); + }); + + it("18. a window that never settles still fails the sync, bounded by the timer, leaving the cursor on the first ack", async () => { + const { cwd, eventRoot, runId } = setupRun(600); + let calls = 0; + const hangFetch: Fetch = (_url, _init) => { + calls += 1; + if (calls === 1) return Promise.resolve({ status: 200, json: async () => ({ highWaterSeq: 499 }) }); + return new Promise<{ status: number; json(): Promise }>(() => {}); + }; + const result = await syncRun({ + f: hangFetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 25, + }); + expect(result.ok).toBe(false); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 499 }); + }); + + it("19. a failure on the very first window, with no existing cursor, writes no cursor file at all", async () => { + const { cwd, eventRoot, runId } = setupRun(4); + const fetch: Fetch = async () => { + throw new Error("connect ECONNREFUSED 127.0.0.1:8080"); + }; + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(existsSync(cursorFile(cwd, runId))).toBe(false); + }); + + it("20. a 2xx high-water below the cursor rewinds it on purpose, and a healthy retry resends the recovered range", async () => { + // The downward move is DELIBERATE. When the hub reports a high-water BELOW the on-disk + // cursor - a hub restored from backup, or a different hub at the same URL - the rewind is + // the recovery path: pendingLines only ever sends events above the cursor, so a cursor + // pinned above the hub would silently drop everything below it and nothing self-heals. + // A max(cursor, highWaterSeq) guard would pin the cursor above the hub forever and make + // the gap permanent. Do not add such a guard. + const { cwd, eventRoot, runId } = setupRun(8, 5); + const rewindFetch: Fetch = async () => ({ status: 200, json: async () => ({ highWaterSeq: 3 }) }); + const first = await syncRun({ + f: rewindFetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(first).toEqual({ ok: true, ackedSeq: 3 }); + expect(readCursor(cwd, runId)).toEqual({ ackedSeq: 3 }); + expect(readFileSync(cursorFile(cwd, runId), "utf8")).toBe('{"ackedSeq":3}\n'); + + let resend: EventBatch | null = null; + const healthyFetch: Fetch = async (_url, init) => { + resend = JSON.parse(init.body ?? "null") as EventBatch; + return { status: 200, json: async () => ({ highWaterSeq: 7 }) }; + }; + const second = await syncRun({ + f: healthyFetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(second).toEqual({ ok: true, ackedSeq: 7 }); + + const seqs = resend!.events.map((line) => (JSON.parse(line) as { seq: number }).seq); + expect(seqs).toEqual([4, 5, 6, 7]); + }); + + it("21. a failed sync leaves the cursor file byte-identical", async () => { + const { cwd, eventRoot, runId } = setupRun(6, 1); + const path = cursorFile(cwd, runId); + const before = readFileSync(path, "utf8"); + const fetch: Fetch = async () => { + throw new Error("connect ECONNRESET"); + }; + const result = await syncRun({ + f: fetch, + cfg: CFG, + cwd, + eventRoot, + runId, + state: makeState(runId), + opts: OPTS, + timeoutMs: 1000, + }); + expect(result.ok).toBe(false); + expect(readFileSync(path, "utf8")).toBe(before); + }); +}); \ No newline at end of file diff --git a/src/team/sync.ts b/src/team/sync.ts new file mode 100644 index 0000000..f3b9955 --- /dev/null +++ b/src/team/sync.ts @@ -0,0 +1,262 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { EventLog, type LgEventKind } from "../core/events.js"; +import type { RunState } from "../core/types.js"; +import type { EventBatch } from "../hub/wire.js"; +import { projectState, safeText, type ProjectionIdentity } from "./project.js"; +import { postEvents, type Fetch, type HubConfig } from "./transport.js"; + +/** At most this many event lines per request. */ +const BATCH_LIMIT = 500; + +/** + * The identity `projectState` and `sanitizeEventLine` both need. An alias, not + * a second declaration: two copies would let one gain a field the other lacks, + * which is precisely how `hostname` came to be missing here while + * `rewritePaths` had accepted it all along. + */ +export type ProjectionOpts = ProjectionIdentity; + +/** What `readCursor` returns: the highest event seq the hub has acked. */ +export interface Cursor { + ackedSeq: number; +} + +function cursorPath(cwd: string, runId: string): string { + return join(cwd, ".loomgraph", "sync", `${runId}.cursor`); +} + +/** + * A CORRUPT CURSOR MEANS ABSENT. Truncated, empty, non-JSON, a string + * `ackedSeq`, or a negative / non-integer value all read as null, and sync + * resends from the start. The server is idempotent by primary key, so resending + * is free, whereas trusting a garbage value would skip events permanently. + * + * NEVER throws - one bad byte must not break a sync. + */ +export function readCursor(cwd: string, runId: string): Cursor | null { + try { + const parsed: unknown = JSON.parse(readFileSync(cursorPath(cwd, runId), "utf8")); + if (parsed === null || typeof parsed !== "object") return null; + const n = (parsed as Record).ackedSeq; + if (typeof n !== "number" || !Number.isInteger(n) || n < 0) return null; + return { ackedSeq: n }; + } catch { + return null; + } +} + +/** + * Temp-then-rename, exactly as CheckpointStore.save does it: write + * `.tmp`, then rename over the target, so a process killed between the + * write and the rename leaves only a stray `.tmp`, never a torn cursor. + * + * NO LOCK FILES. Concurrent writers are safe by temp-then-rename plus server + * idempotency - the last writer wins and either batch covers the same run. + * Do not add locking here. + */ +export function writeCursor(cwd: string, runId: string, ackedSeq: number): void { + const dir = join(cwd, ".loomgraph", "sync"); + mkdirSync(dir, { recursive: true }); + const target = cursorPath(cwd, runId); + const tmp = `${target}.tmp`; + writeFileSync(tmp, `${JSON.stringify({ ackedSeq })}\n`, "utf8"); + renameSync(tmp, target); +} + +/** + * The lines that still need pushing: those whose seq is strictly above + * `ackedSeq`. An unreadable line is not one we can ack, so it is left out of + * the batch rather than risking a stale ack over something we could not parse. + */ +export function pendingLines(lines: string[], ackedSeq: number): string[] { + const pending: string[] = []; + for (const line of lines) { + try { + const parsed: unknown = JSON.parse(line); + const seq = (parsed as { seq?: unknown }).seq; + if (typeof seq === "number" && seq > ackedSeq) pending.push(line); + } catch { + // Not a line we can ack; never trust an unparsed line to skip anything. + } + } + return pending; +} + +/** + * WHICH `data` FIELDS EACH EVENT KIND MAY PUBLISH, AND HOW. + * + * The same hand-written allowlist discipline `projectState` uses, for the same + * reason and with the same rule: a future content-carrying field must not + * silently start publishing itself. Every field is named here; anything not + * named is DROPPED from the pushed line. Never replace this with a generic walk + * over `data`, and never add a field without deciding which column it belongs + * in. + * + * "pass" the value is an engine- or graph-derived identifier, enum, number + * or boolean. It carries no operator content, and the projection + * already publishes the same class of fact (node ids, graph name, + * costs, budgets). Copied as-is. + * "text" the value is operator- or environment-derived text: an adapter + * error (which can be a whole agent result or a raw stderr dump), an + * absolute cwd, an INTERPOLATED human question (`{{vars.x}}` and + * `{{nodes.x.output}}` already substituted), or a typed answer. Run + * through `safeText` - the same rewrite/mask/cap the projection + * applies to a node error. + * + * `budget_exceeded.reason` is engine-generated and could be "pass"; it is + * "text" because it costs nothing and a string that reaches the wire should + * have gone through the sanitiser unless there is a reason it cannot. + */ +const EVENT_DATA_ALLOWLIST: Record> = { + run_started: { graph: "pass", resumed: "pass", cwd: "text", streamId: "pass" }, + node_started: { attempt: "pass", type: "pass" }, + node_finished: { status: "pass", attempts: "pass", costUsd: "pass", error: "text" }, + edge_crossed: { from: "pass", to: "pass", when: "pass" }, + budget_checked: { spent: "pass", budget: "pass", ready: "pass" }, + budget_exceeded: { reason: "text", spent: "pass", budget: "pass" }, + human_requested: { question: "text" }, + human_resolved: { answer: "text" }, + run_finished: { status: "pass", error: "text", spent: "pass" }, +}; + +function isEventKind(value: unknown): value is LgEventKind { + return typeof value === "string" && value in EVENT_DATA_ALLOWLIST; +} + +/** + * SANITISE ONE EVENT LINE FOR THE WIRE. THE LOCAL LOG IS NEVER TOUCHED. + * + * `.loomgraph/runs//events.jsonl` stays raw and complete - that is the + * author's own debugging record and it must keep full fidelity. This transform + * runs at PUSH time, on the copy that crosses to the hub, so `lg-hub export` + * still reproduces the INGESTED lines byte for byte; those lines simply stop + * carrying secrets. + * + * The event object is rebuilt field by field rather than mutated, for the same + * reason `projectState` is: no spread, no delete, no unknown key riding along. + * + * Returns null when the line cannot be classified - unparseable, not an object, + * or an unrecognised kind. An unclassifiable line is DROPPED, never passed + * through: a kind this function does not know is a kind whose `data` nobody has + * reviewed. Dropping costs an audit line; passing through costs a leak. + */ +export function sanitizeEventLine(line: string, opts: ProjectionOpts): string | null { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + return null; + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return null; + + const event = parsed as Record; + if (!isEventKind(event.kind)) return null; + + const fields = EVENT_DATA_ALLOWLIST[event.kind]; + const raw = event.data; + const source: Record = + raw !== null && typeof raw === "object" && !Array.isArray(raw) + ? (raw as Record) + : {}; + + const data: Record = {}; + for (const [key, handling] of Object.entries(fields)) { + if (!(key in source)) continue; + const value = source[key]; + if (handling === "pass") { + data[key] = value; + continue; + } + // "text": a value that is neither a string nor null is a field whose shape + // changed under us, so it is dropped rather than published unsanitised. + if (value === null) data[key] = null; + else if (typeof value === "string") data[key] = safeText(value, opts); + } + + // Same key order the engine writes (`src/core/events.ts`), so an event with + // nothing to sanitise serialises byte-identically to its local log line. + return JSON.stringify({ + ts: event.ts, + runId: event.runId, + seq: event.seq, + kind: event.kind, + ...(typeof event.nodeId === "string" ? { nodeId: event.nodeId } : {}), + data, + }); +} + +/** + * Assemble one push: `runId`, `streamId` and `graphName` come from the loaded + * RunState, `state` is the projected projection, and `events` are the lines + * chosen by `pendingLines`, each sanitised for the wire. The projection is + * recomputed per batch so the `updatedAt`/`seq` the hub hears tracks the state + * that was current for that window. + * + * SANITISING HAPPENS HERE, not in `syncRun`, because `buildBatch` is the single + * choke point both push paths go through: `lg sync` and the live `LiveBatcher` + * in `./batch.ts`. Moving it up into `syncRun` would leave the live stream + * publishing raw lines. + */ +export function buildBatch(state: RunState, opts: ProjectionOpts, lines: string[]): EventBatch { + const events: string[] = []; + for (const line of lines) { + const safe = sanitizeEventLine(line, opts); + if (safe !== null) events.push(safe); + } + + return { + runId: state.runId, + streamId: state.streamId, + graphName: state.graphName, + state: projectState(state, opts), + events, + }; +} + +export interface SyncDeps { + f: Fetch; + cfg: HubConfig; + /** Repo root; holds `.loomgraph/sync/.cursor`. */ + cwd: string; + /** EventLog root; production passes `/.loomgraph/runs/`. */ + eventRoot: string; + runId: string; + state: RunState; + opts: ProjectionOpts; + timeoutMs?: number; +} + +export type SyncResult = { ok: true; ackedSeq: number } | { ok: false; error: string }; + +/** + * Push a whole run to the hub in windows of at most 500 lines. Local events are + * read with `EventLog.read` and sanitised by `buildBatch` on the way out - the + * only file sync writes is the cursor under `.loomgraph/sync/`; nothing under + * `runs/` is ever touched, and the log on disk keeps its raw values. + * + * The cursor advances ONLY on a 2xx naming `highWaterSeq`. Any `{ok:false}` + * leaves the cursor exactly as it was, so a cut mid-batch resends the same + * lines next time - resending is free because the server is idempotent by + * primary key. + */ +export async function syncRun(deps: SyncDeps): Promise { + const lines = new EventLog(deps.eventRoot).read(deps.runId).map((e) => JSON.stringify(e)); + const cursor = readCursor(deps.cwd, deps.runId); + const acked = cursor === null ? -1 : cursor.ackedSeq; + const pending = pendingLines(lines, acked); + + let lastAcked = acked; + for (let i = 0; i < pending.length; i += BATCH_LIMIT) { + const chunk = pending.slice(i, i + BATCH_LIMIT); + const batch = buildBatch(deps.state, deps.opts, chunk); + const result = await postEvents(deps.f, deps.cfg, batch, deps.timeoutMs ?? 10_000); + if (!result.ok) { + return { ok: false, error: result.error }; + } + lastAcked = result.highWaterSeq; + writeCursor(deps.cwd, deps.runId, result.highWaterSeq); + } + + return { ok: true, ackedSeq: lastAcked }; +} \ No newline at end of file diff --git a/src/team/transport.test.ts b/src/team/transport.test.ts new file mode 100644 index 0000000..0e8be34 --- /dev/null +++ b/src/team/transport.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { EventBatch } from "../hub/wire.js"; +import { loadHubConfig, postEvents, repoSyncEnabled, type Fetch, type HubConfig } from "./transport.js"; + +function batch(): EventBatch { + return { + runId: "run-1", + streamId: "s-1", + graphName: "g", + state: { + runId: "run-1", + graphName: "g", + status: "running", + createdAt: "2026-08-25T00:00:00.000Z", + updatedAt: "2026-08-25T00:00:01.000Z", + cwd: "${REPO_ROOT}", + varKeys: [], + budget: { maxUsd: 1, maxWallClockSec: 60, maxNodeRuns: 10 }, + spent: { usd: 0, wallClockSec: 0, nodeRuns: 0 }, + nodes: {}, + completed: [], + seq: 1, + }, + events: [], + }; +} + +const cfg: HubConfig = { url: "http://hub.test", token: "lgt_00000000.FAKEfake0000FAKEfake0000" }; + +function okFetch(status: number, body: unknown): Fetch { + return async () => ({ status, json: async () => body }); +} + +describe("postEvents", () => { + it("1. returns ok with the highWaterSeq on a 200", async () => { + const f = okFetch(200, { highWaterSeq: 7, accepted: 2, duplicates: 0 }); + const result = await postEvents(f, cfg, batch(), 1000); + expect(result).toEqual({ ok: true, highWaterSeq: 7 }); + }); + + it("2. a non-2xx status becomes {ok:false}", async () => { + const f = okFetch(500, { error: "boom" }); + const result = await postEvents(f, cfg, batch(), 1000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("500"); + }); + + it("3. f rejecting (ECONNREFUSED) becomes {ok:false} and the call does not reject", async () => { + const f: Fetch = async () => { + const err = new Error("connect ECONNREFUSED 127.0.0.1:8080"); + (err as NodeJS.ErrnoException).code = "ECONNREFUSED"; + throw err; + }; + const result = await postEvents(f, cfg, batch(), 1000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("ECONNREFUSED"); + }); + + it("4. the timeout firing becomes {ok:false} - no hang, no rejection", async () => { + const f: Fetch = (_url, init) => + new Promise<{ status: number; json(): Promise }>((_resolve, reject) => { + init.signal?.addEventListener("abort", () => { + const err = new Error("signal aborted"); + err.name = "AbortError"; + reject(err); + }); + }); + const started = Date.now(); + const result = await postEvents(f, cfg, batch(), 25); + expect(Date.now() - started).toBeLessThan(5000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("timed out"); + }); + + it("4a. a transport that ignores the abort signal is still bounded", async () => { + const f: Fetch = () => new Promise(() => {}); // never settles, ignores the signal + const started = Date.now(); + const result = await postEvents(f, cfg, batch(), 25); + expect(Date.now() - started).toBeLessThan(5000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toBe("hub request timed out after 25ms"); + }); + + it("4b. with a timeout of 0, a cooperative transport still resolves normally", async () => { + const f = okFetch(200, { highWaterSeq: 7, accepted: 2, duplicates: 0 }); + const result = await postEvents(f, cfg, batch(), 0); + expect(result).toEqual({ ok: true, highWaterSeq: 7 }); + }); + + it("5. json() rejecting becomes {ok:false}", async () => { + const f: Fetch = async () => ({ + status: 200, + json: async () => { + throw new Error("Unexpected token < in JSON"); + }, + }); + const result = await postEvents(f, cfg, batch(), 1000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("Unexpected token"); + }); + + it("6. a 200 whose body has no numeric highWaterSeq becomes {ok:false}", async () => { + const nonNumeric: unknown[] = [ + { error: "something else" }, + { highWaterSeq: "3" }, + { highWaterSeq: null }, + null, + "plain text", + 42, + ]; + for (const body of nonNumeric) { + const result = await postEvents(okFetch(200, body), cfg, batch(), 1000); + expect(result.ok).toBe(false); + } + }); +}); + +describe("loadHubConfig", () => { + let tmp: string; + let home: string; + + const writeHub = (content: string): void => { + mkdirSync(join(home, ".config", "loomgraph"), { recursive: true }); + writeFileSync(join(home, ".config", "loomgraph", "hub.json"), content, "utf8"); + }; + + const env = (url: string | undefined, token: string | undefined): NodeJS.ProcessEnv => ({ + ...(url === undefined ? {} : { LOOMGRAPH_HUB_URL: url }), + ...(token === undefined ? {} : { LOOMGRAPH_HUB_TOKEN: token }), + }); + + it("7a. env wins over the config file", () => { + writeHub(JSON.stringify({ url: "http://file", token: "file-token" })); + const result = loadHubConfig(env("http://env", "env-token"), home); + expect(result).toEqual({ url: "http://env", token: "env-token" }); + }); + + it("7b. the config file is used when env vars are absent", () => { + writeHub(JSON.stringify({ url: "http://file", token: "file-token" })); + const result = loadHubConfig({}, home); + expect(result).toEqual({ url: "http://file", token: "file-token" }); + }); + + it("7c. only a url set -> null (no usable file either)", () => { + expect(loadHubConfig(env("http://env", undefined), home)).toBeNull(); + }); + + it("7d. only a token set -> null (no usable file either)", () => { + expect(loadHubConfig(env(undefined, "env-token"), home)).toBeNull(); + }); + + it("7e. a missing config file -> null", () => { + expect(loadHubConfig({}, home)).toBeNull(); + }); + + it("7f. a non-JSON config file -> null", () => { + writeHub("this is {not json]"); + expect(loadHubConfig({}, home)).toBeNull(); + }); + + it("7g. JSON missing a field -> null", () => { + writeHub(JSON.stringify({ url: "http://file" })); + expect(loadHubConfig({}, home)).toBeNull(); + writeHub(JSON.stringify({ token: "file-token" })); + expect(loadHubConfig({}, home)).toBeNull(); + }); + + it("7h. whitespace-only env credentials are treated as absent, so the file is consulted", () => { + writeHub(JSON.stringify({ url: "http://file", token: "file-token" })); + expect(loadHubConfig(env(" ", "env-token"), home)).toEqual({ + url: "http://file", + token: "file-token", + }); + expect(loadHubConfig(env("http://env", " "), home)).toEqual({ + url: "http://file", + token: "file-token", + }); + // Without a usable file, whitespace-only credentials mean no config at all. + expect(loadHubConfig(env(" ", "env-token"), join(tmp, "empty"))).toBeNull(); + }); + + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "loomgraph-transport-")); + home = join(tmp, "home"); + }); + + afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); + }); +}); + +describe("repoSyncEnabled", () => { + let tmp: string; + let cwd: string; + + const writeOptIn = (content: string): void => { + mkdirSync(join(cwd, ".loomgraph"), { recursive: true }); + writeFileSync(join(cwd, ".loomgraph", "hub.json"), content, "utf8"); + }; + + it("8a. true only for {\"sync\":true}", () => { + writeOptIn(JSON.stringify({ sync: true })); + expect(repoSyncEnabled(cwd)).toBe(true); + }); + + it("8b. a missing opt-in file -> false", () => { + expect(repoSyncEnabled(cwd)).toBe(false); + }); + + it("8c. a non-JSON opt-in file -> false", () => { + writeOptIn("sync=true oops"); + expect(repoSyncEnabled(cwd)).toBe(false); + }); + + it("8d. {\"sync\":\"yes\"} -> false", () => { + writeOptIn(JSON.stringify({ sync: "yes" })); + expect(repoSyncEnabled(cwd)).toBe(false); + }); + + it("8e. {} -> false", () => { + writeOptIn(JSON.stringify({})); + expect(repoSyncEnabled(cwd)).toBe(false); + }); + + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "loomgraph-optin-")); + cwd = join(tmp, "repo"); + }); + + afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); + }); +}); \ No newline at end of file diff --git a/src/team/transport.ts b/src/team/transport.ts new file mode 100644 index 0000000..d1be436 --- /dev/null +++ b/src/team/transport.ts @@ -0,0 +1,171 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import type { EventBatch } from "../hub/wire.js"; + +/** + * The single network seam for the client, shaped after the subset of the global + * `fetch` we use. Tests inject a fake; the real global `fetch` is only ever + * reached through this seam and no test in this repo uses it. + * + * Every request carries an `AbortSignal` so a caller can bound the request with + * an `AbortController` without touching the seam's implementation. + */ +export type Fetch = ( + url: string, + init: { + method: string; + headers: Record; + body?: string; + signal?: AbortSignal; + }, +) => Promise<{ status: number; json(): Promise }>; + +/** Where to push and who we are. Never printed and never logged. */ +export interface HubConfig { + url: string; + token: string; +} + +const HUB_URL_KEY = "LOOMGRAPH_HUB_URL"; +const HUB_TOKEN_KEY = "LOOMGRAPH_HUB_TOKEN"; + +/** + * Resolve the hub identity config: env first, then `~/.config/loomgraph/hub.json` + * (`{url, token}`), else null. BOTH parts must be present - a url with no token + * is null, not a partial config, because a config that cannot authenticate is a + * config that would be retried forever. + * + * NEVER throws. A missing file, an unreadable file, non-JSON content, or JSON + * missing either field all mean null. Commit 1.13 calls this from inside a live + * run, and a hub outage must not affect a run - an absent hub must read as "no + * hub configured", never as an error. + */ +export function loadHubConfig(env: NodeJS.ProcessEnv, home: string): HubConfig | null { + if (isNonEmptyString(env[HUB_URL_KEY]) && isNonEmptyString(env[HUB_TOKEN_KEY])) { + return { url: env[HUB_URL_KEY] as string, token: env[HUB_TOKEN_KEY] as string }; + } + try { + const parsed: unknown = JSON.parse( + readFileSync(join(home, ".config", "loomgraph", "hub.json"), "utf8"), + ); + if (parsed !== null && typeof parsed === "object") { + const url = (parsed as Record).url; + const token = (parsed as Record).token; + if (isNonEmptyString(url) && isNonEmptyString(token)) { + return { url, token }; + } + } + } catch { + // Missing, unreadable or non-JSON identity file reads as "not configured". + } + return null; +} + +/** + * The repo-level opt-in flag. `/.loomgraph/hub.json` holds only + * `{"sync": true}` and NEVER contains a token - it is the repo's consent to + * sync, while the identity (url + token) lives in `~/.config/loomgraph/hub.json`. + * Two different files, two different jobs; keep them separate. + * + * NEVER throws. Missing, unreadable, non-JSON, or `sync` anything other than + * `true` all read as disabled. + */ +export function repoSyncEnabled(cwd: string): boolean { + try { + const parsed: unknown = JSON.parse(readFileSync(join(cwd, ".loomgraph", "hub.json"), "utf8")); + if (parsed !== null && typeof parsed === "object") { + return (parsed as Record).sync === true; + } + } catch { + // No opt-in file reads as no opt-in. + } + return false; +} + +/** + * Push one batch to `POST {url}/v1/events`. + * + * NEVER REJECTS AND NEVER THROWS. Every failure becomes `{ok:false}` with a + * useful message: a non-2xx status, the timeout firing, `f` itself rejecting + * (connection refused / DNS failure - a dead hub is the most likely real + * condition and must not surface as an unhandled rejection), `json()` rejecting, + * or a 2xx whose body is not an object carrying a numeric `highWaterSeq`. + * + * The timer is cleared in a finally so a fast response cannot leave a dangling + * timer behind. + */ +export async function postEvents( + f: Fetch, + cfg: HubConfig, + batch: EventBatch, + timeoutMs: number, +): Promise<{ ok: true; highWaterSeq: number } | { ok: false; error: string }> { + const controller = new AbortController(); + let timer: ReturnType | undefined; + try { + // The timeout must hold even when the transport ignores the abort signal: + // the batcher in commit 1.13 relies on this bound to guarantee that a hub + // outage cannot delay a run. Racing the transport against the timer is the + // backstop; aborting the controller is still done so a cooperative fetch + // releases its socket. A future "simplification" back to signal-only + // reintroduces an unbounded wait. + const init = { + method: "POST", + headers: { + authorization: `Bearer ${cfg.token}`, + "content-type": "application/json", + }, + body: JSON.stringify(batch), + signal: controller.signal, + }; + const timeout = + timeoutMs <= 0 + ? null + : new Promise((_resolve, reject) => { + timer = setTimeout(() => { + controller.abort(); + const err = new Error("hub request timed out"); + err.name = "AbortError"; + reject(err); + }, timeoutMs); + timer.unref(); + }); + const res = + timeout === null + ? await f(`${cfg.url}/v1/events`, init) + : await Promise.race([f(`${cfg.url}/v1/events`, init), timeout]); + if (res.status < 200 || res.status >= 300) { + return { ok: false, error: `hub rejected the batch: HTTP ${res.status}` }; + } + const body: unknown = await res.json(); + if (body === null || typeof body !== "object") { + return { ok: false, error: "hub returned a non-object body on 2xx" }; + } + const highWaterSeq = (body as Record).highWaterSeq; + if (typeof highWaterSeq !== "number" || !Number.isInteger(highWaterSeq)) { + return { ok: false, error: "hub 2xx response names no numeric highWaterSeq" }; + } + return { ok: true, highWaterSeq }; + } catch (err) { + if (isAbortError(err)) { + return { ok: false, error: `hub request timed out after ${timeoutMs}ms` }; + } + return { ok: false, error: `hub request failed: ${errorText(err)}` }; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function isNonEmptyString(v: unknown): v is string { + return typeof v === "string" && v.trim() !== ""; +} + +function isAbortError(err: unknown): boolean { + // Node surfaces timeout-driven aborts as an AbortError rejection, not a status. + return err instanceof Error && err.name === "AbortError"; +} + +function errorText(err: unknown): string { + if (err instanceof Error) return err.message; + return String(err); +} \ No newline at end of file diff --git a/tsup.config.ts b/tsup.config.ts index 988e810..f868775 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,11 +1,16 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/cli.ts", "src/handoff/cli.ts"], + entry: ["src/index.ts", "src/cli.ts", "src/handoff/cli.ts", "src/hub/cli.ts"], format: ["esm"], target: "node22", platform: "node", clean: true, sourcemap: true, dts: false, + // tsup's default node-protocol plugin rewrites `node:sqlite` to a bare `sqlite` + // import (it does not recognize the experimental builtin), which resolves to + // nothing at runtime. Disable the rewrite so `node:` prefixes survive the build + // and node resolves the builtin itself. + removeNodeProtocol: false, });