Observability, a human in the loop and the Skillhook Cloud link (0.5.0)

.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
   "name": "skillhook",
   "displayName": "skillhook",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex. Two skills teach the agent to install and expose skillhook and to write good webhook skills; the bundled MCP server manages skills, secrets, jobs and exposure.",
   "author": {
     "name": "Meter",

.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "skillhook",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex. Two skills teach the agent to install and expose skillhook and to write good webhook skills; the bundled MCP server manages skills, secrets, jobs and exposure.",
   "author": {
     "name": "Meter",

.cursor-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "skillhook",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Turn this machine into a permanent, secure webhook endpoint that runs Agent Skills with Claude Code or Codex. Two skills teach the agent to install and expose skillhook and to write good webhook skills; the bundled MCP server manages skills, secrets, jobs and exposure.",
   "author": {
     "name": "Meter",

.github/workflows/ci.yml

@@ -62,6 +62,15 @@ jobs:
           node dist/cli.js link . --dir "$RUNNER_TEMP/skillhook" --json
           node dist/cli.js projects --dir "$RUNNER_TEMP/skillhook" --json
           node dist/cli.js run pull-after-merge --dir "$RUNNER_TEMP/skillhook" --payload '{}' --dry-run --json > /dev/null
+          node dist/cli.js run --file examples/skills/hello/SKILL.md --dir "$RUNNER_TEMP/skillhook" --payload '{"name":"ci"}' --dry-run --json > /dev/null
+          node dist/cli.js deliveries list --dir "$RUNNER_TEMP/skillhook" --json > /dev/null
+          node dist/cli.js jobs list --waiting --dir "$RUNNER_TEMP/skillhook" --json > /dev/null
+          node dist/cli.js stats --since 7d --dir "$RUNNER_TEMP/skillhook" --json > /dev/null
+          node dist/cli.js config path --dir "$RUNNER_TEMP/skillhook" --json > /dev/null
+          node dist/cli.js cloud status --dir "$RUNNER_TEMP/skillhook" --json > /dev/null
+          node dist/cli.js runners --local --dir "$RUNNER_TEMP/skillhook" --json > /dev/null || true   # no claude/codex on the runner
+          SKILLHOOK_NO_UPDATE_CHECK=1 node dist/cli.js health --quick --local --dir "$RUNNER_TEMP/skillhook" --json > "$RUNNER_TEMP/health.json" || true
+          node -e 'const r = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); if (!Array.isArray(r.checks) || !r.groups) process.exit(1);' "$RUNNER_TEMP/health.json"
 
       - name: Audit production dependencies
         run: npm audit --omit=dev
@@ -95,7 +104,7 @@ jobs:
           const fs = require("node:fs");
           const [info] = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
           const files = new Set(info.files.map((f) => f.path));
-          const required = ["package.json", "README.md", "CHANGELOG.md", "LICENSE", "dist/cli.js", "dist/index.js", "dist/index.d.ts", "dist/update.js", "schema/skillhook.schema.json", "schema/skillhook.yaml.schema.json", "examples/skills/hello/SKILL.md", "examples/skills/sentry-triage/SKILL.md"];
+          const required = ["package.json", "README.md", "CHANGELOG.md", "LICENSE", "dist/cli.js", "dist/index.js", "dist/index.d.ts", "dist/update.js", "dist/cloud/protocol.js", "dist/cloud/protocol.d.ts", "schema/skillhook.schema.json", "schema/skillhook.yaml.schema.json", "examples/skills/hello/SKILL.md", "examples/skills/sentry-triage/SKILL.md"];
           const missing = required.filter((f) => !files.has(f));
           const unwanted = [...files].filter((f) => /^(src|test|scripts|docs|skills|\.github)\//.test(f) || /\.test\.|\.env|\.tgz$|\.map$/.test(f));
           if (missing.length || unwanted.length) {
@@ -123,3 +132,7 @@ jobs:
           skillhook doctor --dir "$home" --json > "$RUNNER_TEMP/doctor.json" || true
           node -e 'const r = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); if (!Array.isArray(r.checks)) process.exit(1);' "$RUNNER_TEMP/doctor.json"
           skillhook update --dir "$home" --json || true
+          skillhook stats --dir "$home" --json > /dev/null
+          skillhook deliveries list --dir "$home" --json > /dev/null
+          skillhook run --file examples/skills/hello/SKILL.md --dir "$home" --payload '{"name":"ci"}' --dry-run --json > /dev/null
+          SKILLHOOK_NO_UPDATE_CHECK=1 skillhook health --quick --local --dir "$home" --json > /dev/null || true

AGENTS.md

@@ -27,10 +27,16 @@ is `skillhook`. User docs: `README.md`, `docs/`, `llms.txt`.
 | `src/prompt.ts` | Placeholders, event block, unattended-run guardrails. |
 | `src/schedule.ts`, `src/scheduler.ts` | Cron parsing and next/previous occurrence in an IANA zone (pure, no deps); the scheduler that fires `schedule:` hooks from `serve` (wall-clock tick, `catch_up` / `overlap`, exactly-once slots via the delivery index, state in `jobs/.schedules.json`). `src/commands/schedules.ts` is the CLI. |
 | `src/jobs.ts`, `src/queue.ts`, `src/run.ts` | Job directories on disk, the concurrency queue, invocation preparation. |
-| `src/runners/` | `claude.ts`, `codex.ts`, `shell.ts`: build argv, parse output; `env.ts` is the env allow-list. |
+| `src/events.ts` | The in-process event bus (`Events`, `EventMap`): the queue publishes `job.*`, the scheduler `schedule.*`, the registry `skill.changed`, `serve` `server.*`; `GET /events` and `GET /jobs/<id>/events` stream it (SSE, `openEventStream` in `src/server.ts`). The cloud link will subscribe to the same bus. |
+| `src/progress.ts`, `src/answer.ts`, `src/mcp-job.ts`, `src/commands/job.ts` | The job API for the running agent and the human loop. `progress.ts` is the file model in the job directory (`progress.jsonl`, `progress.json`, `question.json`, `answer.json`) that the queue watches; `mcp-job.ts` serves it as the per-run MCP server (`skillhook mcp --job`, injected by the runners) and `commands/job.ts` as `skillhook job progress\|ask\|outcome\|note\|context`; `answer.ts` (leaf, like `manual.ts`) delivers a person's answer live or as a `trigger: resume` job that reopens the session. |
+| `src/runners/` | `claude.ts`, `codex.ts`, `shell.ts`: build argv, parse output; `env.ts` is the env allow-list (`baseRunEnv` is also what probes run with); `failure.ts` classifies a failed run (`failure.kind`, from the CLIs' captured lines) and holds the `fallback` / `retry` schemas. |
+| `src/cloud/` | The Skillhook Cloud side of this machine (`control.ts`: the commands that act on it, with the config keys the cloud may never change; `seal.ts`: X25519 + AES-GCM sealing for secrets). `protocol.ts` is the wire protocol as pure zod (no `node:` imports; exported as `@meterapp/skillhook/protocol`, the cloud repo imports it), with the vocabulary repeated as literals and a drift test; `config.ts` holds the URL rules, the kill switch and `commandAllowed`; `link.ts` is the sync loop `serve` runs (idle until `cloud.enabled`), `outbox.ts` the spool and ledgers in `jobs/.cloud/`, `redact.ts` what is removed before anything leaves, `commands.ts` the command dispatcher, `ingress.ts` hosted deliveries replayed to the local server, `pair.ts` / `src/commands/cloud.ts` pairing. Tests talk to `src/test-support/fake-cloud.ts`, never to a real cloud. |
+| `src/stats.ts` | Pure aggregation over job records and delivery records (`computeStats`) and `collectStats` over the store and the log: `GET /stats`, `skillhook stats`, MCP `get_stats`. New numbers go here with a unit test on synthetic records. |
+| `src/readiness.ts` | Is a runner installed and logged in (`checkReadiness`, `ReadinessCache`): the queue's pre-flight before every job, `GET /runners`, `skillhook runners`, `runners.changed`. A not-ready runner fails the job fast or hands it to a `fallback:` runner; a failed run may be retried or handed over only before the agent produced anything. |
 | `src/ops.ts` | Shared operations (create skill, run locally, sign+send, resolve URLs). CLI and MCP both call this; do not duplicate logic in either. |
 | `src/mcp.ts` | MCP server (`@modelcontextprotocol/server` v2, stdio). Tools wrap `ops.ts`. |
-| `src/tailscale.ts`, `src/service.ts`, `src/doctor.ts` | Funnel/Serve, launchd/systemd, diagnostics. |
+| `src/tailscale.ts`, `src/service.ts` | Funnel/Serve, launchd/systemd. |
+| `src/health.ts`, `src/doctor.ts`, `src/tools.ts` | The grouped health report (`runHealth`, `HealthCache` behind `GET /health/checks`, `health.changed`); `doctor.ts` is its quick flavour printed flat; `tools.ts` probes `claude` / `codex` (version, login, `mcp list`, `plugin list`, `codex doctor`) with the job environment (`baseRunEnv`) and holds the pure parsers of their output. New checks: add them in `runHealth` with a group, a fixture answer in `test/fixtures/` when a CLI is involved, a row in `docs/operations.md`. |
 | `src/update.ts`, `src/commands/update.ts` | The daily update check (registry lookup, 24 h cache in `<home>/update-check.json`, install-method detection, background refresh) and `skillhook update`. |
 | `scripts/release.ts` | Version bump / consistency check / release notes across `package.json`, the lockfile, the plugin manifests and `CHANGELOG.md`. |
 | `.github/workflows/` | `ci.yml` (PRs and main: checks + packed-tarball install), `release.yml` (tags merged version bumps), `publish.yml` (npm publish with provenance, GitHub release, verification). |
@@ -41,21 +47,22 @@ is `skillhook`. User docs: `README.md`, `docs/`, `llms.txt`.
 | `test/fixtures/` | `fake-claude.mjs` / `fake-codex.mjs` emulate the real CLIs' output formats. |
 
 Runtime state lives outside the repo in `~/.skillhook` (`SKILLHOOK_HOME`):
-`skillhook.json`, `.env` (mode 600), `skills/`, `jobs/` (including `.deliveries.json` and `.schedules.json`), `logs/`, `server.json`.
+`skillhook.json`, `.env` (mode 600), `skills/`, `jobs/` (including `.deliveries.json`, `.schedules.json`, `.delivery-log/`, `.cloud/`), `logs/`, `server.json`.
 
 ## Hard rules
 
 - **Runtime dependencies stay at three**: `@modelcontextprotocol/server`, `yaml`, `zod`. Everything else is `node:` built-ins. Node >= 22, ESM, TypeScript strict, imports end in `.js`.
 - **Security is not optional.** The server binds `127.0.0.1` by default; TLS and public exposure are Tailscale's job. Every webhook goes through `verifyRequest`; every admin route through `requireAdmin`. Compare secrets only with `safeEqual`. A skill without `auth` gets a bearer token (`SKILLHOOK_SECRET_<NAME>`); `auth: none` must be explicit and is warned about. Secret values are never logged, never returned by an API/tool except once at generation, and never written into job files (`redactHeaders`). The agent's environment is an allow-list (`src/runners/env.ts`); `SKILLHOOK_ADMIN_TOKEN` and `SKILLHOOK_SECRET_*` are never forwarded implicitly.
 - **Payloads are data.** Anything that reaches the prompt from a webhook is wrapped in `<webhook_payload>` and the guardrails say so. Never build a prompt by concatenating payload text outside those blocks.
 - **Skills are Agent Skills.** Standard frontmatter (`name`, `description`, `license`, `compatibility`, `metadata`, `allowed-tools`) plus a `skillhook:` block. `name` must equal the directory name. New fields: add to the zod schema in `src/skills.ts`, to `docs/skills.md`, to `skills/skillhook-authoring/SKILL.md`, and cover them in `src/skills.test.ts` — in the same PR. `schedule` and `webhook` are block fields like any other (normalized by `resolveSchedule`, documented in `docs/schedules.md`). A hook in `skillhook.yaml` is the same block plus exactly one of `run` / `skill` / `prompt` (`HookSchema` in `src/projects.ts` extends `SkillhookBlockSchema`, so new block fields reach hooks automatically); hook-only fields go in `src/projects.ts`, `docs/projects.md`, `npm run schema` and `src/projects.test.ts`. A compiled hook is an ordinary `Skill` (with `source.type === "project"`); never special-case hooks in the server, queue or runners.
-- **Config changes** go in `src/config.ts` (zod, `.prefault({})` for nested objects so defaults apply), then `npm run schema`, then `docs/operations.md`. `projects` is the one key the server re-reads without a restart (`configProjects` in `src/registry.ts`); keep it that way.
+- **Config changes** go in `src/config.ts` (zod, `.prefault({})` for nested objects so defaults apply), then `npm run schema`, then `docs/operations.md`. The running server owns one live `Config` object (`ConfigRef`): a reload (`PATCH /config`, `POST /config/reload`, `skillhook config set`, a file edit noticed within 5 s) patches that object in place, so read config values at use time, never copy them at construction (the rate limiter takes a getter; the logger has `setLevel`, the job store `configure`). Only `host` and `port` need a restart (`RESTART_CONFIG_KEYS`); a new key is hot unless it is added there, and `config.changed` says what a reload did.
 - **Runners never shell-interpolate.** Argv arrays only; the prompt travels on stdin; parse the CLI's structured output (`stream-json`, JSONL). When Claude Code or Codex change flags, update the runner, `test/fixtures/`, `docs/runners.md` and the version note in `README.md` together.
-- **Jobs are directories.** `job.json` is the record; artifacts sit next to it; nothing outside `~/.skillhook/jobs` is written by the server. Statuses: `queued running succeeded failed timed_out cancelled interrupted`.
+- **Jobs are directories.** `job.json` is the record; artifacts sit next to it; nothing outside `~/.skillhook/jobs` is written by the server, with the cloud link's control commands as the documented exceptions (`skill.put` writes `skills/<name>/SKILL.md`, `secret.generate` / `secret.set` and a token rotation write `.env`, `config.patch` writes `skillhook.json`); its own state is `jobs/.cloud/`, skills it removes go to `jobs/.removed-skills/`. Statuses: `queued running succeeded failed timed_out cancelled interrupted`. The running agent talks to skillhook only through files in its job directory (`src/progress.ts`): no token, no HTTP, so the shell runner and a restart are covered; the queue turns them into events and record fields.
+- **State changes are events.** Whatever the server learns (a job changing state, a schedule firing or skipping, a skill file appearing or changing) is emitted on `Events` (`src/events.ts`) at the place it happens, after the record on disk is updated, with the full record in the payload. Consumers (the SSE routes, later the cloud link) subscribe; they never poll job files. A new kind of state change gets a new `EventMap` entry, an emit, a row in `docs/api.md` and a test. Listener errors are logged, never thrown into the publisher.
 - **Every CLI command supports `--json`** and returns non-zero on failure. Register new commands in `COMMANDS` and `HELP` in `src/commands/main.ts`, then in the README table.
 - **Third-party facts** (Granola, Sentry, GitHub, Tailscale) are stated in `docs/` and the examples with the exact header names; change them only with a source.
 - **Tests are hermetic**: `tempHome()` from `src/test-support/helpers.ts`, fake runners, ephemeral ports. Never touch `~/.skillhook`, the real `claude`/`codex`, `launchctl` or `tailscale` from a test. Never reach the real npm registry either: point `SKILLHOOK_NPM_REGISTRY` at a local `node:http` server or set `SKILLHOOK_NO_UPDATE_CHECK=1`.
-- **The CLI phones home exactly once a day, and only for the update check** (`src/update.ts`: the registry's `latest` dist-tag, cached 24 h, never on `--json`, in CI, or when `SKILLHOOK_NO_UPDATE_CHECK` / `update_check: false` say so). Do not add other outbound requests the user did not ask for, and never auto-install anything.
+- **Outbound requests are opt-in and enumerated.** By default the CLI phones home once a day, and only for the update check (`src/update.ts`: the registry's `latest` dist-tag, cached 24 h, never on `--json`, in CI, or when `SKILLHOOK_NO_UPDATE_CHECK` / `update_check: false` say so). The one other outbound connection is the Skillhook Cloud link (`src/cloud/link.ts`), and only after `skillhook cloud connect` wrote `cloud.enabled` and `SKILLHOOK_CLOUD_TOKEN`: it talks to `cloud.url` over HTTPS only, sends only what `docs/cloud.md` lists (redacted, scrubbed of every `.env` value), obeys `cloud.mode` and the local allow/deny lists (which the cloud cannot change), and stops on `cloud.enabled: false`, `SKILLHOOK_NO_CLOUD=1` or `cloud disconnect`. Never enable it by default, from `init` or from a job; do not add other outbound requests the user did not ask for, and never auto-install anything.
 
 ## Checks