From f1480233cccbd430295f41c8a75cf46967a07c1a Mon Sep 17 00:00:00 2001 From: SarthakWade Date: Thu, 27 Aug 2026 14:20:03 +0530 Subject: [PATCH] docs(benchmark): refresh task-aware measurements --- README.md | 5 +- apps/headless/docs/BENCHMARK.md | 20 +-- apps/web/lib/repository-content.d.mts | 2 + apps/web/lib/repository-content.mjs | 53 ++++++++ docs/ROADMAP.md | 56 ++++---- docs/roadmap/improvements-backlog.md | 6 +- docs/roadmap/what-is-excellent.md | 29 ++-- packages/benchmark-results/results.json | 168 ++++++++++++------------ 8 files changed, 200 insertions(+), 139 deletions(-) diff --git a/README.md b/README.md index 06e5876..9ae3b62 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,8 @@ judgments (not lab benchmarks): Runs on macOS and Linux (Ubuntu and common CI/test distros); Windows is the exception. -**Measured agent surface** (same P2 fixture flow, Docker ARM64, 17 Jul 2026 — -point-in-time): Headless warm **147** est. tokens vs Selenium **410** / +**Measured agent surface** (same P2 fixture flow, Docker ARM64, 27 Aug 2026 — +point-in-time): Headless warm **174** est. tokens vs Selenium **410** / Puppeteer **499**. Full method and limits: [BENCHMARK.md](apps/headless/docs/BENCHMARK.md). @@ -236,7 +236,6 @@ published GHCR image under Docker Desktop: docker run --shm-size=1g ghcr.io/lockintime/headless:latest headless --version ``` - ### npm / npx JavaScript-based agent harnesses can run the verified launcher without a diff --git a/apps/headless/docs/BENCHMARK.md b/apps/headless/docs/BENCHMARK.md index 630e219..c6e5b91 100644 --- a/apps/headless/docs/BENCHMARK.md +++ b/apps/headless/docs/BENCHMARK.md @@ -5,26 +5,26 @@ record through the `Continue` transition, and save a final screenshot. ## Current snapshot -Five fresh containers were run for each case on 12 August 2026, on Apple +Five fresh containers were run for each case on 27 August 2026, on Apple Silicon with Docker Linux ARM64. The table reports the median of each metric. These remain point-in-time measurements; repeat the benchmark before using them to compare a change. The generated [`results.json`](../../../packages/benchmark-results/results.json) preserves all 20 raw samples and the aggregation provenance. -| Workflow | Estimated tokens | Wall time | CPU time | Peak memory | -| --- | ---: | ---: | ---: | ---: | -| Headless, cold | 218 | 3,484 ms | 2,028 ms | 368 MiB | -| Headless, warm | 174 | 3,248 ms | 1,379 ms | 366 MiB | -| Selenium with Python | 410 | 2,880 ms | 2,010 ms | 363 MiB | -| Puppeteer | 499 | 2,402 ms | 1,860 ms | 358 MiB | +| Workflow | Estimated tokens | Wall time | CPU time | Peak memory | +| -------------------- | ---------------: | --------: | -------: | ----------: | +| Headless, cold | 218 | 3,413 ms | 1,794 ms | 381 MiB | +| Headless, warm | 174 | 3,239 ms | 1,218 ms | 379 MiB | +| Selenium with Python | 410 | 2,828 ms | 1,788 ms | 378 MiB | +| Puppeteer | 499 | 2,400 ms | 1,849 ms | 367 MiB | Estimated tokens are `ceil(workflow source bytes / 4)`. They compare the agent workflow surface, not billed LLM tokens, tool schemas, prompts, or responses. Headless has the smallest measured agent surface: the warm workflow uses about 58% fewer estimated tokens than Selenium and 65% fewer than Puppeteer. Its -median CPU time is about 31% lower than Selenium and 26% lower than Puppeteer. +median CPU time is about 32% lower than Selenium and 34% lower than Puppeteer. Puppeteer is fastest and has the lowest median peak memory; Headless does not lead those dimensions. The reusable P2 flow command reduces orchestration work for real agent-driven repeats, but this benchmark retains the comparable @@ -38,9 +38,9 @@ lookup. ## Method -Each workflow uses Chromium 151 and FFmpeg 5.1 to produce the same two +Each workflow uses Chromium 151.0.7922.173 and FFmpeg 5.1.9 to produce the same two artifacts: an MP4 that tours both pages and a final viewport PNG. Selenium 4.8.3 -uses ChromeDriver 151; Puppeteer Core is 22.15.0. All waits use page load or an +uses ChromeDriver 151.0.7922.173; Puppeteer Core is 22.15.0. All waits use page load or an explicit URL condition. Every measured run gets a fresh container. The warm Headless case starts its host and session before timing; the cold case includes them. diff --git a/apps/web/lib/repository-content.d.mts b/apps/web/lib/repository-content.d.mts index 3eeb3d8..627db45 100644 --- a/apps/web/lib/repository-content.d.mts +++ b/apps/web/lib/repository-content.d.mts @@ -1,5 +1,6 @@ export type BenchmarkWorkflow = { case: string; + label: string; workflow: string; name: string; variant: string; @@ -19,6 +20,7 @@ export type BenchmarkWorkflow = { export type BenchmarkContent = { sectionLabel: string; + methodDate: string; headline: string; summary: string; proofs: Array<{ diff --git a/apps/web/lib/repository-content.mjs b/apps/web/lib/repository-content.mjs index c458bc2..2c4be5d 100644 --- a/apps/web/lib/repository-content.mjs +++ b/apps/web/lib/repository-content.mjs @@ -165,12 +165,19 @@ export function loadBenchmarkContent() { year: "numeric", timeZone: "UTC", }).format(generatedAt); + const methodDate = new Intl.DateTimeFormat("en-GB", { + day: "numeric", + month: "long", + year: "numeric", + timeZone: "UTC", + }).format(generatedAt); const workflows = BENCHMARK_CASES.map((caseName) => { const entry = cases.get(caseName); const presentation = PRESENTATION[caseName]; return { case: caseName, + label: entry.label, workflow: entry.label.replace(",", ""), ...presentation, tokens: entry.median.estimatedTokens, @@ -189,6 +196,7 @@ export function loadBenchmarkContent() { benchmarkCache = { sectionLabel: `P2 benchmark / ${date}`, + methodDate, headline: warm.estimatedTokens === Math.min(...workflows.map((workflow) => workflow.tokens)) @@ -431,6 +439,51 @@ export function loadDocumentationContent() { export function validateRepositoryContent() { const benchmark = loadBenchmarkContent(); const documentation = loadDocumentationContent(); + const readme = readRepositoryFile("README.md"); + const method = readRepositoryFile("apps/headless/docs/BENCHMARK.md"); + const measuredClaim = readme.match( + /\*\*Measured agent surface\*\*[\s\S]*?Docker ARM64, ([0-9]{1,2} [A-Z][a-z]{2} [0-9]{4})[\s\S]*?Headless warm \*\*(\d+)\*\* est\. tokens vs Selenium \*\*(\d+)\*\* \/\s*Puppeteer \*\*(\d+)\*\*/, + ); + if (!measuredClaim) fail("README measured agent-surface claim is missing"); + const workflows = new Map( + benchmark.workflows.map((workflow) => [workflow.case, workflow]), + ); + const expectedClaim = [ + benchmark.sectionLabel.replace("P2 benchmark / ", ""), + String(workflows.get("headless-warm").tokens), + String(workflows.get("selenium").tokens), + String(workflows.get("puppeteer").tokens), + ]; + if ( + JSON.stringify(measuredClaim.slice(1)) !== JSON.stringify(expectedClaim) + ) { + fail( + "README measured agent-surface claim does not match generated results", + ); + } + if (!method.includes(`for each case on ${benchmark.methodDate},`)) { + fail("benchmark method date does not match generated results"); + } + for (const workflow of benchmark.workflows) { + const row = method + .split("\n") + .find((line) => line.startsWith(`| ${workflow.label}`)); + if (!row) fail(`benchmark method is missing row: ${workflow.label}`); + const cells = row + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()); + const expectedCells = [ + workflow.label, + workflow.tokens.toLocaleString("en-US"), + `${workflow.wallMs.toLocaleString("en-US")} ms`, + `${workflow.cpuMs.toLocaleString("en-US")} ms`, + `${workflow.memoryMiB.toLocaleString("en-US")} MiB`, + ]; + if (JSON.stringify(cells) !== JSON.stringify(expectedCells)) { + fail(`benchmark method row is stale: ${workflow.label}`); + } + } return { benchmarkCases: benchmark.workflows.length, commandGroups: documentation.commandGroups.length, diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 45b79dc..2dc8f6a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -81,12 +81,13 @@ of them: Honest snapshot, so newcomers know what is real: **Working and verified** + - 39-verb JSON protocol over a `0600` Unix socket with peer-UID checks (`apps/headless/Sources/HeadlessProtocol/`). - Two engines behind one CLI: macOS `WKWebView` app, Linux sandboxed Chromium over the DevTools fd-3/4 pipe (no debug port). - Progressive context pruning (`inspect --context - summary|outline|text|actions|full`, `--task`, `--within @rN`, `--budget`) +summary|outline|text|actions|full`, `--task`, `--within @rN`, `--budget`) with a measured **94.5 % token reduction** on the 120-section fixture. - Evidence capture: PNG/JPG/PDF screenshots, viewport/section series, MP4/MOV/ WebM/GIF recordings, visual diffs, flows, QA reports. @@ -107,6 +108,7 @@ Honest snapshot, so newcomers know what is real: command reference, and a Docker sandbox wrapper. **Not yet real** + - ~~No CI on pull requests or `main`~~ — PR CI landed (`.github/workflows/ci.yml`, backlog §D1/§D3). Correctness fixes in §A are still outstanding, and the macOS E2E is nightly/label-gated rather than a per-PR gate. @@ -121,7 +123,7 @@ Honest snapshot, so newcomers know what is real: three places each and will drift; the site has no deploy pipeline. - Windows is not supported. - A list of real code defects (thread-safety on shutdown, oversized `qa - report` responses, `@eN` ref invalidation surprises, host code duplication) +report` responses, `@eN` ref invalidation surprises, host code duplication) — all catalogued in the [improvements backlog](roadmap/improvements-backlog.md). --- @@ -152,23 +154,23 @@ codebase's existing behavior because they are the product: ## 4. Platform support -| Platform | Status today | Target | -| --- | --- | --- | -| **macOS 13+** (Apple Silicon) | Universal signed release pipeline awaiting tag | Signed + notarized, Homebrew, universal binary | -| **macOS Intel** | Universal release pipeline awaiting tag | Universal binary in release CI | -| **Linux** (Debian/Ubuntu, non-Snap Chromium) | Verified installer and GHCR pipeline awaiting tag | curl installer, published Docker image (GHCR), apt guidance | -| **Linux other distros** | Works where a non-Snap Chromium exists | Documented candidate paths per distro family | -| **Windows 10/11** | Not supported | **Stretch goal (Phase W)** — Chromium host ported; not required for "done". See [architecture decisions §6](roadmap/architecture-decisions.md). | -| **Any OS via Docker** | Works (build locally) | `docker run ghcr.io/…/headless` one-liner, including as the practical Windows answer (WSL2/Docker Desktop) until Phase W lands | +| Platform | Status today | Target | +| -------------------------------------------- | ------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **macOS 13+** (Apple Silicon) | Universal signed release pipeline awaiting tag | Signed + notarized, Homebrew, universal binary | +| **macOS Intel** | Universal release pipeline awaiting tag | Universal binary in release CI | +| **Linux** (Debian/Ubuntu, non-Snap Chromium) | Verified installer and GHCR pipeline awaiting tag | curl installer, published Docker image (GHCR), apt guidance | +| **Linux other distros** | Works where a non-Snap Chromium exists | Documented candidate paths per distro family | +| **Windows 10/11** | Not supported | **Stretch goal (Phase W)** — Chromium host ported; not required for "done". See [architecture decisions §6](roadmap/architecture-decisions.md). | +| **Any OS via Docker** | Works (build locally) | `docker run ghcr.io/…/headless` one-liner, including as the practical Windows answer (WSL2/Docker Desktop) until Phase W lands | Agent-harness support (the other axis of "platform"): -| Harness | Today | Target | -| --- | --- | --- | -| MCP clients (Claude Code, Cursor, Codex, …) | `headless-mcp` stdio server | unchanged core; per-client setup docs + `.mcp.json` example | -| Claude Code | manual skill pointer | root `CLAUDE.md` + discoverable skill | -| Codex / OpenCode / Amp / others | `AGENTS.md` convention | root `AGENTS.md` (done in this change) | -| Plain shell agents | CLI + `headless capabilities` | unchanged; capabilities doc kept machine-checked | +| Harness | Today | Target | +| ------------------------------------------- | ----------------------------- | ----------------------------------------------------------- | +| MCP clients (Claude Code, Cursor, Codex, …) | `headless-mcp` stdio server | unchanged core; per-client setup docs + `.mcp.json` example | +| Claude Code | manual skill pointer | root `CLAUDE.md` + discoverable skill | +| Codex / OpenCode / Amp / others | `AGENTS.md` convention | root `AGENTS.md` (done in this change) | +| Plain shell agents | CLI + `headless capabilities` | unchanged; capabilities doc kept machine-checked | --- @@ -184,7 +186,7 @@ every item carries file/line detail. Write down what the product is, what must not change, and everything that is wrong. Add agent rule files so every harness can work on this repo. -*Exit test:* this document set is merged; `AGENTS.md`/`CLAUDE.md` exist at +_Exit test:_ this document set is merged; `AGENTS.md`/`CLAUDE.md` exist at root. ### Phase 1 — Trust the build (CI + correctness) @@ -202,7 +204,7 @@ known races exist. `ELEMENT_NOT_FOUND` (backlog §A7). - Web app gets `next build` + eslint in the same CI (backlog §D3). -*Exit test:* a PR cannot merge with failing tests; the E2E suites pass on CI +_Exit test:_ a PR cannot merge with failing tests; the E2E suites pass on CI runners, not just laptops; the known-crash list in the backlog §A is empty. ### Phase 2 — One host, written once (deduplication refactor) @@ -216,7 +218,7 @@ errors end-to-end (backlog §B). This is also the **prerequisite for Windows**: after it, a Windows port is one new engine + one new transport backend, not a third copy of everything. -*Exit test:* adding a hypothetical new verb touches one dispatch site; the +_Exit test:_ adding a hypothetical new verb touches one dispatch site; the error-code mapping is a typed enum, not string matching; the capability matrix (clipboard, network mock, PDF fidelity, …) is generated from code and asserted in tests. @@ -237,7 +239,7 @@ in tests. - **npm wrapper** (`npx headless-browser` style) that downloads the platform binary — the cheapest path into JS-centric agent stacks. (Backlog §E.) -*Exit test:* a new user on a clean macOS or Linux machine gets from zero to +_Exit test:_ a new user on a clean macOS or Linux machine gets from zero to `headless start` + first `visit` in under two minutes without touching a compiler, and without a Gatekeeper override on macOS. @@ -258,7 +260,7 @@ Make Headless the obvious choice inside every harness: implemented so `fill` can type literal `--json`; response pagination for large reports remains (backlog §G). -*Exit test:* a fresh Claude Code, Cursor, and Codex session can each discover +_Exit test:_ a fresh Claude Code, Cursor, and Codex session can each discover and drive Headless with zero manual prompting beyond repo checkout. ### Phase 5 — Website and docs as a product surface @@ -269,11 +271,11 @@ and drive Headless with zero manual prompting beyond repo checkout. generated from the CLI) (backlog §F). - Add the missing pages: install, security model, MCP setup, command reference, changelog, platform matrix. -- Re-run the benchmark with the task-aware inspect flow before quoting any - token number — the site currently markets `--task` while quoting pre-`--task` - measurements (backlog §F4). +- Keep the generated benchmark current with the task-aware inspect flow before + quoting any token number. The site imports the generated medians and carries + their point-in-time caveat (backlog §F4). -*Exit test:* site deploys on merge; every number and command on it is +_Exit test:_ site deploys on merge; every number and command on it is generated or test-asserted; the "stale benchmark" warning is gone because the benchmark is current. @@ -295,7 +297,7 @@ Phase 2's engine/transport split. Shape of the work (detailed in - Until then, the documented Windows answer is Docker Desktop/WSL2 with the published image (Phase 3 dependency). -*Exit test:* the Linux E2E scenario passes on a Windows runner with the +_Exit test:_ the Linux E2E scenario passes on a Windows runner with the Chromium engine; `winget install headless` works. --- @@ -315,7 +317,7 @@ of the following hold: returns an explicit capability error; the matrix is generated and asserted. 4. **Agent-native:** any MCP-capable harness and any AGENTS.md-reading harness can drive Headless from a fresh checkout with no human glue; `headless - capabilities` is machine-accurate. +capabilities` is machine-accurate. 5. **Evidence current:** benchmark re-run on the shipping workflow; QA evidence regenerated for the release; website deployed and drift-free. 6. **Docs:** this roadmap's Phases 1–5 checked off, with Windows either diff --git a/docs/roadmap/improvements-backlog.md b/docs/roadmap/improvements-backlog.md index f9009e4..a4a5e6b 100644 --- a/docs/roadmap/improvements-backlog.md +++ b/docs/roadmap/improvements-backlog.md @@ -431,9 +431,11 @@ Owner-decided scope: package managers, no hosted service. matrix, the README's comparison table (strongest positioning content, not on site). Plus `robots.txt`, `sitemap`, OG metadata, per-page `metadata`, 404 page. -- **F4. Stale-benchmark honesty:** ([#50](https://github.com/LockInTime/headless/issues/50)) site quotes pre-`--task` numbers while +- **F4. Stale-benchmark honesty:** ([#50](https://github.com/LockInTime/headless/issues/50)) ~~site quotes pre-`--task` numbers while marketing `--task`, and drops BENCHMARK.md's re-run warning; headline says - "Measured, not claimed." Fix by refresh (D5) or by carrying the caveat. + "Measured, not claimed." Fix by refresh (D5) or by carrying the caveat.~~ + **Done:** a new five-repeat, task-aware snapshot was generated on 27 August 2026. The website imports its medians and date, preserves the point-in-time + warning, and web lint rejects README claims that drift from generated data. - **F5. Dead weight:** ([#51](https://github.com/LockInTime/headless/issues/51)) `components/ui/side-rays.tsx` + `ogl` dep (unused), `public/scan-dashboard.png` (unreferenced), leftover shadcn `.dark` block in `globals.css`, 8 unused button variants; two WebGL stacks (~700 KB) for diff --git a/docs/roadmap/what-is-excellent.md b/docs/roadmap/what-is-excellent.md index 47fe71f..6af4ce7 100644 --- a/docs/roadmap/what-is-excellent.md +++ b/docs/roadmap/what-is-excellent.md @@ -1,13 +1,13 @@ # What is excellent — do not change Companion to [ROADMAP.md](../ROADMAP.md). These are the parts of Headless where -the *idea* is right and the implementation expresses it well. They are the +the _idea_ is right and the implementation expresses it well. They are the product's identity. None of them may be weakened, "simplified away", or regressed by a refactor without a written decision recorded in [architecture-decisions.md](architecture-decisions.md). Each item states the idea, where it lives, and what specifically must be -preserved. (Known implementation bugs *inside* these areas are still fixable — +preserved. (Known implementation bugs _inside_ these areas are still fixable — they're listed in the [backlog](improvements-backlog.md) — but the fix must preserve the contract described here.) @@ -54,15 +54,16 @@ feature returns unbounded output. This is the product's measurable moat. ## 3. Fail-closed safety at the host, not in the prompt **Idea:** the safety rules are enforced by the host process, so a -prompt-injected or confused agent *cannot* violate them. +prompt-injected or confused agent _cannot_ violate them. **Where and what (each of these is a hard contract):** + - **No arbitrary JavaScript verb.** The protocol exposes only fixed runtime functions; `capabilities` advertises `arbitraryJavaScript: false`. - **HTTP/HTTPS only.** `normalizedWebURL` / `agentMayNavigate` (`Protocol.swift:530-609`): no `file:`, `javascript:`, `data:`, external app schemes, or credential-bearing URLs; bare hosts default to https except - local dev addresses. Enforced at *three* layers: visit, host navigation + local dev addresses. Enforced at _three_ layers: visit, host navigation policy (macOS `decidePolicyFor`, Linux frame-event enforcement), and the in-page click guard. - **Downloads denied.** `Browser.setDownloadBehavior deny` on Linux @@ -78,14 +79,14 @@ prompt-injected or confused agent *cannot* violate them. extension allow-list); `O_EXCL` create at `0600` in a `0700` owner-checked root; never overwrite; the only read primitive is bounded and root-confined (`Artifacts.swift`). -- **Sensitive values gated twice.** Cookie/storage *values* need both +- **Sensitive values gated twice.** Cookie/storage _values_ need both `--values` and a host started with `HEADLESS_ALLOW_SENSITIVE_DIAGNOSTICS=1`; auth/cookie/token/secret headers and URL credentials are always redacted (`Diagnostics.swift:204-235`). - **Typed values never persisted.** Flow recording excludes `fill` — replay files can never contain credentials (`Flows.swift:25-27`). - **Linux never weakens the sandbox.** No `--no-sandbox`, refuses root - (`BrowserProcess.swift:161-163`); Snap Chromium rejected *before launch* by + (`BrowserProcess.swift:161-163`); Snap Chromium rejected _before launch_ by path and shebang sniffing rather than failing mysteriously later (`ChromiumRuntime.swift:167-184`). - **Agent takeover starts clean.** A window showing a local file when the @@ -112,9 +113,10 @@ prompt-injection defense and reviewers rely on it. **Idea:** claims ship with reproducible proof. **Where:** + - `docs/qa/evidence/` — 11 recorded scenario videos + JSON + `SHA256SUMS`, regenerated by `apps/headless/Tests/qa-videos.sh` in disposable Docker. -- E2E suites assert *artifact truth*, not just exit codes: `file(1)` magic, +- E2E suites assert _artifact truth_, not just exit codes: `file(1)` magic, `ffprobe` codec/duration, framemd5 unique-frame counts (video isn't a still), PNG IHDR height parsing (full-page really is taller), `0600` modes, no-TCP assertions via `lsof`/`/proc/net/tcp`. @@ -128,16 +130,17 @@ scenario, not just a unit test. **Idea:** the benchmark (`apps/headless/docs/BENCHMARK.md`, `benchmark.sh`) states exactly what it measures (workflow source bytes / 4 as -an *agent-surface* proxy — explicitly "not billed LLM tokens"), publishes the -dimensions it *loses* (Puppeteer is faster on wall time), refuses to claim +an _agent-surface_ proxy — explicitly "not billed LLM tokens"), publishes the +dimensions it _loses_ (Puppeteer is faster on wall time), refuses to claim unmeasured wins (flows), marks results as point-in-time, and even carries a self-deprecating staleness warning ("re-run before quoting"). The qualitative README comparison table has a spec that forbids presenting it as a lab result and forbids claiming an overall win (`docs/superpowers/specs/2026-07-18-…`). **Preserve:** this candor is rare and is itself a feature. Numbers may be -refreshed; the caveat culture may not be dropped. (The one violation — the -website quoting stale numbers without the staleness warning — is backlog §F4.) +refreshed; the caveat culture may not be dropped. The website now derives its +claims from the generated result artifact and carries the same point-in-time +warning. ## 7. One CLI contract across engines, with explicit capability errors @@ -146,7 +149,7 @@ when an engine can't do something, it says `UNSUPPORTED_CAPABILITY` with a suggestion (macOS network mocking, Linux clipboard) instead of pretending (`main.swift:1231-1232`, `LinuxHost/main.swift:141-143`, P2.md:59-61). -**Preserve:** never ship a silent partial emulation. (Existing *accidental* +**Preserve:** never ship a silent partial emulation. (Existing _accidental_ divergences — PDF raster vs vector, screenshot coordinate spaces — are bugs to fix or to promote into declared capabilities: backlog §B6.) @@ -179,7 +182,7 @@ should improve (Phase 4), content should stay. ## 10. MCP as a thin argv adapter **Idea:** `headless-mcp` (`apps/headless/MCP/main.swift`, 81 lines) exposes -**one tool** whose input is the CLI argv. The MCP surface *is* the CLI +**one tool** whose input is the CLI argv. The MCP surface _is_ the CLI surface — zero schema drift, every new command instantly available to every MCP client, stdio-only with no TCP listener, remote via plain SSH. diff --git a/packages/benchmark-results/results.json b/packages/benchmark-results/results.json index 62d2896..3754877 100644 --- a/packages/benchmark-results/results.json +++ b/packages/benchmark-results/results.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "generatedAt": "2026-08-12T13:50:38Z", + "generatedAt": "2026-08-27T08:42:55Z", "provenance": { "generator": "apps/headless/benchmark.sh", "method": "apps/headless/docs/BENCHMARK.md", @@ -16,55 +16,55 @@ "samples": [ { "iteration": 1, - "wallMs": 3930, - "cpuMs": 2187, - "memoryPeakBytes": 752189440, - "artifactBytes": 102472, + "wallMs": 3551, + "cpuMs": 1873, + "memoryPeakBytes": 665522176, + "artifactBytes": 138624, "workflowBytes": 870, "estimatedTokens": 218 }, { "iteration": 2, - "wallMs": 3484, - "cpuMs": 2028, - "memoryPeakBytes": 380952576, - "artifactBytes": 99135, + "wallMs": 3413, + "cpuMs": 1812, + "memoryPeakBytes": 395333632, + "artifactBytes": 136998, "workflowBytes": 870, "estimatedTokens": 218 }, { "iteration": 3, - "wallMs": 3473, - "cpuMs": 2036, - "memoryPeakBytes": 386068480, - "artifactBytes": 137356, + "wallMs": 3428, + "cpuMs": 1794, + "memoryPeakBytes": 397774848, + "artifactBytes": 136998, "workflowBytes": 870, "estimatedTokens": 218 }, { "iteration": 4, - "wallMs": 3492, - "cpuMs": 1908, - "memoryPeakBytes": 388087808, - "artifactBytes": 132276, + "wallMs": 3378, + "cpuMs": 1575, + "memoryPeakBytes": 399491072, + "artifactBytes": 138274, "workflowBytes": 870, "estimatedTokens": 218 }, { "iteration": 5, - "wallMs": 3479, - "cpuMs": 1972, - "memoryPeakBytes": 381677568, - "artifactBytes": 101808, + "wallMs": 3411, + "cpuMs": 1783, + "memoryPeakBytes": 400982016, + "artifactBytes": 141710, "workflowBytes": 870, "estimatedTokens": 218 } ], "median": { - "wallMs": 3484, - "cpuMs": 2028, - "memoryPeakBytes": 386068480, - "artifactBytes": 102472, + "wallMs": 3413, + "cpuMs": 1794, + "memoryPeakBytes": 399491072, + "artifactBytes": 138274, "workflowBytes": 870, "estimatedTokens": 218 } @@ -75,54 +75,54 @@ "samples": [ { "iteration": 1, - "wallMs": 3248, - "cpuMs": 1379, - "memoryPeakBytes": 390807552, - "artifactBytes": 141710, + "wallMs": 3239, + "cpuMs": 1166, + "memoryPeakBytes": 397705216, + "artifactBytes": 136988, "workflowBytes": 693, "estimatedTokens": 174 }, { "iteration": 2, - "wallMs": 3237, - "cpuMs": 1349, - "memoryPeakBytes": 383737856, - "artifactBytes": 142986, + "wallMs": 3248, + "cpuMs": 1219, + "memoryPeakBytes": 401375232, + "artifactBytes": 136998, "workflowBytes": 693, "estimatedTokens": 174 }, { "iteration": 3, - "wallMs": 3233, - "cpuMs": 1414, - "memoryPeakBytes": 383971328, - "artifactBytes": 101808, + "wallMs": 3237, + "cpuMs": 1273, + "memoryPeakBytes": 397455360, + "artifactBytes": 138253, "workflowBytes": 693, "estimatedTokens": 174 }, { "iteration": 4, - "wallMs": 3256, - "cpuMs": 1376, - "memoryPeakBytes": 383102976, - "artifactBytes": 103840, + "wallMs": 3234, + "cpuMs": 1192, + "memoryPeakBytes": 396791808, + "artifactBytes": 136998, "workflowBytes": 693, "estimatedTokens": 174 }, { "iteration": 5, - "wallMs": 3277, - "cpuMs": 1455, - "memoryPeakBytes": 389758976, - "artifactBytes": 136998, + "wallMs": 3242, + "cpuMs": 1218, + "memoryPeakBytes": 401682432, + "artifactBytes": 138253, "workflowBytes": 693, "estimatedTokens": 174 } ], "median": { - "wallMs": 3248, - "cpuMs": 1379, - "memoryPeakBytes": 383971328, + "wallMs": 3239, + "cpuMs": 1218, + "memoryPeakBytes": 397705216, "artifactBytes": 136998, "workflowBytes": 693, "estimatedTokens": 174 @@ -134,54 +134,54 @@ "samples": [ { "iteration": 1, - "wallMs": 2865, - "cpuMs": 1959, - "memoryPeakBytes": 380690432, + "wallMs": 2800, + "cpuMs": 1784, + "memoryPeakBytes": 396455936, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 }, { "iteration": 2, - "wallMs": 2813, - "cpuMs": 1893, - "memoryPeakBytes": 383242240, + "wallMs": 2836, + "cpuMs": 1788, + "memoryPeakBytes": 395948032, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 }, { "iteration": 3, - "wallMs": 2924, - "cpuMs": 2117, - "memoryPeakBytes": 380301312, + "wallMs": 2799, + "cpuMs": 1780, + "memoryPeakBytes": 396189696, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 }, { "iteration": 4, - "wallMs": 2897, - "cpuMs": 2078, - "memoryPeakBytes": 385925120, + "wallMs": 2834, + "cpuMs": 1808, + "memoryPeakBytes": 394080256, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 }, { "iteration": 5, - "wallMs": 2880, - "cpuMs": 2010, - "memoryPeakBytes": 378208256, + "wallMs": 2828, + "cpuMs": 1822, + "memoryPeakBytes": 395735040, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 } ], "median": { - "wallMs": 2880, - "cpuMs": 2010, - "memoryPeakBytes": 380690432, + "wallMs": 2828, + "cpuMs": 1788, + "memoryPeakBytes": 395948032, "artifactBytes": 173975, "workflowBytes": 1638, "estimatedTokens": 410 @@ -193,54 +193,54 @@ "samples": [ { "iteration": 1, - "wallMs": 2486, - "cpuMs": 1942, - "memoryPeakBytes": 377262080, + "wallMs": 2499, + "cpuMs": 1917, + "memoryPeakBytes": 403075072, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499 }, { "iteration": 2, - "wallMs": 2437, - "cpuMs": 1942, - "memoryPeakBytes": 375758848, + "wallMs": 2400, + "cpuMs": 1861, + "memoryPeakBytes": 385277952, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499 }, { "iteration": 3, - "wallMs": 2362, - "cpuMs": 1827, - "memoryPeakBytes": 375615488, + "wallMs": 2415, + "cpuMs": 1849, + "memoryPeakBytes": 382992384, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499 }, { "iteration": 4, - "wallMs": 2402, - "cpuMs": 1860, - "memoryPeakBytes": 376266752, + "wallMs": 2369, + "cpuMs": 1810, + "memoryPeakBytes": 379813888, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499 }, { "iteration": 5, - "wallMs": 2398, - "cpuMs": 1849, - "memoryPeakBytes": 374726656, + "wallMs": 2391, + "cpuMs": 1842, + "memoryPeakBytes": 389476352, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499 } ], "median": { - "wallMs": 2402, - "cpuMs": 1860, - "memoryPeakBytes": 375758848, + "wallMs": 2400, + "cpuMs": 1849, + "memoryPeakBytes": 385277952, "artifactBytes": 178320, "workflowBytes": 1996, "estimatedTokens": 499