diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 017b61b..095706d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,33 @@ jobs: release-evidence/ opencode-artifacts-*.tgz retention-days: 14 + + portable-platforms: + name: portable-platform-${{ matrix.os }} + needs: verify + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: npm + - run: npm install -g npm@latest + - run: npm ci + - run: npm run build + - uses: actions/download-artifact@v4 + with: + name: exact-candidate-evidence + path: candidate-input + - name: Clean packed install and portable-output smoke + run: node scripts/candidate-platform-smoke.ts --output platform-evidence/${{ runner.os }}.json --candidate-dir candidate-input + - uses: actions/upload-artifact@v4 + with: + name: portable-platform-${{ runner.os }}-${{ runner.arch }} + path: platform-evidence/*.json + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index 9affde4..4083d46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ +.goal5-private/ docs/evidence/journeys/raw/ dist/ *.tgz diff --git a/benchmarks/page-quality/v1/benchmark-manifest.json b/benchmarks/page-quality/v1/benchmark-manifest.json new file mode 100644 index 0000000..5ee4298 --- /dev/null +++ b/benchmarks/page-quality/v1/benchmark-manifest.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "corpusId": "page-quality-v1", + "systems": ["opencode", "claude"], + "taskIds": [ + "dashboard", + "incident", + "pr-walkthrough", + "system-explainer", + "compare", + "plan-checklist", + "findings-table", + "interactive-decision" + ], + "minimumRunsPerSystemTask": 3, + "captureModes": [ + { "id": "desktop-light", "width": 1440, "height": 900, "colorScheme": "light" }, + { "id": "mobile-dark", "width": 390, "height": 844, "colorScheme": "dark" } + ], + "hardGates": [ + "facts", + "interactions", + "layout", + "composition", + "accessibility", + "dataHonesty", + "offlineBudget" + ], + "dimensions": [ + "taskOrientation", + "informationHierarchy", + "compositionDensity", + "typographyReadability", + "visualEncoding", + "coherenceCraft", + "interactionQuality", + "responsiveAdaptation" + ], + "thresholds": { + "openCodeBetterOrEquivalent": 0.8, + "openCodeDimensionMedian": 4, + "minimumReviewers": 3, + "requiredReviewerRoles": ["design-ux", "technical"] + } +} diff --git a/benchmarks/page-quality/v1/benchmark.template.json b/benchmarks/page-quality/v1/benchmark.template.json new file mode 100644 index 0000000..b878eec --- /dev/null +++ b/benchmarks/page-quality/v1/benchmark.template.json @@ -0,0 +1,17 @@ +{ + "schemaVersion": 1, + "runId": "pending-authorized-run", + "corpusId": "page-quality-v1", + "manifestSha256": "2badb79c0f21649b8ed6cb72a25fd8ca2b0a9436faf29d7ce8296384cf5a8897", + "authorization": { + "status": "pending", + "authorizedBy": null, + "authorizedAt": null, + "claudeAccountScope": null, + "modelProtocol": null, + "retentionDisposition": null + }, + "generations": [], + "reviewers": [], + "pairs": [] +} diff --git a/benchmarks/page-quality/v1/corpus.json b/benchmarks/page-quality/v1/corpus.json new file mode 100644 index 0000000..ee6377e --- /dev/null +++ b/benchmarks/page-quality/v1/corpus.json @@ -0,0 +1,124 @@ +{ + "schemaVersion": 1, + "corpusId": "page-quality-v1", + "license": "CC0-1.0", + "provenance": "Project-authored synthetic fixtures; no private repository, account, or Anthropic output is included.", + "captures": [ + { "id": "desktop", "width": 1440, "height": 900 }, + { "id": "mobile", "width": 390, "height": 844 } + ], + "bundles": [ + { + "id": "dashboard", + "task": "Operational KPI dashboard", + "fixture": "examples/patterns/dashboard.md", + "fixtureSha256": "06081c9519780f9f74c22886026552d6f49deb3243de134c619b5a00b8c6b658", + "primaryReader": "Release engineering lead", + "readerDecision": "Prioritize the cache TTL correction before investigating lower-volume services.", + "prompt": "Create an offline operational dashboard from only the supplied synthetic facts. Lead with the weekly deploy-failure finding, show the daily trend and service breakdown, preserve provenance, and do not invent causes or dates.", + "requiredFacts": ["There were 61 failures across 1,284 deploys.", "Wednesday and Thursday account for 26 failures tied to platform#412.", "svc-payments has 19 failures and 8 rollbacks."], + "forbiddenClaims": ["The cache TTL caused failures outside Wednesday and Thursday.", "Any service not present in the source bundle was measured."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic deployment telemetry", "missingValues": ["No owner is supplied for svc-web."], "longLabel": "Mean recovery time after a failed multi-service production deployment" }, + "interactions": [{ "id": "inspect-chart", "action": "Focus the primary chart and read its equivalent.", "expectedState": "The equivalent names the Wednesday peak of 14 and Saturday low of 5." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "incident", + "task": "Investigation timeline and postmortem", + "fixture": "examples/patterns/incident.md", + "fixtureSha256": "d4235196c8543314107206ce1e8fed61b4a5bb2f758fe24ae5d2cdfcb77704ce", + "primaryReader": "Incident commander", + "readerDecision": "Keep the rollback and implement the fraud check asynchronously with a 200 ms budget.", + "prompt": "Turn the supplied incident facts into an offline postmortem with status, chronology, trend, root cause, tested evidence, and next action. Preserve exact versions and times; do not infer customer loss.", + "requiredFacts": ["Peak checkout p99 was 2.6 seconds during a 38-minute incident.", "svc-payments 1.88.0 added a synchronous fraud check.", "Rollback to 1.87.2 restored p99 below 400 ms."], + "forbiddenClaims": ["Orders were lost.", "The fraud provider had an outage."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic incident log", "missingValues": ["No affected-user count was collected."], "longLabel": "Time from synchronous fraud-check deployment until three-minute recovery confirmation" }, + "interactions": [{ "id": "scan-timeline", "action": "Read the timeline in document order.", "expectedState": "Alert, suspect, rollback, and mitigation remain chronologically ordered." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "pr-walkthrough", + "task": "Annotated change review", + "fixture": "examples/patterns/pr-walkthrough.md", + "fixtureSha256": "dc1b092883ff432949dad08476cf01262f2f6cf0d0710aeae8a0ac4c8ecb54d5", + "primaryReader": "Pull-request reviewer", + "readerDecision": "Request changes until the default returns to 300 seconds and invalid TTL values are rejected.", + "prompt": "Create an offline PR walkthrough from the supplied diff and findings. Order findings by severity, annotate exact locations, show the relevant diff, tests, and a clear merge verdict without inventing code.", + "requiredFacts": ["The default TTL changes from 300 seconds to 5 seconds.", "ttlSeconds lacks validation.", "The feature flag itself is acceptable after the two fixes."], + "forbiddenClaims": ["The PR is safe to merge unchanged.", "Validation exists elsewhere in the repository."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic PR diff", "missingValues": ["No benchmark result accompanies the PR."], "longLabel": "Default cache time-to-live behavior when callers omit an explicit ttlSeconds value" }, + "interactions": [{ "id": "copy-verdict", "action": "Select the verdict for return to the coding session.", "expectedState": "The exact two blocking changes remain visible." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "system-explainer", + "task": "Architecture and data-flow explanation", + "fixture": "examples/patterns/system-explainer.md", + "fixtureSha256": "80fda196b54484e4902eb25bbf1e4a6248e7dc772ea99fff061c0cd9a53f0739", + "primaryReader": "Security-conscious developer", + "readerDecision": "Use portable HTML offline unless local collaboration or a registered datasource is explicitly needed.", + "prompt": "Explain the supplied artifact publication data flow as an offline page. Label validation, permission, storage, viewing, and datasource boundaries; include a sequence or relationship diagram and a clear reader decision.", + "requiredFacts": ["Validation happens before permission and writes.", "A denied operation writes nothing.", "Portable HTML uses connect-src none and adds no view-time authority."], + "forbiddenClaims": ["Portable HTML can execute a datasource.", "The viewer needs OpenCode installed."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic architecture contract", "missingValues": ["No hosted identity boundary is in this local-core diagram."], "longLabel": "Registered datasource command execution under a separately approved loopback-service permission" }, + "interactions": [{ "id": "read-diagram", "action": "Read the diagram equivalent without graphics.", "expectedState": "The same validation, permission, commit, and offline-view order is conveyed." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "compare", + "task": "Alternative comparison", + "fixture": "examples/patterns/compare-layouts.md", + "fixtureSha256": "7f2e8571019c4572c915427c412cbf5e1edb2ea4c14c10d04a58b58b89c00070", + "primaryReader": "Settings product owner", + "readerDecision": "Ship two-column tabs by default and keep the dense table behind a toggle.", + "prompt": "Compare the four supplied settings layouts in a consistent offline decision page. Make each variant genuinely distinct, preserve every tradeoff, and recommend only from the supplied hallway-test result.", + "requiredFacts": ["Two-column tabs are best for known-item visits.", "The dense table is fastest for power users.", "Card-grid modals break keyboard flow."], + "forbiddenClaims": ["A production A/B test was run.", "The card grid is accessible with no changes."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic design workshop", "missingValues": ["No numeric task-completion times are available."], "longLabel": "Two-column tabs with deep links for returning users who already know the target setting" }, + "interactions": [{ "id": "compare-variants", "action": "Read every variant and tradeoff in the same order.", "expectedState": "Four complete alternatives remain distinguishable at narrow width." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "plan-checklist", + "task": "Implementation and release plan", + "fixture": "examples/patterns/release-checklist.md", + "fixtureSha256": "c94a67977c27fdb1c06d0b085a8fb2033f8c1d787c2a7e050d2e42c2b28fc288", + "primaryReader": "Release owner", + "readerDecision": "Restore the performance benchmark cluster and obtain the security sign-off before tagging.", + "prompt": "Create an offline release checklist from the supplied state. Show progress, completed and open work, dependencies, risks, and the next blocking action. Do not treat post-tag work as a pre-tag gate.", + "requiredFacts": ["Seven of eleven items are complete.", "The performance baseline is blocked because the benchmark cluster is down.", "Security sign-off on the token endpoint is still open."], + "forbiddenClaims": ["The release is ready to tag.", "Post-tag dashboard watch blocks the tag."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic release plan", "missingValues": ["The benchmark cluster recovery owner is not assigned."], "longLabel": "Performance baseline re-run against the restored two-core four-gibibyte benchmark environment" }, + "interactions": [{ "id": "scan-checklist", "action": "Navigate the task list by keyboard.", "expectedState": "Checked state and the two pre-tag blockers are announced." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "findings-table", + "task": "Prioritized findings table", + "fixture": "examples/patterns/license-audit.md", + "fixtureSha256": "7703cfec91a82278ea9f7d222fdd247eddb3632733a4ff4bad0310079a26e8d5", + "primaryReader": "Release and legal reviewer", + "readerDecision": "Replace or isolate imgopt and ship the required termui notice before release.", + "prompt": "Create an offline dependency-license findings page from the supplied synthetic inventory. Prioritize copyleft issues, provide sortable detail and provenance, and do not generalize beyond the listed packages.", + "requiredFacts": ["Twelve production dependencies are listed.", "vendor/imgopt is GPL-3.0 and linked into the CLI.", "vendor/termui is LGPL-2.1 and requires a notice file."], + "forbiddenClaims": ["Legal counsel approved the distribution.", "All transitive dependencies were audited beyond the supplied inventory."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic dependency inventory", "missingValues": ["No legal-review ticket is supplied."], "longLabel": "Redistribution disposition for dynamically linked native terminal user-interface bindings" }, + "interactions": [{ "id": "sort-license-table", "action": "Sort the table by package and numeric size using keyboard controls.", "expectedState": "Sort direction is announced and all twelve rows remain available." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + }, + { + "id": "interactive-decision", + "task": "Tune values and return a decision", + "fixture": "examples/patterns/tune-controls.md", + "fixtureSha256": "7646b761c15b76b9e249aa9827db07f557ec574030df1db3502bc73f4996063c", + "primaryReader": "Developer tuning a generated transition", + "readerDecision": "Choose frequency and amplitude, inspect the noisy-series window, then copy the chosen values back to the session.", + "prompt": "Build an offline tuning page from the supplied ranges and series. Controls must be discoverable, keyboard operable, immediate, reversible, and include a copy-back path; do not add network-backed data.", + "requiredFacts": ["Frequency ranges from 0.5 to 8 in 0.5 steps.", "Amplitude ranges from 0.2 to 3 in 0.2 steps.", "The twelve-week series rises overall from 12 to 40."], + "forbiddenClaims": ["The selected values are persisted remotely.", "The noisy series is live data."], + "sourceBundle": { "license": "CC0-1.0", "provenance": "Synthetic tuning inputs", "missingValues": ["No preferred final frequency or amplitude is supplied."], "longLabel": "Amplitude applied to the generated sine-wave transition preview before copying values back" }, + "interactions": [{ "id": "tune-and-copy", "action": "Change both sliders by keyboard, reset to supplied defaults, and activate copy.", "expectedState": "The chart updates immediately and copy confirmation names the return path." }], + "stressCases": ["long-label", "missing-value", "dense-data", "narrow-viewport"] + } + ] +} diff --git a/docs/component-spec.md b/docs/component-spec.md index ee22587..82dd616 100644 --- a/docs/component-spec.md +++ b/docs/component-spec.md @@ -87,10 +87,15 @@ Component JSON schemas have exactly one home: `skills/artifact-pages/reference/c rationale; never copy schema tables into it. Available fences: `stats`, `timeline`, `findings`, `compare`, `callout`, `progress`, `diff`, -`copy`, `mermaid`, `decisions`, `table`; chart fences `vega-lite` / `vega` / `echarts`; +`copy`, `mermaid`, `decisions`, `table`, `frame`; chart fences `vega-lite` / `vega` / `echarts`; interactive controls via vega-lite `params.bind` and echarts `dataZoom` (verified live in browser QA); free-form interactivity stays in raw-HTML mode (`format: "html"`). +Frontmatter `composition:` selects one bounded layout grammar: `standard`, `narrative`, +`dashboard`, `split`, `dense`, `quiet`, or `full`. These modes change visual scale, section +grouping, and density while retaining source/DOM order. Invalid values refuse publication. +At narrow widths every multi-column mode becomes the same logical single-column sequence. + ### Data honesty rules (from Claude Code's dashboard/dataviz skills) - Format numbers for scanning: unit + 2–3 significant figures, thousands separators; at most diff --git a/docs/evidence/journeys/goal-5-status-2026-08-18.md b/docs/evidence/journeys/goal-5-status-2026-08-18.md new file mode 100644 index 0000000..3ee7bc2 --- /dev/null +++ b/docs/evidence/journeys/goal-5-status-2026-08-18.md @@ -0,0 +1,19 @@ +# Goal 5 first-use and comprehension status — 2026-08-18 + +- Study owner: **unassigned; recruitment is not authorized**. +- Candidate: commit `84f866ed742eeb08ba668115342ef0e5896dead7`, package `0.15.0`, SHA-256 + `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2`. +- Corpus: journey corpus version 1 and participant materials version 1 are frozen. +- Target install-capable policy cells: Ubuntu LTS desktop, current macOS desktop, previous macOS + desktop, Windows 11 native desktop, and Windows 11 WSL desktop. Exact release-time versions + remain unresolved. Android Chrome and iOS Safari are viewer/support cells, not README install + tasks, and remain unresolved in the certification support matrix. +- Consented representative primary participants: 0. +- Eligible README-only first-use runs: 0; every target install cell is missing. +- Eligible comprehension participants: 0 of the minimum 10. +- Withdrawals, failures, and exclusions: 0 because collection has not started. +- Raw participant evidence: none collected. + +The checked-in validator, redacted aggregate, and boundary tests are ready. This record is +`incomplete`, not a participant failure and not a pass. It blocks `OUT-02`, `OUT-03`, supported +platform, equal-or-better, and Local artifact core certification claims. diff --git a/docs/evidence/journeys/goal-5-status-2026-08-19.md b/docs/evidence/journeys/goal-5-status-2026-08-19.md new file mode 100644 index 0000000..a561fb0 --- /dev/null +++ b/docs/evidence/journeys/goal-5-status-2026-08-19.md @@ -0,0 +1,16 @@ +# Goal 5 first-use and comprehension status — 2026-08-19 + +- Study owner: **unassigned; recruitment is not authorized**. +- Candidate: commit `2908d52ef94bb0f8cf74d7f2be021ada78a0d661`, package `0.15.0`, + SHA-256 `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd`. +- Corpus: journey corpus version 1 and participant materials version 1 are frozen. +- Consented representative primary participants: 0. +- Eligible README-only first-use runs: 0; every target install cell is missing. +- Eligible comprehension participants: 0 of the minimum 10. +- Withdrawals, failures, and exclusions: 0 because collection has not started. +- Raw participant evidence: none collected. + +Automated exact-tarball technical observations now pass on one Ubuntu runner, one macOS +runner, and one Windows Server runner. They are not participants and do not cover Windows 11, +WSL, previous macOS, browsers, or mobile devices. This record remains `incomplete`, blocks +`OUT-02`/`OUT-03`, and cannot support usability, support, comparison, or certification claims. diff --git a/docs/evidence/page-quality/2026-08-18-benchmark-status.md b/docs/evidence/page-quality/2026-08-18-benchmark-status.md new file mode 100644 index 0000000..c1e8997 --- /dev/null +++ b/docs/evidence/page-quality/2026-08-18-benchmark-status.md @@ -0,0 +1,19 @@ +# Goal 5 comparative benchmark status — 2026-08-18 + +Status: **incomplete; equal-or-better is unverified** + +- Corpus: `page-quality-v1`, eight hash-bound permission-safe tasks. +- Manifest SHA-256: `2badb79c0f21649b8ed6cb72a25fd8ca2b0a9436faf29d7ce8296384cf5a8897`. +- Required unselected end-to-end generations: 48 (three per system/task); collected: 0. +- Required randomized pairs: 24; scored: 0. +- Required eligible independent reviewers: at least 3 including design/UX and technical; + recruited: 0. +- Claude execution authority, comparable model/settings protocol, and retention disposition: + absent. +- Renderer-only browser cells: 16/16 pass in the separately scoped + [`local composition report`](2026-08-18-local-composition.md). + +`npm run quality:benchmark -- benchmarks/page-quality/v1/benchmark.template.json` validates +the frozen input shape and returns `incomplete`. Synthetic unit records exercise pass/fail, +no-cherry-pick, blinding mappings, confidence, all rubric medians, and hard-gate boundaries; +they are diagnostics only and are never benchmark evidence. diff --git a/docs/evidence/page-quality/2026-08-18-local-composition.md b/docs/evidence/page-quality/2026-08-18-local-composition.md new file mode 100644 index 0000000..d793ad7 --- /dev/null +++ b/docs/evidence/page-quality/2026-08-18-local-composition.md @@ -0,0 +1,50 @@ +# Goal 5 local composition evidence + +Status: **pass for the renderer-only local browser track; comparative and certification claims remain unverified** + +Captured: 2026-08-18 + +Candidate: `agent/goal-5-local-core` working tree after Goal 5 packet approval + +## Scope + +This report covers the eight checked-in `page-quality-v1` fixtures rendered through the +ordinary Markdown/component pipeline. It is local regression evidence for the renderer track +defined in `docs/page-quality-benchmark.md`; it is not a Claude comparison, a supported-platform +matrix, a representative-user study, or Local artifact core certification. + +The retained set contains each fixture at 1440 × 900 in light mode and 390 × 844 in dark mode +with reduced motion. Chromium was supplied by `selenium/standalone-chromium` version 151. +Screenshots and machine-readable observations are under +[`2026-08-18-local-composition/`](2026-08-18-local-composition/), with the hash-bound aggregate +in [`summary.json`](2026-08-18-local-composition/summary.json). + +## Result + +- 16 of 16 required cells passed. +- Maximum useful-content time was 971 ms. +- Maximum observed layout shift was 0.03243 against the 0.1 local threshold. +- Minimum primary-visual utilization was 0.907; minimum chart fill was 1.0. +- No cell reported document/main horizontal overflow, clipped text, renderer errors, severe + browser logs, or external HTTP requests. +- Keyboard traces retained the skip-link path, sortable table state, decision/range behavior, + and copy behavior where applicable. +- The system-explainer Mermaid SVG and dashboard Vega-Lite chart were measured after settled + rendering; both recompose at the narrow viewport without clipping or accidental dead space. + +The deterministic gate is rerunnable with: + +```sh +npm run quality:corpus +npm run quality:local -- docs/evidence/page-quality/2026-08-18-local-composition \ + docs/evidence/page-quality/2026-08-18-local-composition/summary.json +node --test test/page-quality-corpus.test.ts test/page-quality-local-report.test.ts +``` + +## Evidence boundary and open gates + +No current Claude Artifact outputs were generated, no Claude account or retention authority was +assumed, and no independent reviewer scores were collected. Therefore the equal-or-better claim +remains unverified. Manual screen-reader review of these new composition changes, supported +Firefox/Safari/OS cells, the consented first-use/comprehension study, and exact-candidate release +evidence also remain pending. This report cannot be reused to mark those gates passed. diff --git a/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.json b/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.json new file mode 100644 index 0000000..535c9d3 --- /dev/null +++ b/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.json @@ -0,0 +1,88 @@ +{ + "schemaVersion": 1, + "capturedAt": "2026-08-18T15:56:31.870Z", + "browser": "Chromium 151 via selenium/standalone-chromium", + "fixture": "http://127.0.0.1:43115/compare.html", + "requested": { + "width": 1440, + "height": 900, + "colorScheme": "light", + "reducedMotion": false + }, + "usefulContentMs": 270, + "ready": true, + "keyboard": { + "firstTab": { + "className": "skip-link", + "tag": "A", + "text": "Skip to main content" + } + }, + "observations": { + "chartGeometry": [], + "clippedText": [], + "colorScheme": "light", + "composition": "composition-split", + "diagramGeometry": [], + "documentHorizontalOverflow": false, + "headings": [ + "Settings panel — four layout directions", + "Recommendation" + ], + "interactive": { + "buttons": 1, + "radios": 0, + "ranges": 0 + }, + "layoutShift": 0, + "mainHorizontalOverflow": false, + "readyState": "complete", + "reducedMotion": false, + "renderErrors": [], + "sections": [ + { + "classes": "section-card", + "heading": "Recommendation", + "utilization": null, + "visualWidth": null, + "width": 1240 + } + ], + "viewport": { + "devicePixelRatio": 1, + "height": 757, + "width": 1440 + } + }, + "accessibilityTree": [ + { + "role": "RootWebArea", + "name": "Settings panel — four layout directions" + }, + { + "role": "banner", + "name": "" + }, + { + "role": "main", + "name": "" + }, + { + "role": "heading", + "name": "Settings panel — four layout directions" + }, + { + "role": "button", + "name": "Theme: system. Activate to switch." + }, + { + "role": "heading", + "name": "Recommendation" + } + ], + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/compare.html" + ], + "externalHttpRequests": [] +} diff --git a/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.png b/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.png new file mode 100644 index 0000000..5ab8374 Binary files /dev/null and b/docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.png differ diff --git a/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.json b/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.json new file mode 100644 index 0000000..2f0de55 --- /dev/null +++ b/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.json @@ -0,0 +1,88 @@ +{ + "schemaVersion": 1, + "capturedAt": "2026-08-18T15:57:13.548Z", + "browser": "Chromium 151 via selenium/standalone-chromium", + "fixture": "http://127.0.0.1:43115/compare.html", + "requested": { + "width": 390, + "height": 844, + "colorScheme": "dark", + "reducedMotion": true + }, + "usefulContentMs": 233, + "ready": true, + "keyboard": { + "firstTab": { + "className": "skip-link", + "tag": "A", + "text": "Skip to main content" + } + }, + "observations": { + "chartGeometry": [], + "clippedText": [], + "colorScheme": "dark", + "composition": "composition-split", + "diagramGeometry": [], + "documentHorizontalOverflow": false, + "headings": [ + "Settings panel — four layout directions", + "Recommendation" + ], + "interactive": { + "buttons": 1, + "radios": 0, + "ranges": 0 + }, + "layoutShift": 0, + "mainHorizontalOverflow": false, + "readyState": "complete", + "reducedMotion": true, + "renderErrors": [], + "sections": [ + { + "classes": "section-card", + "heading": "Recommendation", + "utilization": null, + "visualWidth": null, + "width": 343 + } + ], + "viewport": { + "devicePixelRatio": 1, + "height": 701, + "width": 390 + } + }, + "accessibilityTree": [ + { + "role": "RootWebArea", + "name": "Settings panel — four layout directions" + }, + { + "role": "banner", + "name": "" + }, + { + "role": "main", + "name": "" + }, + { + "role": "heading", + "name": "Settings panel — four layout directions" + }, + { + "role": "button", + "name": "Theme: system. Activate to switch." + }, + { + "role": "heading", + "name": "Recommendation" + } + ], + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/compare.html" + ], + "externalHttpRequests": [] +} diff --git a/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.png b/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.png new file mode 100644 index 0000000..27ebda7 Binary files /dev/null and b/docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.png differ diff --git a/docs/evidence/page-quality/2026-08-18-local-composition/dashboard-desktop.json b/docs/evidence/page-quality/2026-08-18-local-composition/dashboard-desktop.json new file mode 100644 index 0000000..d0bb06c --- /dev/null +++ b/docs/evidence/page-quality/2026-08-18-local-composition/dashboard-desktop.json @@ -0,0 +1,116 @@ +{ + "schemaVersion": 1, + "capturedAt": "2026-08-18T15:56:15.884Z", + "browser": "Chromium 151 via selenium/standalone-chromium", + "fixture": "http://127.0.0.1:43115/dashboard.html", + "requested": { + "width": 1440, + "height": 900, + "colorScheme": "light", + "reducedMotion": false + }, + "usefulContentMs": 676, + "ready": true, + "keyboard": { + "firstTab": { + "className": "skip-link", + "tag": "A", + "text": "Skip to main content" + } + }, + "observations": { + "chartGeometry": [ + { + "childHeight": 560, + "childWidth": 1184, + "height": 560, + "markup": " +claudeAccountScope: +modelProtocol: +retentionDisposition: +``` + +The coordinator records the visible Claude Code/OpenCode versions, model/settings, capture +date, prompt and fixture hashes, and whether a human edited an output. Three consecutive, +unselected generations are required for each of eight tasks in both systems: 48 total. + +After the generation-only private run is complete, create the mapping with a new private seed +of at least 32 random bytes: + +```sh +npm run quality:benchmark -- prepare \ + .goal5-private/generation-run.json .goal5-private/blinding-seed \ + .goal5-private/paired-run.json .goal5-private/reviewer-packet.json +``` + +The command refuses existing outputs. `.goal5-private/paired-run.json` contains the private +system mapping; only `reviewer-packet.json` may go to reviewers. Its 24 A/B pairs contain neutral +`blinded://...` resource names and empty score forms. The coordinator stages the corresponding +desktop, mobile, and interaction resources under those neutral names without exposing source +paths, generation IDs, the seed, or the private mapping. + +## 2. Independent corpus review + +One reviewer who did not author or maintain this repository inspects every bundle in +`benchmarks/page-quality/v1/` and records: + +```text +reviewerId: +independent: true|false +conflicts: +reviewedAt: +corpusId and manifestSha256: +result: pass|fail +findings: +``` + +For each of the eight tasks, the reviewer answers yes/no with a reason: + +- Can every required/forbidden fact be decided from the supplied bundle alone? +- Is the prompt neutral between systems and free of preferred-output wording? +- Is the reader decision and every required interaction unambiguous? +- Are all inputs synthetic/project-owned, non-private, hash-bound, and correctly licensed? +- Is redistribution status explicit, with no private or ambiguous third-party material? + +Any “no”, unresolved conflict, or omitted task fails this review and blocks corpus verification. + +## 3. Blinded page-quality panel + +At least three independent, conflict-free reviewers consent to retained scores. The combined +panel includes `design-ux` and domain-appropriate `technical` roles; a third `reader` role is +recommended. Each receives only the neutral reviewer packet and staged A/B resources. + +For every pair, every reviewer scores A and B from 1–5 on all eight named dimensions, chooses +`a`, `equivalent`, or `b` overall, and supplies a short reason. No score may be inferred, +rounded, copied between reviewers, or omitted. The coordinator merges scores into the private +paired run, validates it, and retains the complete distribution: + +```sh +npm run quality:benchmark -- .goal5-private/paired-run-with-scores.json +``` + +## 4. Goal 5 manual screen-reader checklist + +This is a new candidate-bound run; the earlier Goal 3 attestation cannot be reused. Record the +reviewer, date, exact OS, screen reader/version, browser/version, candidate digest, task IDs, +every failure, and an overall `pass` or `fail`. + +Open all eight normalized pages and verify for each: + +- the page title, banner, main landmark, section headings, and footer are announced in logical order; +- visual layout does not change DOM reading or focus order; +- primary charts/diagrams/frames have useful names and text equivalents; +- dense, split, full, quiet, and narrow compositions do not hide or repeat content; +- skip-link and keyboard focus are discoverable and visible; +- tables announce caption, headers, and sort state; +- controls announce role, name, current state/value, and changed state; +- status/copy feedback is announced without moving focus unexpectedly; and +- no unlabeled control, empty landmark, confusing repetition, trap, or reading-order defect occurs. + +The `findings-table` and `system-explainer` pages must exercise table sorting. The +`interactive-decision` page must exercise radio choice, range change, and copy feedback. The +`pr-walkthrough` page must exercise verdict copy feedback. A failure on any task keeps +`QUAL-04` incomplete. + +## 5. Representative-user study + +The authorized study owner uses [`goal-5-participant-materials.md`](../../journeys/goal-5-participant-materials.md) +and the strict private-record workflow in [`docs/journeys/README.md`](../../journeys/README.md). +No recruitment begins until the owner, candidate, claimed platform IDs, raw-record location, +access list, withdrawal contact, and deletion date are recorded. + +At least ten eligible representative-primary participants are required. Every claimed +install-capable support cell needs one first-time README-only create/reopen pass; these people +may also enter the ten-person comprehension denominator. Raw answers and participant codes +remain access-controlled and uncommitted. Only the aggregate, raw-file digest, failures, +exclusions, withdrawals, owner, and deletion disposition enter git. + +## 6. Exact support/browser matrix + +Resolve moving labels to exact versions on the test date. Supply the exact candidate to every +target cell in `docs/governance-policy.json`; do not promote the existing Linux/macOS/Windows +CI diagnostics to support. + +For a standards-compatible local or remote WebDriver endpoint, run the browser-neutral smoke +at desktop and mobile sizes as applicable. Put endpoint credentials in the process environment, +not the command line: + +```sh +WEBDRIVER_ENDPOINT='' \ +WEBDRIVER_CAPABILITIES_FILE='.goal5-private/provider-capabilities.json' \ +node scripts/support-browser-smoke.ts \ + --url '' --report .goal5-private/result.json --screenshot .goal5-private/result.png \ + --browser firefox --browser-version '' --platform '' \ + --width 1440 --height 900 +``` + +The optional capabilities file supplies non-secret vendor device/options needed by a browser +lab. Core browser/platform fields cannot be overridden, credential-looking fields are refused, +and the file is bounded to 64 KiB; authenticate only through the private endpoint environment +value. The command refuses overwrite, redacts endpoint credentials and target queries, records exact +returned browser/platform capabilities, and checks settlement, overflow, clipping, render +errors, keyboard interactions, screenshots, and external resource requests. Standard WebDriver +does not expose a portable accessibility tree or console-log endpoint, so the report explicitly +cannot replace manual accessibility, console, physical-device, or first-use evidence. + +## 7. Sign-offs and publication boundary + +After all rows pass, record separate accountable `release`, `security`, and `support` sign-offs +with actor and ISO timestamp. Packet approval is not any of those sign-offs. + +Tagging and npm publication remain a separately authorized mutation. Only after exact +contemporaneous authority may the release workflow publish these bytes and perform registry +integrity, signature, and provenance readback. Until then the deterministic decision remains +`refused`, all certification/comparison/support claims remain disabled, and provider mutation +count remains zero. diff --git a/docs/evidence/releases/2026-08-18-local-core-candidate.json b/docs/evidence/releases/2026-08-18-local-core-candidate.json new file mode 100644 index 0000000..cdb6f4d --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-candidate.json @@ -0,0 +1,152 @@ +{ + "schemaVersion": 1, + "recordId": "local-core-2026-08-18-candidate", + "level": "local-artifact-core", + "decision": "pending", + "candidate": { + "status": "frozen", + "commit": "84f866ed742eeb08ba668115342ef0e5896dead7", + "version": "0.15.0", + "tarball": "opencode-artifacts-0.15.0.tgz", + "sha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "sri": "sha512-ErzQBzSxNz/nuxsTfwi5uozrJvK5joo7c6SyHeWz9THM49ETSY5PajdrxdfXq0J86lGAK4HdahCKyEo1gPIjKA==", + "corpusVersions": { "pageQuality": "page-quality-v1", "journey": 1 } + }, + "requirements": [ + { + "id": "code-tests", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-ci.md", "date": "2026-08-18", + "owner": "GitHub Actions run 32161443356", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Install, build, 255 tests, structural assertions, and exact-candidate evidence generation.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }] + }, + { + "id": "package-host", + "status": "pass", + "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-host-matrix.json", + "date": "2026-08-18", + "owner": "Goal 5 local evidence run", + "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Clean temporary install, stable OpenCode discovery, tool schemas, read-only packed smoke, and skill discovery.", + "environment": "Linux x64, Node 24, OpenCode stable 1.18.18" + }] + }, + { + "id": "browser-accessibility", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/page-quality/2026-08-18-local-composition.md", "date": "2026-08-18", + "owner": "Goal 5 automated browser run", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Sixteen Chromium desktop/mobile/mode cells; manual screen-reader review remains missing.", + "environment": "Linux x64, Chromium 151" + }] + }, + { + "id": "migrations", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-ci.md", "date": "2026-08-18", + "owner": "GitHub Actions run 32161443356", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Schema, lifecycle, migration, rollback, fault-boundary, and transaction tests in the complete suite.", + "environment": "GitHub ubuntu-latest, Node 24" + }] + }, + { + "id": "security-privacy", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-ci.md", "date": "2026-08-18", + "owner": "GitHub Actions run 32161443356", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Adversarial renderer, path, permission, secret-scan, CSP, governance, and privacy-policy checks.", + "environment": "GitHub ubuntu-latest, Node 24" + }] + }, + { + "id": "performance", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-performance.md", "date": "2026-08-18", + "owner": "Goal 5 constrained benchmark run", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Three renderer workloads, 36 CLI samples, 42 fresh-profile browser samples, time and byte budgets.", + "environment": "Linux x64, Node 24.19.0, Chromium 151, 2 cores, 4 GiB" + }] + }, + { + "id": "support-matrix", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-platform-linux.json", "date": "2026-08-19", + "owner": "Goal 5 clean packed-platform smoke", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Linux clean tarball install, installed CLI render, strict offline CSP, package-tree removal, and byte-identical reopen. Browser, representative-user, and support claims are excluded.", + "environment": "Linux 6.8.0-137-generic x64, Node 24.19.0" + }] + }, + { "id": "page-quality", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "first-use", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "comprehension", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { + "id": "audit-license-sbom", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-ci.md", "date": "2026-08-18", + "owner": "GitHub Actions run 32161443356", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Zero-vulnerability audit, 217-entry license inventory/dispositions, 211-component CycloneDX SBOM, dependency signatures and attestations.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }] + }, + { + "id": "integrity-provenance", "status": "pending", "applicabilityReason": null, + "evidence": [ + { + "path": "docs/evidence/releases/2026-08-18-local-core-ci.md", "date": "2026-08-18", + "owner": "GitHub Actions run 32161443356", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Reproducible pack digest and CI in-toto subject; no registry publication/signature/readback for this candidate.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }, + { + "path": "docs/evidence/releases/2026-08-18-local-core-consumer-verification.md", "date": "2026-08-18", + "owner": "Goal 5 consumer-verification run", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Two byte-identical packs, SHA-256, npm shasum, SRI, archive shape, and offline verification procedure; registry readback remains absent.", + "environment": "Linux x64, Node 24.19.0, two isolated npm caches" + } + ] + }, + { + "id": "claims-rollback", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-18-local-core-claims.md", "date": "2026-08-18", + "owner": "Goal 5 claim and rollback audit", "result": "pass", + "candidateSha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "scope": "Exact packaged/public claim surfaces, fail-closed certification, pre-release refusal, and zero-mutation rollback procedure.", + "environment": "Frozen candidate archive and repository policy surfaces" + }] + } + ], + "signoffs": [ + { "role": "release", "status": "pending", "by": null, "at": null }, + { "role": "security", "status": "pending", "by": null, "at": null }, + { "role": "support", "status": "pending", "by": null, "at": null } + ], + "blockers": [ + "Manual screen-reader review of the new composition behavior is absent.", + "Authorized current Claude same-input runs and retention disposition are absent.", + "Three independent benchmark reviewers are absent.", + "Representative-user first-use and comprehension records are absent.", + "The exact claimed support matrix is incomplete.", + "Accountable release, security, and support sign-offs are absent." + ], + "claims": { + "certification": false, + "equalOrBetter": false, + "supportedPlatformIds": [] + }, + "providerMutationCount": 0 +} diff --git a/docs/evidence/releases/2026-08-18-local-core-ci.md b/docs/evidence/releases/2026-08-18-local-core-ci.md new file mode 100644 index 0000000..2dfe60a --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-ci.md @@ -0,0 +1,40 @@ +# Local-core exact-candidate CI evidence — 2026-08-18 + +Status: **pass for automated code, package, audit, license, SBOM, signature, and stable-host gates** + +- GitHub Actions run: `32161443356`, attempt 1, job `95791053464`. +- Workflow result: green after install, build, 255 tests, structural checks, candidate evidence, + and artifact upload. +- Frozen package: `opencode-artifacts-0.15.0.tgz`. +- Candidate SHA-256: `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2`. +- npm integrity: `sha512-ErzQBzSxNz/nuxsTfwi5uozrJvK5joo7c6SyHeWz9THM49ETSY5PajdrxdfXq0J86lGAK4HdahCKyEo1gPIjKA==`. +- Pull-request merge coordinate used by the workflow: + `a3d3b2f04d4c195b6ad082fc8f3225d5111023c5`; its packed subject exactly matches the + frozen source candidate's reproducible tarball digest. +- Audit: 0 vulnerabilities at info, low, moderate, high, and critical across 217 dependency + entries. +- Licenses: 217 entries inventoried; the one missing declaration and two compound choices match + their exact checked-in file/version/hash dispositions. +- Registry dependency verification: 212 packages with verified signatures and 22 with verified + attestations. +- CycloneDX SBOM: 211 components, serial + `urn:uuid:0191ab18-e6ee-452f-9a0e-cc93f0bf917d`. +- Packed stable-host matrix: pass for OpenCode `1.18.18`; broader ranges remain unproven. + +The uploaded `exact-candidate-evidence` artifact was downloaded and independently rehashed. +Its retained file hashes are: + +| File | SHA-256 | +|---|---| +| `audit.json` | `1a6880655b7fe998c3f6cb838d1afedac09b478a552e3f1f38bff5a0416b74b8` | +| `licenses.json` | `4bc307c41b17ccdc6e6590f6f940939033b86cf3444617abeb2b0f356e101cea` | +| `opencode-host-matrix.json` | `acdcd69fe954fb175ae84950b014b6e46b41cf04268c6a19f47015f470430bf6` | +| `pack.json` | `bfc85967fa2d160c77a8b72de15743949111935a5826eed54030872e3875b705` | +| `provenance.intoto.json` | `bbc308c9a263f7b089db75f74e69d15bbac8d60245edfb17dc888d947cb859f8` | +| `sbom.cdx.json` | `c943f29ec4d1b73cd70ab4b2525603ba58e093d8c413a6cd9e3ea7e1e28cf8a5` | +| `signatures.txt` | `96dbc2926fd8b9c4d367ef9d0b827d3c6ad9301a3aba7b872601edb3d234df12` | +| `opencode-artifacts-0.15.0.tgz` | `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2` | + +This evidence does not supply manual accessibility, broad platform support, current Claude +comparison, representative-user outcomes, accountable sign-offs, or registry publication +readback for this candidate. diff --git a/docs/evidence/releases/2026-08-18-local-core-claims.md b/docs/evidence/releases/2026-08-18-local-core-claims.md new file mode 100644 index 0000000..4b41093 --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-claims.md @@ -0,0 +1,52 @@ +# Local artifact core candidate: claims and rollback audit + +Date: 2026-08-18 + +Result: **pass** + +Candidate SHA-256: `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2` + +## Claim audit + +The exact candidate archive was inspected, not only the later working tree. Its packaged +README labels the project a public preview, says it is unsupported and uncertified, reports +zero fully supported platform/browser cells, and disclaims representative-user first-use and +comprehension baselines. + +The remaining authoritative public surfaces agree: + +- `docs/support-policy.md` reports no fully supported platform/browser cells and keeps every + target desktop/mobile cell unverified. +- `docs/page-quality-benchmark.md` says equal-or-better quality is a target and is not + verified. +- `docs/roadmap.md` permits no capability-level claim for an unsupported public preview and + identifies the missing comparative, human, and platform inputs. +- `docs/governance-policy.json` contains no supported target cell. +- the candidate certification record has `certification: false`, `equalOrBetter: false`, an + empty `supportedPlatformIds` list, decision `pending`, and provider mutation count zero. + +No local-artifact-core certification, equal-or-better comparison, production-readiness, or +supported-platform statement is active. The present claim is therefore strictly narrower +than the evidence, not broader. + +## Fail-closed and rollback audit + +The focused release-model and transition tests passed on 2026-08-18. They verify that missing +requirements or sign-offs refuse certification and return zero provider mutations, that an +unsupported preview cannot claim certification, and that pre-publish/post-publish gates fail +when required evidence is absent. + +The current candidate transition was executed and returned `refused`. The absent manual, +comparative, representative-user, support-matrix, registry, and sign-off evidence remained +visible. No tag, registry publish, deployment, or provider mutation was performed. + +Rollback for this pre-release candidate is refusal: keep the public-preview disclosures, +leave certification/support/comparison claims disabled, and do not create a release tag or +invoke the tag-triggered publishing workflow. If later evidence contradicts the candidate, +amend and reapprove the affected packets, freeze new exact bytes, and rerun every candidate- +bound gate. If any claim text is accidentally promoted before those gates pass, revert that +claim to the disclosures audited here; no user data or provider state needs restoration +because this candidate has made zero provider mutations. + +This row does not approve a release. Final publication still requires the three named +sign-offs plus explicit tag, npm, and provider-mutation authority. diff --git a/docs/evidence/releases/2026-08-18-local-core-consumer-verification.md b/docs/evidence/releases/2026-08-18-local-core-consumer-verification.md new file mode 100644 index 0000000..c2b1cc3 --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-consumer-verification.md @@ -0,0 +1,56 @@ +# Local artifact core candidate: consumer verification + +Date: 2026-08-18 + +Result: **pass for local byte verification; registry verification remains pending** + +Candidate source commit: `84f866ed742eeb08ba668115342ef0e5896dead7` + +Candidate archive: `opencode-artifacts-0.15.0.tgz` + +## Exact coordinates + +| Field | Value | +|---|---| +| SHA-256 | `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2` | +| npm SHA-1 shasum | `9eeff3bbc6f016aaa2b10c71df15e891904a8ad0` | +| npm SRI | `sha512-ErzQBzSxNz/nuxsTfwi5uozrJvK5joo7c6SyHeWz9THM49ETSY5PajdrxdfXq0J86lGAK4HdahCKyEo1gPIjKA==` | +| Packed size | 126383 bytes | +| Unpacked size | 542885 bytes | +| Entries | 69 | + +Two separate temporary destinations and npm caches produced archives with the same three +coordinates. `cmp` exited zero, so the archives were byte-identical. The resulting SHA-256 +also equals the frozen candidate record and the exact tarball retained by GitHub Actions run +`32161443356`. + +## Offline verification procedure + +After obtaining the candidate archive through an authorized channel, a consumer can verify +its bytes without running package code: + +```sh +sha256sum opencode-artifacts-0.15.0.tgz +sha1sum opencode-artifacts-0.15.0.tgz +tar -tzf opencode-artifacts-0.15.0.tgz +``` + +The first two outputs must equal the coordinates above. The archive listing must use only +`package/` paths and contain `package/package.json`, `package/README.md`, `package/dist/`, +`package/skills/`, `package/agents/`, and `package/LICENSE`. + +The SRI can be checked with a local SHA-512 implementation: + +```sh +openssl dgst -sha512 -binary opencode-artifacts-0.15.0.tgz | base64 +``` + +Prefix the output with `sha512-` and compare the complete value above. + +## Boundary + +This is consumer-verifiable evidence for candidate bytes and reproducible packing. It does +not establish that npm served those bytes, that a registry signature exists, or that npm +provenance binds them to the intended tag, commit, and workflow. Those checks require an +authorized publication and registry readback, so the certification record's +`integrity-provenance` requirement remains pending and no provider mutation occurred. diff --git a/docs/evidence/releases/2026-08-18-local-core-host-matrix.json b/docs/evidence/releases/2026-08-18-local-core-host-matrix.json new file mode 100644 index 0000000..ea0baf7 --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-host-matrix.json @@ -0,0 +1,765 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-18T16:21:22.360Z", + "candidate": { + "filename": "opencode-artifacts-0.15.0.tgz", + "sha256": "f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2", + "packageVersion": "0.15.0" + }, + "compatibility": { + "currentStable": "1.18.18", + "oldestTested": "1.18.18", + "executedVersions": [ + "1.18.18" + ], + "deduplicated": true, + "broaderRangeProven": false, + "v2BetaExcluded": true + }, + "environment": { + "node": "v24.19.0", + "platform": "linux", + "architecture": "x64", + "cleanRoots": [ + "config", + "data", + "cache", + "state" + ], + "providerInference": false + }, + "install": { + "package": { + "command": [ + "npm", + "install", + "--prefix", + "/tmp/opencode-host-matrix-FqauE1/package", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "/tmp/goal5-candidate/opencode-artifacts-0.15.0.tgz" + ], + "exitCode": 0, + "output": "(node:247879) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n(Use `node --trace-deprecation ...` to show where the warning was created)\n\nadded 207 packages in 19s\n" + }, + "currentResolution": { + "command": [ + "npm", + "view", + "opencode-ai", + "version", + "--json" + ], + "exitCode": 0, + "output": "\"1.18.18\"\n" + }, + "hosts": [ + { + "version": "1.18.18", + "command": { + "command": [ + "npm", + "install", + "--prefix", + "/tmp/opencode-host-matrix-FqauE1/host-1-18-18", + "--no-audit", + "--no-fund", + "opencode-ai@1.18.18" + ], + "exitCode": 0, + "output": "(node:248039) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n(Use `node --trace-deprecation ...` to show where the warning was created)\n\nadded 5 packages in 16s\n" + } + } + ], + "cliPlugins": [ + { + "version": "1.18.18", + "command": { + "command": [ + "/tmp/opencode-host-matrix-FqauE1/host-1-18-18/node_modules/.bin/opencode", + "plugin", + "file:///tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts" + ], + "exitCode": 0, + "output": "\u001b[0m\n┌ Install plugin file:///tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts\n\u001b[?25l│\n◇ Plugin package ready\n\u001b[?25h\u001b[?25l│\n◇ Detected server target\n\u001b[?25h\u001b[?25l│\n◒ Updating plugin config\u001b[999D\u001b[J◇ Plugin config updated\n\u001b[?25h│\n● Added to /tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18/.opencode/opencode.json\n│\n◆ Installed file:///tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts\n│\n● Scope: local (/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18/.opencode)\n│\n└ Done\n\n" + } + } + ] + }, + "routes": [ + { + "route": "cli-install", + "hostVersion": "1.18.18", + "health": { + "healthy": true, + "version": "1.18.18" + }, + "toolIds": [ + "invalid", + "question", + "bash", + "read", + "glob", + "grep", + "edit", + "write", + "task", + "webfetch", + "todowrite", + "websearch", + "skill", + "apply_patch", + "artifact_publish", + "artifact_lifecycle", + "artifact_db", + "artifact_state", + "artifact_comments" + ], + "tools": [ + { + "id": "artifact_publish", + "description": "Render a Markdown document into a single self-contained interactive HTML artifact under .opencode/artifacts/ and return its path. The Markdown may carry frontmatter (title:, icon:) and fenced chart spec blocks (```vega-lite, ```vega, ```echarts). Use it for dashboards, PR walkthroughs, incident timelines, comparisons - anything easier to see as a page than to read as terminal text. Republish with the same title to update the page in place; pass version:true to keep numbered history.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "markdown": { + "description": "Full Markdown source of the artifact page", + "type": "string" + }, + "title": { + "description": "Title override; defaults to frontmatter title", + "type": "string" + }, + "open": { + "description": "Open the artifact in the system browser", + "type": "boolean" + }, + "version": { + "description": "Also keep a numbered version file next to the stable path", + "type": "boolean" + }, + "format": { + "description": "'html' embeds the input as raw trusted HTML instead of rendering Markdown", + "type": "string", + "enum": [ + "markdown", + "html" + ] + }, + "expectedHash": { + "description": "Hash from a previous publish result; publishing fails with a conflict if the artifact changed since", + "type": "string" + }, + "artifact": { + "description": "Exact artifact ID, active slug, contained path, or registered URL for an update", + "type": "string" + }, + "expectedRevision": { + "description": "Expected schema-2 head revision for an update", + "type": "number" + }, + "force": { + "description": "Publish even when the sensitive-content scan finds credential-looking strings", + "type": "boolean" + }, + "dataSources": { + "description": "Named read-only shell commands the served page may poll via opencodeArtifacts.data(name) (raw-HTML pages)", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "command" + ] + } + }, + "deploy": { + "description": "Also push the artifact to a hosted site (requires repo or workerName)", + "type": "boolean" + }, + "repo": { + "description": "GitHub Pages target as owner/name; created public if missing", + "type": "string" + }, + "target": { + "description": "Deploy target; defaults to github when repo is set", + "type": "string", + "enum": [ + "github", + "cloudflare" + ] + }, + "workerName": { + "description": "Cloudflare Worker name (target cloudflare)", + "type": "string" + } + }, + "required": [ + "markdown" + ] + } + }, + { + "id": "artifact_lifecycle", + "description": "List, inspect, read, reopen, restore, archive/unarchive, export, or import schema-2 artifacts by exact identity/reference. Archive is recoverable and requires a preview-bound permission confirmation.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": [ + "list", + "status", + "read", + "reopen", + "restore", + "archive-preview", + "archive-confirm", + "unarchive", + "export", + "import" + ] + }, + "artifact": { + "description": "Exact artifact reference or opaque ID", + "type": "string" + }, + "revision": { + "description": "Revision to read or restore", + "type": "number" + }, + "expectedRevision": { + "description": "Expected current head for restore", + "type": "number" + }, + "token": { + "description": "One-use archive confirmation token", + "type": "string" + }, + "slug": { + "description": "Explicit non-conflicting slug for unarchive", + "type": "string" + }, + "path": { + "description": "Export destination or import bundle directory", + "type": "string" + } + }, + "required": [ + "op" + ] + } + }, + { + "id": "artifact_db", + "description": "Read or write an artifact's shared mini-database (collections of JSON documents, stored under .opencode/artifacts/.db/). Mirrors what the served page can do through the opencodeArtifacts.db bridge.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "type": "string" + }, + "collection": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "get", + "list", + "set", + "delete" + ] + }, + "id": { + "description": "Document id (required for get/set/delete)", + "type": "string" + }, + "doc": { + "description": "Document body for set" + }, + "q": { + "description": "Equality filter for list, as field:value", + "type": "string" + }, + "expectedRevision": { + "description": "Current collection revision for set/delete", + "type": "number" + }, + "expectedDocumentHash": { + "description": "Current document SHA-256 for update/delete", + "type": "string" + }, + "createOnly": { + "description": "Require the document ID to be absent", + "type": "boolean" + }, + "operationId": { + "description": "UUID retained across retries of one mutation", + "type": "string" + } + }, + "required": [ + "slug", + "collection", + "op" + ] + } + }, + { + "id": "artifact_state", + "description": "Read the saved decision/state of an artifact published in this worktree (answers a reader gave on the served page, e.g. workshop decisions).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + }, + { + "id": "artifact_comments", + "description": "Read comment threads a reader left on a served artifact page, or resolve a thread after acting on it. Pass digest: true for a compact triage view (unresolved first, oldest unresolved at top).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + }, + "resolveId": { + "description": "Thread id to mark resolved after you have acted on it", + "type": "string" + }, + "digest": { + "description": "Return a compact triage digest instead of raw threads", + "type": "boolean" + }, + "expectedRevision": { + "description": "Current comment-store revision required with resolveId", + "type": "number" + }, + "expectedHash": { + "description": "Current comment-store hash required with resolveId", + "type": "string" + }, + "operationId": { + "description": "UUID retained across retries of one resolve mutation", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + } + ], + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-18T16:19:28.228Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:19:28.271Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:19:28.285Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-18T16:19:28.976Z level=INFO run=72f8c6ab message=\"creating instance\" directory=/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18\ntimestamp=2026-08-18T16:19:28.979Z level=INFO run=72f8c6ab message=fromDirectory directory=/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18\ntimestamp=2026-08-18T16:19:29.038Z level=INFO run=72f8c6ab message=bootstrapping directory=/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18\ntimestamp=2026-08-18T16:19:29.070Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:19:29.087Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:19:29.102Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-18T16:19:29.319Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18/.opencode/opencode.json\ntimestamp=2026-08-18T16:19:29.364Z level=INFO run=72f8c6ab message=loading path=/tmp/opencode-host-matrix-FqauE1/cli-project-1-18-18/.opencode/opencode.jsonc\ntimestamp=2026-08-18T16:20:48.636Z level=INFO run=72f8c6ab message=\"all LSPs are disabled\"\ntimestamp=2026-08-18T16:20:48.640Z level=INFO run=72f8c6ab message=\"all formatters are disabled\"\ntimestamp=2026-08-18T16:20:48.640Z level=INFO run=72f8c6ab message=init\ntimestamp=2026-08-18T16:20:48.673Z level=INFO run=72f8c6ab message=\"shell tool using shell\" shell=/bin/bash\n", + "config": { + "plugin": [ + "file:///tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts" + ] + }, + "effectiveCommand": { + "artifact-reopen": { + "template": "Call artifact_lifecycle with op \"reopen\" and artifact \"$ARGUMENTS\". Require one exact active ID, slug, contained path, or registered URL; do not guess.", + "description": "Reopen an exact artifact reference" + } + } + }, + { + "route": "config-array", + "hostVersion": "1.18.18", + "health": { + "healthy": true, + "version": "1.18.18" + }, + "toolIds": [ + "invalid", + "question", + "bash", + "read", + "glob", + "grep", + "edit", + "write", + "task", + "webfetch", + "todowrite", + "websearch", + "skill", + "apply_patch", + "artifact_publish", + "artifact_lifecycle", + "artifact_db", + "artifact_state", + "artifact_comments" + ], + "tools": [ + { + "id": "artifact_publish", + "description": "Render a Markdown document into a single self-contained interactive HTML artifact under .opencode/artifacts/ and return its path. The Markdown may carry frontmatter (title:, icon:) and fenced chart spec blocks (```vega-lite, ```vega, ```echarts). Use it for dashboards, PR walkthroughs, incident timelines, comparisons - anything easier to see as a page than to read as terminal text. Republish with the same title to update the page in place; pass version:true to keep numbered history.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "markdown": { + "description": "Full Markdown source of the artifact page", + "type": "string" + }, + "title": { + "description": "Title override; defaults to frontmatter title", + "type": "string" + }, + "open": { + "description": "Open the artifact in the system browser", + "type": "boolean" + }, + "version": { + "description": "Also keep a numbered version file next to the stable path", + "type": "boolean" + }, + "format": { + "description": "'html' embeds the input as raw trusted HTML instead of rendering Markdown", + "type": "string", + "enum": [ + "markdown", + "html" + ] + }, + "expectedHash": { + "description": "Hash from a previous publish result; publishing fails with a conflict if the artifact changed since", + "type": "string" + }, + "artifact": { + "description": "Exact artifact ID, active slug, contained path, or registered URL for an update", + "type": "string" + }, + "expectedRevision": { + "description": "Expected schema-2 head revision for an update", + "type": "number" + }, + "force": { + "description": "Publish even when the sensitive-content scan finds credential-looking strings", + "type": "boolean" + }, + "dataSources": { + "description": "Named read-only shell commands the served page may poll via opencodeArtifacts.data(name) (raw-HTML pages)", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "command" + ] + } + }, + "deploy": { + "description": "Also push the artifact to a hosted site (requires repo or workerName)", + "type": "boolean" + }, + "repo": { + "description": "GitHub Pages target as owner/name; created public if missing", + "type": "string" + }, + "target": { + "description": "Deploy target; defaults to github when repo is set", + "type": "string", + "enum": [ + "github", + "cloudflare" + ] + }, + "workerName": { + "description": "Cloudflare Worker name (target cloudflare)", + "type": "string" + } + }, + "required": [ + "markdown" + ] + } + }, + { + "id": "artifact_lifecycle", + "description": "List, inspect, read, reopen, restore, archive/unarchive, export, or import schema-2 artifacts by exact identity/reference. Archive is recoverable and requires a preview-bound permission confirmation.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": [ + "list", + "status", + "read", + "reopen", + "restore", + "archive-preview", + "archive-confirm", + "unarchive", + "export", + "import" + ] + }, + "artifact": { + "description": "Exact artifact reference or opaque ID", + "type": "string" + }, + "revision": { + "description": "Revision to read or restore", + "type": "number" + }, + "expectedRevision": { + "description": "Expected current head for restore", + "type": "number" + }, + "token": { + "description": "One-use archive confirmation token", + "type": "string" + }, + "slug": { + "description": "Explicit non-conflicting slug for unarchive", + "type": "string" + }, + "path": { + "description": "Export destination or import bundle directory", + "type": "string" + } + }, + "required": [ + "op" + ] + } + }, + { + "id": "artifact_db", + "description": "Read or write an artifact's shared mini-database (collections of JSON documents, stored under .opencode/artifacts/.db/). Mirrors what the served page can do through the opencodeArtifacts.db bridge.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "type": "string" + }, + "collection": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "get", + "list", + "set", + "delete" + ] + }, + "id": { + "description": "Document id (required for get/set/delete)", + "type": "string" + }, + "doc": { + "description": "Document body for set" + }, + "q": { + "description": "Equality filter for list, as field:value", + "type": "string" + }, + "expectedRevision": { + "description": "Current collection revision for set/delete", + "type": "number" + }, + "expectedDocumentHash": { + "description": "Current document SHA-256 for update/delete", + "type": "string" + }, + "createOnly": { + "description": "Require the document ID to be absent", + "type": "boolean" + }, + "operationId": { + "description": "UUID retained across retries of one mutation", + "type": "string" + } + }, + "required": [ + "slug", + "collection", + "op" + ] + } + }, + { + "id": "artifact_state", + "description": "Read the saved decision/state of an artifact published in this worktree (answers a reader gave on the served page, e.g. workshop decisions).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + }, + { + "id": "artifact_comments", + "description": "Read comment threads a reader left on a served artifact page, or resolve a thread after acting on it. Pass digest: true for a compact triage view (unresolved first, oldest unresolved at top).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + }, + "resolveId": { + "description": "Thread id to mark resolved after you have acted on it", + "type": "string" + }, + "digest": { + "description": "Return a compact triage digest instead of raw threads", + "type": "boolean" + }, + "expectedRevision": { + "description": "Current comment-store revision required with resolveId", + "type": "number" + }, + "expectedHash": { + "description": "Current comment-store hash required with resolveId", + "type": "string" + }, + "operationId": { + "description": "UUID retained across retries of one resolve mutation", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + } + ], + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-18T16:20:51.528Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:20:51.600Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:20:51.604Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-18T16:20:52.185Z level=INFO run=16865c83 message=\"creating instance\" directory=/tmp/opencode-host-matrix-FqauE1/config-project-1-18-18\ntimestamp=2026-08-18T16:20:52.188Z level=INFO run=16865c83 message=fromDirectory directory=/tmp/opencode-host-matrix-FqauE1/config-project-1-18-18\ntimestamp=2026-08-18T16:20:52.341Z level=INFO run=16865c83 message=bootstrapping directory=/tmp/opencode-host-matrix-FqauE1/config-project-1-18-18\ntimestamp=2026-08-18T16:20:52.372Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:20:52.388Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:20:52.403Z level=INFO run=16865c83 message=loading path=/tmp/opencode-host-matrix-FqauE1/config-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-18T16:21:15.479Z level=INFO run=16865c83 message=\"all LSPs are disabled\"\ntimestamp=2026-08-18T16:21:15.484Z level=INFO run=16865c83 message=\"all formatters are disabled\"\ntimestamp=2026-08-18T16:21:15.484Z level=INFO run=16865c83 message=init\ntimestamp=2026-08-18T16:21:15.517Z level=INFO run=16865c83 message=\"shell tool using shell\" shell=/bin/bash\n", + "config": { + "plugin": [ + "file:///tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts" + ], + "permission": { + "*": "allow", + "artifact_publish": "ask", + "artifact_datasource": "ask", + "artifact_deploy": "deny", + "artifact_audience": "deny" + } + }, + "effectivePermission": { + "*": "allow", + "artifact_publish": "ask", + "artifact_datasource": "ask", + "artifact_deploy": "deny", + "artifact_audience": "deny" + }, + "effectiveCommand": { + "artifact-reopen": { + "template": "Call artifact_lifecycle with op \"reopen\" and artifact \"$ARGUMENTS\". Require one exact active ID, slug, contained path, or registered URL; do not guess.", + "description": "Reopen an exact artifact reference" + } + } + } + ], + "smoke": { + "tool": "artifact_lifecycle", + "operation": "list", + "result": "{\n \"schemaVersion\": 1,\n \"artifacts\": []\n}", + "filesystemUnchanged": true, + "executionBoundary": "exact-packed-module" + }, + "skill": { + "install": { + "command": [ + "/home/ubuntu/.npm/_npx/387698761821791d/node_modules/node/bin/node", + "/tmp/opencode-host-matrix-FqauE1/package/node_modules/opencode-artifacts/dist/cli.js", + "skill", + "install", + "--project" + ], + "exitCode": 0, + "output": "{\n \"schemaVersion\": 1,\n \"status\": \"installed\",\n \"destination\": \"/tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/skills/artifact-pages\",\n \"files\": [\n \"SKILL.md\",\n \"reference/components.md\",\n \"reference/visuals.md\"\n ],\n \"digest\": \"8ba7e0716799b90c5c576eb9c28ff26574de3c1c2c86ff60906761551d208e03\",\n \"removal\": \"Remove manually only after review: /tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/skills/artifact-pages\"\n}\n" + }, + "destination": "/tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/skills/artifact-pages", + "sourcePackageRemoved": true, + "files": [ + { + "path": "SKILL.md", + "sha256": "1f8a6377b312d4f7b3759d897e4901cdbc8c82aeb39c60e788913db3e961a117", + "bytes": 6454 + }, + { + "path": "reference/components.md", + "sha256": "b5f88df20e10709947c76a87724b2d922489b854c7da99d8bcb8cb383b2b598d", + "bytes": 5389 + }, + { + "path": "reference/visuals.md", + "sha256": "0228e8d7f496da912e9ba1102bcccb9a82584c1b1ae84f5a9d5734e4a8dd1b99", + "bytes": 2280 + } + ], + "hosts": [ + { + "hostVersion": "1.18.18", + "name": "artifact-pages", + "description": "Publish session output as interactive artifact pages via the artifact_publish tool (opencode-artifacts plugin). Use proactively whenever a deliverable has an audience — reports, plans, PR walkthroughs, dashboards, incident timelines, comparisons, checklists — or whenever output is easier to see as a page than to read as terminal text. Covers when to publish, page naming, and the comment loop; component schemas live in reference/components.md.", + "location": "/tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/skills/artifact-pages/SKILL.md", + "contentSha256": "86dcbaa3b04513cdc08cb60f715419acf1147f7d52cfddc3759f81b7c77589af", + "contentBytes": 5961, + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-18T16:21:20.652Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:21:20.872Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:21:20.874Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-18T16:21:21.435Z level=INFO run=68dfb329 message=\"creating instance\" directory=/tmp/opencode-host-matrix-FqauE1/skill-project\ntimestamp=2026-08-18T16:21:21.437Z level=INFO run=68dfb329 message=fromDirectory directory=/tmp/opencode-host-matrix-FqauE1/skill-project\ntimestamp=2026-08-18T16:21:21.487Z level=INFO run=68dfb329 message=bootstrapping directory=/tmp/opencode-host-matrix-FqauE1/skill-project\ntimestamp=2026-08-18T16:21:21.521Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-18T16:21:21.535Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-18T16:21:21.536Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-18T16:21:21.994Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/opencode.json\ntimestamp=2026-08-18T16:21:21.995Z level=INFO run=68dfb329 message=loading path=/tmp/opencode-host-matrix-FqauE1/skill-project/.opencode/opencode.jsonc\ntimestamp=2026-08-18T16:21:22.152Z level=INFO run=68dfb329 message=\"all LSPs are disabled\"\ntimestamp=2026-08-18T16:21:22.156Z level=INFO run=68dfb329 message=\"all formatters are disabled\"\ntimestamp=2026-08-18T16:21:22.156Z level=INFO run=68dfb329 message=init\ntimestamp=2026-08-18T16:21:22.258Z level=INFO run=68dfb329 message=init count=2\n" + } + ] + }, + "result": "pass" +} diff --git a/docs/evidence/releases/2026-08-18-local-core-performance-browser.json b/docs/evidence/releases/2026-08-18-local-core-performance-browser.json new file mode 100644 index 0000000..0ac85fa --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-performance-browser.json @@ -0,0 +1,936 @@ +{ + "schemaVersion": 1, + "benchmark": "renderer-browser-v1", + "capturedAt": "2026-08-18T16:49:33.795Z", + "configPath": "benchmarks/renderer/v1/budgets.json", + "configSha256": "bf6961cdb235d513d4e604659bba19cd4fa4f8bbf059998552c5e65cb7a2b90e", + "percentileMethod": "nearest-rank", + "navigationState": "new WebDriver session and browser profile for every cold sample", + "excludedSetup": [ + "Selenium container start", + "artifact rendering", + "local server start", + "session creation before navigation", + "report serialization" + ], + "environment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "browserName": "chrome", + "browserMajor": 151, + "browserVersion": "151.0.7922.108", + "cgroupCpuMax": null, + "cgroupMemoryMax": null, + "constraintSource": "explicit Docker constraints; verify retained docker inspect values" + }, + "expectedEnvironment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "browserName": "chrome", + "browserMajor": 151 + }, + "environmentComparison": { + "comparable": true, + "mismatches": [] + }, + "sampling": { + "cliSamples": 12, + "browserSamples": 7, + "minimumSamples": 5, + "noiseFloorMs": 250, + "maxRelativeP95Spread": 1 + }, + "workloads": { + "no-runtime": { + "fixture": "no-runtime.md", + "fixtureSha256": "53068378d5707429795aaba82f864de5064f139364ec35490104e964bfb2d28b", + "finalBytes": 43589, + "byteBudget": { + "totalBytes": 43589, + "warningBytes": 131072, + "hardBytes": 196608, + "remainingToHardBytes": 153019, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 803.0895549999996, + 623.6341269999994, + 781.9600850000006, + 611.7876059999999, + 692.171749000001, + 691.3762529999985, + 695.225986999998 + ], + "count": 7, + "minMs": 611.7876059999999, + "maxMs": 803.0895549999996, + "p50Ms": 692.171749000001, + "p95Ms": 803.0895549999996, + "relativeP95Spread": 0.16024607499546822, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 218.3835680000002, + 192.44346699999915, + 218.7962860000007, + 285.92852700000003, + 195.4503900000018, + 284.63092400000096, + 202.36449800000264 + ], + "count": 7, + "minMs": 192.44346699999915, + "maxMs": 285.92852700000003, + "p50Ms": 218.3835680000002, + "p95Ms": 285.92852700000003, + "relativeP95Spread": 0.27017983599999934, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 1500, + "p95Ms": 803.0895549999996, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 285.92852700000003, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 803.0895549999996, + "keyboardAdditionalMs": 218.3835680000002, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 623.6341269999994, + "keyboardAdditionalMs": 192.44346699999915, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 781.9600850000006, + "keyboardAdditionalMs": 218.7962860000007, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 611.7876059999999, + "keyboardAdditionalMs": 285.92852700000003, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 692.171749000001, + "keyboardAdditionalMs": 195.4503900000018, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 691.3762529999985, + "keyboardAdditionalMs": 284.63092400000096, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 695.225986999998, + "keyboardAdditionalMs": 202.36449800000264, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 627.9647420000001, + 684.73315, + 625.119201999998, + 675.9820409999993, + 696.6340469999996, + 957.6471930000043, + 731.2594509999981 + ], + "count": 7, + "minMs": 625.119201999998, + "maxMs": 957.6471930000043, + "p50Ms": 684.73315, + "p95Ms": 957.6471930000043, + "relativeP95Spread": 0.3985699290300233, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 137.8352589999995, + 113.90114900000117, + 202.810677999998, + 135.00827700000082, + 204.52616500000295, + 190.732849, + 192.35893999999826 + ], + "count": 7, + "minMs": 113.90114900000117, + "maxMs": 204.52616500000295, + "p50Ms": 190.732849, + "p95Ms": 204.52616500000295, + "relativeP95Spread": 0.05517326400001184, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 3000, + "p95Ms": 957.6471930000043, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 204.52616500000295, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 627.9647420000001, + "keyboardAdditionalMs": 137.8352589999995, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 684.73315, + "keyboardAdditionalMs": 113.90114900000117, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 625.119201999998, + "keyboardAdditionalMs": 202.810677999998, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 675.9820409999993, + "keyboardAdditionalMs": 135.00827700000082, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 696.6340469999996, + "keyboardAdditionalMs": 204.52616500000295, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 957.6471930000043, + "keyboardAdditionalMs": 190.732849, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 731.2594509999981, + "keyboardAdditionalMs": 192.35893999999826, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/no-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + }, + "one-chart": { + "fixture": "one-chart.md", + "fixtureSha256": "9e2db7cd4b9ffd197baeb1eb539df779dadca192002769fedbe361e934192e26", + "finalBytes": 625792, + "byteBudget": { + "totalBytes": 625792, + "warningBytes": 1048576, + "hardBytes": 1572864, + "remainingToHardBytes": 947072, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 1105.9956759999986, + 1092.7354620000042, + 888.5698590000029, + 1504.8547949999993, + 1108.5570589999988, + 1011.3143920000002, + 900.1050630000027 + ], + "count": 7, + "minMs": 888.5698590000029, + "maxMs": 1504.8547949999993, + "p50Ms": 1092.7354620000042, + "p95Ms": 1504.8547949999993, + "relativeP95Spread": 0.3771446496718467, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 191.63979900000413, + 197.04897199999687, + 227.83707599999616, + 154.7862329999989, + 190.07290199999989, + 296.20217799999955, + 209.64614000000438 + ], + "count": 7, + "minMs": 154.7862329999989, + "maxMs": 296.20217799999955, + "p50Ms": 197.04897199999687, + "p95Ms": 296.20217799999955, + "relativeP95Spread": 0.39661282400001074, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 3000, + "p95Ms": 1504.8547949999993, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 296.20217799999955, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 1105.9956759999986, + "keyboardAdditionalMs": 191.63979900000413, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1092.7354620000042, + "keyboardAdditionalMs": 197.04897199999687, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 888.5698590000029, + "keyboardAdditionalMs": 227.83707599999616, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1504.8547949999993, + "keyboardAdditionalMs": 154.7862329999989, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1108.5570589999988, + "keyboardAdditionalMs": 190.07290199999989, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1011.3143920000002, + "keyboardAdditionalMs": 296.20217799999955, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 900.1050630000027, + "keyboardAdditionalMs": 209.64614000000438, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 904.4179509999958, + 832.4521299999979, + 985.6193960000091, + 888.1543499999971, + 837.0037930000108, + 889.2409289999923, + 1004.9327730000077 + ], + "count": 7, + "minMs": 832.4521299999979, + "maxMs": 1004.9327730000077, + "p50Ms": 889.2409289999923, + "p95Ms": 1004.9327730000077, + "relativeP95Spread": 0.13010179831704127, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 109.97065600000496, + 246.77540399999998, + 88.80269399999816, + 119.84139499999583, + 105.2124829999957, + 64.3552090000012, + 118.89324600000691 + ], + "count": 7, + "minMs": 64.3552090000012, + "maxMs": 246.77540399999998, + "p50Ms": 109.97065600000496, + "p95Ms": 246.77540399999998, + "relativeP95Spread": 0.54721899199998, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 6000, + "p95Ms": 1004.9327730000077, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 246.77540399999998, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 904.4179509999958, + "keyboardAdditionalMs": 109.97065600000496, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 832.4521299999979, + "keyboardAdditionalMs": 246.77540399999998, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 985.6193960000091, + "keyboardAdditionalMs": 88.80269399999816, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 888.1543499999971, + "keyboardAdditionalMs": 119.84139499999583, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 837.0037930000108, + "keyboardAdditionalMs": 105.2124829999957, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 889.2409289999923, + "keyboardAdditionalMs": 64.3552090000012, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1004.9327730000077, + "keyboardAdditionalMs": 118.89324600000691, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/one-chart.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + }, + "multi-runtime": { + "fixture": "multi-runtime.md", + "fixtureSha256": "bd9ed6d6f370ccc67fadb6b5d44b9a9c1d9c66107f4336de9be283d31b0e9b14", + "finalBytes": 5314981, + "byteBudget": { + "totalBytes": 5314981, + "warningBytes": 6291456, + "hardBytes": 8388608, + "remainingToHardBytes": 3073627, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 2078.4542159999983, + 1938.8966579999978, + 2232.5470420000056, + 2040.6676899999875, + 2326.539367999998, + 2206.3162529999972, + 2401.5607569999993 + ], + "count": 7, + "minMs": 1938.8966579999978, + "maxMs": 2401.5607569999993, + "p50Ms": 2206.3162529999972, + "p95Ms": 2401.5607569999993, + "relativeP95Spread": 0.08849343503431205, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 133.1156090000004, + 100.54901699999755, + 98.43572700000368, + 103.21113900000637, + 166.35729100000754, + 134.45431899999676, + 104.18895000001066 + ], + "count": 7, + "minMs": 98.43572700000368, + "maxMs": 166.35729100000754, + "p50Ms": 104.18895000001066, + "p95Ms": 166.35729100000754, + "relativeP95Spread": 0.2486733639999875, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 5000, + "p95Ms": 2401.5607569999993, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 166.35729100000754, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 2078.4542159999983, + "keyboardAdditionalMs": 133.1156090000004, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1938.8966579999978, + "keyboardAdditionalMs": 100.54901699999755, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2232.5470420000056, + "keyboardAdditionalMs": 98.43572700000368, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2040.6676899999875, + "keyboardAdditionalMs": 103.21113900000637, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2326.539367999998, + "keyboardAdditionalMs": 166.35729100000754, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2206.3162529999972, + "keyboardAdditionalMs": 134.45431899999676, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2401.5607569999993, + "keyboardAdditionalMs": 104.18895000001066, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 1782.178356000004, + 1666.2166120000038, + 1962.172384999998, + 1910.2926949999965, + 1804.6119869999966, + 1929.0673530000058, + 1820.7095350000018 + ], + "count": 7, + "minMs": 1666.2166120000038, + "maxMs": 1962.172384999998, + "p50Ms": 1820.7095350000018, + "p95Ms": 1962.172384999998, + "relativeP95Spread": 0.07769655031767388, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 128.36076900000626, + 90.9491190000117, + 97.91671599999245, + 101.60277699999278, + 99.66198799999256, + 283.4066230000026, + 93.01564800000051 + ], + "count": 7, + "minMs": 90.9491190000117, + "maxMs": 283.4066230000026, + "p50Ms": 99.66198799999256, + "p95Ms": 283.4066230000026, + "relativeP95Spread": 0.73497854000004, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 10000, + "p95Ms": 1962.172384999998, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 283.4066230000026, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 1782.178356000004, + "keyboardAdditionalMs": 128.36076900000626, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1666.2166120000038, + "keyboardAdditionalMs": 90.9491190000117, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1962.172384999998, + "keyboardAdditionalMs": 97.91671599999245, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1910.2926949999965, + "keyboardAdditionalMs": 101.60277699999278, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1804.6119869999966, + "keyboardAdditionalMs": 99.66198799999256, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1929.0673530000058, + "keyboardAdditionalMs": 283.4066230000026, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1820.7095350000018, + "keyboardAdditionalMs": 93.01564800000051, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:43115/multi-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + } + }, + "pass": true +} diff --git a/docs/evidence/releases/2026-08-18-local-core-performance-cli.json b/docs/evidence/releases/2026-08-18-local-core-performance-cli.json new file mode 100644 index 0000000..578110c --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-performance-cli.json @@ -0,0 +1,278 @@ +{ + "schemaVersion": 1, + "benchmark": "renderer-cli-v1", + "capturedAt": "2026-08-18T16:46:12.976Z", + "configPath": "benchmarks/renderer/v1/budgets.json", + "configSha256": "bf6961cdb235d513d4e604659bba19cd4fa4f8bbf059998552c5e65cb7a2b90e", + "percentileMethod": "nearest-rank", + "dependencyInstall": { + "includedInTiming": false, + "measured": false, + "durationMs": null, + "reason": "dependencies were preinstalled before the timed harness" + }, + "excludedSetup": [ + "dependency installation", + "TypeScript build", + "fixture discovery", + "report serialization" + ], + "environment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "nodeVersion": "v24.19.0", + "cpuModel": "AMD EPYC Processor (with IBPB)", + "visibleLogicalCpus": 10, + "hostVisibleMemoryBytes": 37831692288, + "cgroupCpuMax": "200000 100000", + "cgroupMemoryMax": "4294967296" + }, + "expectedEnvironment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296 + }, + "environmentComparison": { + "comparable": true, + "mismatches": [] + }, + "sampling": { + "cliSamples": 12, + "browserSamples": 7, + "minimumSamples": 5, + "noiseFloorMs": 250, + "maxRelativeP95Spread": 1 + }, + "workloads": { + "no-runtime": { + "fixture": "no-runtime.md", + "fixtureSha256": "53068378d5707429795aaba82f864de5064f139364ec35490104e964bfb2d28b", + "sourceBytes": 655, + "coldSampleMs": 1066.500952, + "allSamples": { + "samplesMs": [ + 1066.500952, + 724.137471, + 693.069694, + 686.145572, + 734.088641, + 716.513461, + 767.125886, + 921.469949, + 725.474374, + 715.65772, + 710.090489, + 669.687537 + ], + "count": 12, + "minMs": 669.687537, + "maxMs": 1066.500952, + "p50Ms": 716.513461, + "p95Ms": 1066.500952, + "relativeP95Spread": 0.4884590591104053, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 724.137471, + 693.069694, + 686.145572, + 734.088641, + 716.513461, + 767.125886, + 921.469949, + 725.474374, + 715.65772, + 710.090489, + 669.687537 + ], + "count": 11, + "minMs": 669.687537, + "maxMs": 921.469949, + "p50Ms": 716.513461, + "p95Ms": 921.469949, + "relativeP95Spread": 0.2860469469951801, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 2000, + "p95Ms": 1066.500952, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 43589, + "runtimeBytes": 0, + "assetBytes": 0, + "shellAndContentBytes": 43589, + "outputSha256": "ada851484b51bae5fe495c0d4d33f72ca3fcdd3faa6cee06c9d25347c72418e4", + "budget": { + "totalBytes": 43589, + "warningBytes": 131072, + "hardBytes": 196608, + "remainingToHardBytes": 153019, + "status": "pass" + } + }, + "pass": true + }, + "one-chart": { + "fixture": "one-chart.md", + "fixtureSha256": "9e2db7cd4b9ffd197baeb1eb539df779dadca192002769fedbe361e934192e26", + "sourceBytes": 664, + "coldSampleMs": 739.021979, + "allSamples": { + "samplesMs": [ + 739.021979, + 771.664168, + 771.334313, + 682.681037, + 733.294415, + 729.289095, + 727.541897, + 733.340928, + 876.944192, + 700.103405, + 787.293666, + 815.490328 + ], + "count": 12, + "minMs": 682.681037, + "maxMs": 876.944192, + "p50Ms": 733.340928, + "p95Ms": 876.944192, + "relativeP95Spread": 0.19582060473788268, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 771.664168, + 771.334313, + 682.681037, + 733.294415, + 729.289095, + 727.541897, + 733.340928, + 876.944192, + 700.103405, + 787.293666, + 815.490328 + ], + "count": 11, + "minMs": 682.681037, + "maxMs": 876.944192, + "p50Ms": 733.340928, + "p95Ms": 876.944192, + "relativeP95Spread": 0.19582060473788268, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 5000, + "p95Ms": 876.944192, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 625792, + "runtimeBytes": 581244, + "assetBytes": 0, + "shellAndContentBytes": 44548, + "outputSha256": "1d2e1462a17504db7be231759b04dcc36a92dd171333c43247f3fa718e72bfce", + "budget": { + "totalBytes": 625792, + "warningBytes": 1048576, + "hardBytes": 1572864, + "remainingToHardBytes": 947072, + "status": "pass" + } + }, + "pass": true + }, + "multi-runtime": { + "fixture": "multi-runtime.md", + "fixtureSha256": "bd9ed6d6f370ccc67fadb6b5d44b9a9c1d9c66107f4336de9be283d31b0e9b14", + "sourceBytes": 1120, + "coldSampleMs": 1028.530433, + "allSamples": { + "samplesMs": [ + 1028.530433, + 882.207026, + 1006.645935, + 927.123867, + 946.474164, + 915.249453, + 929.936947, + 879.844058, + 943.636705, + 925.633738, + 946.819364, + 905.704597 + ], + "count": 12, + "minMs": 879.844058, + "maxMs": 1028.530433, + "p50Ms": 927.123867, + "p95Ms": 1028.530433, + "relativeP95Spread": 0.10937758115119248, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 882.207026, + 1006.645935, + 927.123867, + 946.474164, + 915.249453, + 929.936947, + 879.844058, + 943.636705, + 925.633738, + 946.819364, + 905.704597 + ], + "count": 11, + "minMs": 879.844058, + "maxMs": 1006.645935, + "p50Ms": 927.123867, + "p95Ms": 1006.645935, + "relativeP95Spread": 0.08577286253811864, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 5000, + "p95Ms": 1028.530433, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 5314981, + "runtimeBytes": 5269229, + "assetBytes": 0, + "shellAndContentBytes": 45752, + "outputSha256": "82cae547e5005e9c3fdaa4dec385e1b62f155203550b668bcdab6316a87bf665", + "budget": { + "totalBytes": 5314981, + "warningBytes": 6291456, + "hardBytes": 8388608, + "remainingToHardBytes": 3073627, + "status": "pass" + } + }, + "pass": true + } + }, + "pass": true +} diff --git a/docs/evidence/releases/2026-08-18-local-core-performance.md b/docs/evidence/releases/2026-08-18-local-core-performance.md new file mode 100644 index 0000000..8cca192 --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-performance.md @@ -0,0 +1,25 @@ +# Local-core renderer performance — 2026-08-18 + +Status: **pass for the exact renderer bytes on the reference Linux container profile** + +Environment: Node 24.19.0, Chromium 151, Linux x64, 2 CPU cores, 4 GiB memory. The CLI run used +a constrained `node:24-bookworm` container. The browser run used a constrained +`selenium/standalone-chromium` container, a loopback-only static server, seven fresh browser +profiles per cell, desktop and mobile widths, and reduced motion. + +| Workload | CLI p95 | Final bytes | Desktop useful/keyboard p95 | Mobile useful/keyboard p95 | +|---|---:|---:|---:|---:| +| no runtime | 1,066.5 ms | 43,589 | 803.1 / 285.9 ms | 957.6 / 204.5 ms | +| one chart | 876.9 ms | 625,792 | 1,504.9 / 296.2 ms | 1,004.9 / 246.8 ms | +| multiple runtimes | 1,028.5 ms | 5,314,981 | 2,401.6 / 166.4 ms | 1,962.2 / 283.4 ms | + +All 36 CLI samples and 42 browser samples are retained in the machine reports. Every workload +passes its time and byte budget; every browser sample reached useful content and keyboard-ready +state with no severe console entry, runtime error, or external request. + +- [`CLI distribution`](2026-08-18-local-core-performance-cli.json), SHA-256 + `5322cdf681c49ec4bcbcce5d7adcc7895b31d24c9dfcec7c8adc744d46009fa1`. +- [`Browser distribution`](2026-08-18-local-core-performance-browser.json), SHA-256 + `f52b923391fd653449e97f2947fcbec67f599365d0c62f0e32335057d3d52b20`. + +This is one reproducible Linux/Chromium reference profile, not a supported-platform matrix. diff --git a/docs/evidence/releases/2026-08-18-local-core-status.md b/docs/evidence/releases/2026-08-18-local-core-status.md new file mode 100644 index 0000000..42ec286 --- /dev/null +++ b/docs/evidence/releases/2026-08-18-local-core-status.md @@ -0,0 +1,33 @@ +# Local artifact core certification status — 2026-08-18 + +Decision status: **not eligible; fail-closed record reports refusal** + +The version-1 certification template and bounded state model are implemented. The model permits +`certified` only for one frozen commit/version/tarball/SHA-256/SRI coordinate when all 13 +requirement rows pass with applicable evidence, release/security/support sign-offs are approved, +claims agree with evidence, at least one supported platform is named, and no blocker remains. +Cross-candidate evidence and `not-applicable` waivers for Local artifact core are rejected. + +Current candidate evidence: + +- frozen candidate commit `84f866ed742eeb08ba668115342ef0e5896dead7`, package `0.15.0`, + SHA-256 `f6d5d5dcdd74ac31522ec9a98acfb38a04c1a4a038c09e65d85d7f4813342bf2`; + the clean stable-OpenCode host matrix and exact-candidate CI run `32161443356` pass after + strict npm output and explicit host-bootstrap fixes; +- exact-subject audit, license dispositions, SBOM, dependency signatures/attestations, code, + migration, security/privacy, and constrained renderer performance gates pass. + +Remaining prerequisites are: + +- no manual screen-reader review of the new composition behavior; +- no authorized current Claude same-input benchmark or independent reviewer panel; +- no representative-user first-use/comprehension records; +- no complete Ubuntu/macOS/Windows/browser/mobile support matrix for the certified target; +- no candidate publication signature/registry readback (publication is not authorized), final + public-claim/rollback disposition, or accountable release/security/support sign-offs. + +The local 16-cell Chromium composition result and earlier preview evidence remain narrowly +scoped inputs; neither waives these rows. Running +`npm run quality:certification -- docs/evidence/releases/2026-08-18-local-core-candidate.json` +returns the explicit missing rows and sign-offs. No tag, npm publish, deployment, trusted- +publisher change, or other provider mutation was performed or authorized by this record. diff --git a/docs/evidence/releases/2026-08-19-local-core-candidate.json b/docs/evidence/releases/2026-08-19-local-core-candidate.json new file mode 100644 index 0000000..25f9f5e --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-candidate.json @@ -0,0 +1,197 @@ +{ + "schemaVersion": 1, + "recordId": "local-core-2026-08-19-candidate", + "level": "local-artifact-core", + "decision": "refused", + "candidate": { + "status": "frozen", + "commit": "2908d52ef94bb0f8cf74d7f2be021ada78a0d661", + "version": "0.15.0", + "tarball": "opencode-artifacts-0.15.0.tgz", + "sha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "sri": "sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==", + "corpusVersions": { "pageQuality": "page-quality-v1", "journey": 1 } + }, + "requirements": [ + { + "id": "code-tests", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-ci.md", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Install, build, 258 tests, structural assertions, and exact-candidate evidence generation.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }] + }, + { + "id": "package-host", + "status": "pass", + "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-host-matrix.json", + "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", + "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Clean temporary install, stable OpenCode discovery, tool schemas, read-only packed smoke, and skill discovery.", + "environment": "Linux x64, Node 24, OpenCode stable 1.18.18" + }] + }, + { + "id": "browser-accessibility", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/page-quality/2026-08-19-local-composition.md", "date": "2026-08-19", + "owner": "Goal 5 automated browser run", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Sixteen Chromium desktop/mobile/mode cells; manual screen-reader review remains missing.", + "environment": "Linux x64, Chromium 151" + }] + }, + { + "id": "migrations", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-ci.md", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Schema, lifecycle, migration, rollback, fault-boundary, and transaction tests in the complete suite.", + "environment": "GitHub ubuntu-latest, Node 24" + }] + }, + { + "id": "security-privacy", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-ci.md", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Adversarial renderer, path, permission, secret-scan, CSP, governance, and privacy-policy checks.", + "environment": "GitHub ubuntu-latest, Node 24" + }] + }, + { + "id": "performance", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-performance.md", "date": "2026-08-19", + "owner": "Goal 5 constrained benchmark run", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Three renderer workloads, 36 CLI samples, 42 fresh-profile browser samples, time and byte budgets.", + "environment": "Linux x64, Node 24.19.0, Chromium 151, 2 cores, 4 GiB" + }] + }, + { + "id": "support-matrix", "status": "pending", "applicabilityReason": null, + "evidence": [ + { + "path": "docs/evidence/releases/2026-08-19-local-core-platform-linux.json", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Exact shared tarball clean install, npm-bin render, strict offline CSP, package-tree removal, and byte-identical reopen. Browser, representative-user, and support claims are excluded.", + "environment": "Ubuntu 24 runner image 20260810.271.1, Linux 6.17.0-1022-azure x64, Node 24.19.0" + }, + { + "path": "docs/evidence/releases/2026-08-19-local-core-platform-macos.json", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Exact shared tarball clean install, npm-bin render, strict offline CSP, package-tree removal, and byte-identical reopen. Browser, representative-user, previous-macOS, and support claims are excluded.", + "environment": "macOS 25.5.0 arm64, runner image macos26 20260728.0273.1, Node 24.18.0" + }, + { + "path": "docs/evidence/releases/2026-08-19-local-core-platform-windows.json", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Exact shared tarball clean install, npm-bin render, strict offline CSP, package-tree removal, and byte-identical reopen. Windows 11, WSL, browser, representative-user, and support claims are excluded.", + "environment": "Windows Server 2025 10.0.26100 x64, runner image win25-vs2026 20260810.198.2, Node 24.19.0" + } + ] + }, + { + "id": "page-quality", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/page-quality/2026-08-19-benchmark-status.md", "date": "2026-08-19", + "owner": "Goal 5 benchmark status audit", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Accurately records the passing local 16-cell renderer track and zero authorized comparative runs or reviewer distributions.", + "environment": "page-quality-v1 corpus; comparative environment not yet authorized" + }] + }, + { + "id": "first-use", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/journeys/goal-5-status-2026-08-19.md", "date": "2026-08-19", + "owner": "Goal 5 journey status audit", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Accurately records no authorized study owner, zero eligible README-only sessions, and every claimed support cell missing.", + "environment": "journey corpus version 1; human collection not started" + }] + }, + { + "id": "comprehension", "status": "pending", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/journeys/goal-5-status-2026-08-19.md", "date": "2026-08-19", + "owner": "Goal 5 journey status audit", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Accurately records zero of ten required eligible representative-primary comprehension sessions.", + "environment": "journey corpus version 1; human collection not started" + }] + }, + { + "id": "audit-license-sbom", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-ci.md", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Zero-vulnerability audit, 217-entry license inventory/dispositions, 211-component CycloneDX SBOM, dependency signatures and attestations.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }] + }, + { + "id": "integrity-provenance", "status": "pending", "applicabilityReason": null, + "evidence": [ + { + "path": "docs/evidence/releases/2026-08-19-local-core-ci.md", "date": "2026-08-19", + "owner": "GitHub Actions run 32212321648", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Reproducible pack digest and CI in-toto subject; no registry publication/signature/readback for this candidate.", + "environment": "GitHub ubuntu-latest, Node 24, npm latest" + }, + { + "path": "docs/evidence/releases/2026-08-19-local-core-consumer-verification.md", "date": "2026-08-19", + "owner": "Goal 5 consumer-verification run", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Two byte-identical packs, SHA-256, npm shasum, SRI, archive shape, and offline verification procedure; registry readback remains absent.", + "environment": "Linux x64, Node 24.19.0, two isolated npm caches" + } + ] + }, + { + "id": "claims-rollback", "status": "pass", "applicabilityReason": null, + "evidence": [{ + "path": "docs/evidence/releases/2026-08-19-local-core-claims.md", "date": "2026-08-19", + "owner": "Goal 5 claim and rollback audit", "result": "pass", + "candidateSha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "scope": "Exact packaged/public claim surfaces, fail-closed certification, pre-release refusal, and zero-mutation rollback procedure.", + "environment": "Frozen candidate archive and repository policy surfaces" + }] + } + ], + "signoffs": [ + { "role": "release", "status": "pending", "by": null, "at": null }, + { "role": "security", "status": "pending", "by": null, "at": null }, + { "role": "support", "status": "pending", "by": null, "at": null } + ], + "blockers": [ + "Independent corpus ambiguity, neutrality, privacy, and redistribution review is absent.", + "Manual screen-reader review of the new composition behavior is absent.", + "Authorized current Claude same-input runs and retention disposition are absent.", + "Three independent benchmark reviewers are absent.", + "Representative-user first-use and comprehension records are absent.", + "The exact claimed support matrix is incomplete.", + "Candidate registry integrity, signature, and published provenance readback are absent because publication is not authorized.", + "Accountable release, security, and support sign-offs are absent." + ], + "claims": { + "certification": false, + "equalOrBetter": false, + "supportedPlatformIds": [] + }, + "providerMutationCount": 0 +} diff --git a/docs/evidence/releases/2026-08-19-local-core-ci.md b/docs/evidence/releases/2026-08-19-local-core-ci.md new file mode 100644 index 0000000..bd5a2c6 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-ci.md @@ -0,0 +1,62 @@ +# Local artifact core exact-candidate CI — 2026-08-19 + +Status: **pass for automated candidate gates; publication gates remain pending** + +GitHub Actions run: `32212321648`, attempt 1 + +Workflow head: `a9b1b13e97f4f187e9b1d931bf7ef54fb9bd54fc` + +Frozen product candidate commit: `2908d52ef94bb0f8cf74d7f2be021ada78a0d661` + +PR merge commit recorded by CI provenance: `5b952e1540ae999fd300aff9e2e9f33c72c2e96f` + +## Exact package + +| Field | Value | +|---|---| +| Archive | `opencode-artifacts-0.15.0.tgz` | +| SHA-256 | `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd` | +| npm shasum | `9dbcd2f6d05e5984c698c97ec9524f0f21a237ef` | +| SRI | `sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==` | +| Packed / unpacked | 126504 / 543313 bytes | +| Entries | 69 | + +The `verify` job passed install, build, 258 tests, structural assertions, pack-coordinate +validation, candidate provenance, audit, license disposition, SBOM generation, registry +dependency-signature verification, and the clean stable-OpenCode host matrix. + +- vulnerabilities: 0 among 217 dependency entries; +- dependency registry signatures: 212 verified; +- dependency attestations: 22 verified; +- CycloneDX components: 211; and +- packed OpenCode host: exact stable `1.18.18`, with real health, tool/skill discovery, + permission policy, config loading, read-only lifecycle smoke, and package-tree-independent + installed skill checks passing. + +Artifact `exact-candidate-evidence` has ID `9351205020`, archive digest +`sha256:71ee5d2e49474285ee5c28240d090786f6b8b43af25afff67d53b8c1aedf74db`, and expires +2026-09-02. Its in-toto subject is the exact SHA-256 above and its invocation is run +`32212321648`, attempt 1. + +## One tarball across three operating systems + +The platform jobs depend on `verify`, download its retained tarball, and never repack it. +Each installed the same SHA-256 with lifecycle scripts disabled, invoked the installed npm +bin, rendered a portable page with strict `connect-src 'none'`, removed the entire package +tree, and read back byte-identical HTML. + +| Observation | Exact environment | Artifact ID | Result | +|---|---|---:|---| +| Linux | Ubuntu 24 image `20260810.271.1`; Linux `6.17.0-1022-azure`; x64; Node `24.19.0` | `9351218207` | pass | +| macOS | macOS `25.5.0`; image `macos26` `20260728.0273.1`; arm64; Node `24.18.0` | `9351220424` | pass | +| Windows | Windows Server 2025 `10.0.26100`; image `win25-vs2026` `20260810.198.2`; x64; Node `24.19.0` | `9351282347` | pass | + +These are technical observations, not supported cells. They do not cover Windows 11/WSL, +previous macOS, browsers, physical mobile devices, assistive technology, or representative +first-use. The matrix deliberately records all of those exclusions. + +## Boundary + +CI provenance describes the unissued candidate; it is not npm registry provenance. No tag, +publication, registry signature, registry byte readback, deployment, or provider mutation +occurred. Those gates cannot pass until separately authorized publication and verification. diff --git a/docs/evidence/releases/2026-08-19-local-core-claims.md b/docs/evidence/releases/2026-08-19-local-core-claims.md new file mode 100644 index 0000000..622c3f9 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-claims.md @@ -0,0 +1,24 @@ +# Local artifact core candidate: claims and rollback audit — 2026-08-19 + +Result: **pass** + +Candidate SHA-256: `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd` + +The exact archive's packaged README labels the project a public preview, says it is +unsupported and uncertified, reports zero fully supported platform/browser cells, and +disclaims representative-user first-use and comprehension baselines. The support policy, +page-quality benchmark, roadmap, governance record, and new certification record agree: +certification and equal-or-better claims are false, and no supported platform ID is named. + +Focused release-transition coverage plus the complete 258-test CI suite verify fail-closed +behavior. The current certification command returns `refused` while mandatory rows, blockers, +and sign-offs are missing. No tag, npm publish, deployment, or provider mutation occurred. + +Rollback for this pre-release candidate is refusal: keep the public-preview disclosures, +leave certification/support/comparison claims disabled, and do not trigger the tag workflow. +The earlier frozen candidate remains historical evidence; the installed-bin defect caused a +new exact candidate to be frozen rather than relabeling old evidence. Any future contradicting +evidence requires the same amend, refreeze, and candidate-bound rerun. + +This audit does not approve release. Publication still requires explicit authority and final +release, security, and support sign-offs. diff --git a/docs/evidence/releases/2026-08-19-local-core-consumer-verification.md b/docs/evidence/releases/2026-08-19-local-core-consumer-verification.md new file mode 100644 index 0000000..759d760 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-consumer-verification.md @@ -0,0 +1,37 @@ +# Local artifact core candidate: consumer verification — 2026-08-19 + +Result: **pass for local byte verification; registry verification remains pending** + +Frozen product commit: `2908d52ef94bb0f8cf74d7f2be021ada78a0d661` + +| Field | Value | +|---|---| +| Archive | `opencode-artifacts-0.15.0.tgz` | +| SHA-256 | `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd` | +| npm SHA-1 shasum | `9dbcd2f6d05e5984c698c97ec9524f0f21a237ef` | +| npm SRI | `sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==` | +| Packed / unpacked | 126504 / 543313 bytes | +| Entries | 69 | + +Two fresh local packs with isolated npm caches and the archive downloaded from GitHub Actions +run `32212321648` were compared. All three had the SHA-256 above and both `cmp` comparisons +exited zero. The Linux, macOS, and Windows platform jobs then installed that one downloaded +archive; they did not create platform-specific candidates. + +Consumers can verify obtained bytes without running package code: + +```sh +sha256sum opencode-artifacts-0.15.0.tgz +sha1sum opencode-artifacts-0.15.0.tgz +tar -tzf opencode-artifacts-0.15.0.tgz +openssl dgst -sha512 -binary opencode-artifacts-0.15.0.tgz | base64 +``` + +The first two outputs must match the coordinates above. Prefix the final output with +`sha512-` and compare the complete SRI. The archive must contain only `package/` paths and the +declared package, README, license, `dist`, `skills`, and `agents` contents. + +This proves reproducible candidate packing and consumer-verifiable bytes. It does not prove +that npm served them, that a registry signature exists, or that published npm provenance +binds them to an authorized release tag. The certification integrity row therefore remains +pending and provider mutation count remains zero. diff --git a/docs/evidence/releases/2026-08-19-local-core-host-matrix.json b/docs/evidence/releases/2026-08-19-local-core-host-matrix.json new file mode 100644 index 0000000..4f23e86 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-host-matrix.json @@ -0,0 +1,774 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-19T03:31:49.442Z", + "candidate": { + "filename": "opencode-artifacts-0.15.0.tgz", + "sha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd", + "packageVersion": "0.15.0" + }, + "compatibility": { + "currentStable": "1.18.18", + "oldestTested": "1.18.18", + "executedVersions": [ + "1.18.18" + ], + "deduplicated": true, + "broaderRangeProven": false, + "v2BetaExcluded": true + }, + "environment": { + "node": "v24.19.0", + "platform": "linux", + "architecture": "x64", + "cleanRoots": [ + "config", + "data", + "cache", + "state" + ], + "providerInference": false + }, + "install": { + "package": { + "command": [ + "npm", + "install", + "--prefix", + "/tmp/opencode-host-matrix-AICHqb/package", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "/home/runner/work/opencode-artifacts/opencode-artifacts/opencode-artifacts-0.15.0.tgz" + ], + "exitCode": 0, + "output": "\nadded 207 packages in 7s\n" + }, + "currentResolution": { + "command": [ + "npm", + "view", + "opencode-ai", + "version", + "--json" + ], + "exitCode": 0, + "output": "[\n \"1.18.18\"\n]\n" + }, + "hosts": [ + { + "version": "1.18.18", + "command": { + "command": [ + "npm", + "install", + "--prefix", + "/tmp/opencode-host-matrix-AICHqb/host-1-18-18", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "opencode-ai@1.18.18" + ], + "exitCode": 0, + "output": "\nadded 3 packages in 5s\n" + }, + "postinstall": { + "command": [ + "/opt/hostedtoolcache/node/24.19.0/x64/bin/node", + "postinstall.mjs" + ], + "exitCode": 0, + "output": "" + } + } + ], + "cliPlugins": [ + { + "version": "1.18.18", + "command": { + "command": [ + "/tmp/opencode-host-matrix-AICHqb/host-1-18-18/node_modules/.bin/opencode", + "plugin", + "file:///tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts" + ], + "exitCode": 0, + "output": "\u001b[0m\n\u001b[90m┌\u001b[39m Install plugin file:///tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts\n\u001b[?25l\u001b[90m│\u001b[39m\n\u001b[32m◇\u001b[39m Plugin package ready\n\u001b[?25h\u001b[?25l\u001b[90m│\u001b[39m\n\u001b[32m◇\u001b[39m Detected server target\n\u001b[?25h\u001b[?25l\u001b[90m│\u001b[39m\n\u001b[32m◇\u001b[39m Plugin config updated\n\u001b[?25h\u001b[90m│\u001b[39m\n\u001b[34m●\u001b[39m Added to /tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18/.opencode/opencode.json\n\u001b[90m│\u001b[39m\n\u001b[32m◆\u001b[39m Installed file:///tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts\n\u001b[90m│\u001b[39m\n\u001b[34m●\u001b[39m Scope: local (/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18/.opencode)\n\u001b[90m│\u001b[39m\n\u001b[90m└\u001b[39m Done\n\n" + } + } + ] + }, + "routes": [ + { + "route": "cli-install", + "hostVersion": "1.18.18", + "health": { + "healthy": true, + "version": "1.18.18" + }, + "toolIds": [ + "invalid", + "question", + "bash", + "read", + "glob", + "grep", + "edit", + "write", + "task", + "webfetch", + "todowrite", + "websearch", + "skill", + "apply_patch", + "artifact_publish", + "artifact_lifecycle", + "artifact_db", + "artifact_state", + "artifact_comments" + ], + "tools": [ + { + "id": "artifact_publish", + "description": "Render a Markdown document into a single self-contained interactive HTML artifact under .opencode/artifacts/ and return its path. The Markdown may carry frontmatter (title:, icon:) and fenced chart spec blocks (```vega-lite, ```vega, ```echarts). Use it for dashboards, PR walkthroughs, incident timelines, comparisons - anything easier to see as a page than to read as terminal text. Republish with the same title to update the page in place; pass version:true to keep numbered history.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "markdown": { + "description": "Full Markdown source of the artifact page", + "type": "string" + }, + "title": { + "description": "Title override; defaults to frontmatter title", + "type": "string" + }, + "open": { + "description": "Open the artifact in the system browser", + "type": "boolean" + }, + "version": { + "description": "Also keep a numbered version file next to the stable path", + "type": "boolean" + }, + "format": { + "description": "'html' embeds the input as raw trusted HTML instead of rendering Markdown", + "type": "string", + "enum": [ + "markdown", + "html" + ] + }, + "expectedHash": { + "description": "Hash from a previous publish result; publishing fails with a conflict if the artifact changed since", + "type": "string" + }, + "artifact": { + "description": "Exact artifact ID, active slug, contained path, or registered URL for an update", + "type": "string" + }, + "expectedRevision": { + "description": "Expected schema-2 head revision for an update", + "type": "number" + }, + "force": { + "description": "Publish even when the sensitive-content scan finds credential-looking strings", + "type": "boolean" + }, + "dataSources": { + "description": "Named read-only shell commands the served page may poll via opencodeArtifacts.data(name) (raw-HTML pages)", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "command" + ] + } + }, + "deploy": { + "description": "Also push the artifact to a hosted site (requires repo or workerName)", + "type": "boolean" + }, + "repo": { + "description": "GitHub Pages target as owner/name; created public if missing", + "type": "string" + }, + "target": { + "description": "Deploy target; defaults to github when repo is set", + "type": "string", + "enum": [ + "github", + "cloudflare" + ] + }, + "workerName": { + "description": "Cloudflare Worker name (target cloudflare)", + "type": "string" + } + }, + "required": [ + "markdown" + ] + } + }, + { + "id": "artifact_lifecycle", + "description": "List, inspect, read, reopen, restore, archive/unarchive, export, or import schema-2 artifacts by exact identity/reference. Archive is recoverable and requires a preview-bound permission confirmation.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": [ + "list", + "status", + "read", + "reopen", + "restore", + "archive-preview", + "archive-confirm", + "unarchive", + "export", + "import" + ] + }, + "artifact": { + "description": "Exact artifact reference or opaque ID", + "type": "string" + }, + "revision": { + "description": "Revision to read or restore", + "type": "number" + }, + "expectedRevision": { + "description": "Expected current head for restore", + "type": "number" + }, + "token": { + "description": "One-use archive confirmation token", + "type": "string" + }, + "slug": { + "description": "Explicit non-conflicting slug for unarchive", + "type": "string" + }, + "path": { + "description": "Export destination or import bundle directory", + "type": "string" + } + }, + "required": [ + "op" + ] + } + }, + { + "id": "artifact_db", + "description": "Read or write an artifact's shared mini-database (collections of JSON documents, stored under .opencode/artifacts/.db/). Mirrors what the served page can do through the opencodeArtifacts.db bridge.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "type": "string" + }, + "collection": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "get", + "list", + "set", + "delete" + ] + }, + "id": { + "description": "Document id (required for get/set/delete)", + "type": "string" + }, + "doc": { + "description": "Document body for set" + }, + "q": { + "description": "Equality filter for list, as field:value", + "type": "string" + }, + "expectedRevision": { + "description": "Current collection revision for set/delete", + "type": "number" + }, + "expectedDocumentHash": { + "description": "Current document SHA-256 for update/delete", + "type": "string" + }, + "createOnly": { + "description": "Require the document ID to be absent", + "type": "boolean" + }, + "operationId": { + "description": "UUID retained across retries of one mutation", + "type": "string" + } + }, + "required": [ + "slug", + "collection", + "op" + ] + } + }, + { + "id": "artifact_state", + "description": "Read the saved decision/state of an artifact published in this worktree (answers a reader gave on the served page, e.g. workshop decisions).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + }, + { + "id": "artifact_comments", + "description": "Read comment threads a reader left on a served artifact page, or resolve a thread after acting on it. Pass digest: true for a compact triage view (unresolved first, oldest unresolved at top).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + }, + "resolveId": { + "description": "Thread id to mark resolved after you have acted on it", + "type": "string" + }, + "digest": { + "description": "Return a compact triage digest instead of raw threads", + "type": "boolean" + }, + "expectedRevision": { + "description": "Current comment-store revision required with resolveId", + "type": "number" + }, + "expectedHash": { + "description": "Current comment-store hash required with resolveId", + "type": "string" + }, + "operationId": { + "description": "UUID retained across retries of one resolve mutation", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + } + ], + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-19T03:30:40.509Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:30:40.514Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:30:40.524Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-19T03:30:40.780Z level=INFO run=24d7bb40 message=\"creating instance\" directory=/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18\ntimestamp=2026-08-19T03:30:40.781Z level=INFO run=24d7bb40 message=fromDirectory directory=/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18\ntimestamp=2026-08-19T03:30:40.790Z level=INFO run=24d7bb40 message=bootstrapping directory=/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18\ntimestamp=2026-08-19T03:30:40.801Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:30:40.801Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:30:40.801Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-19T03:30:40.823Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18/.opencode/opencode.json\ntimestamp=2026-08-19T03:30:40.826Z level=INFO run=24d7bb40 message=loading path=/tmp/opencode-host-matrix-AICHqb/cli-project-1-18-18/.opencode/opencode.jsonc\ntimestamp=2026-08-19T03:31:42.618Z level=INFO run=24d7bb40 message=\"all LSPs are disabled\"\ntimestamp=2026-08-19T03:31:42.620Z level=INFO run=24d7bb40 message=\"all formatters are disabled\"\ntimestamp=2026-08-19T03:31:42.620Z level=INFO run=24d7bb40 message=init\ntimestamp=2026-08-19T03:31:42.633Z level=INFO run=24d7bb40 message=\"shell tool using shell\" shell=/bin/bash\n", + "config": { + "plugin": [ + "file:///tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts" + ] + }, + "effectiveCommand": { + "artifact-reopen": { + "template": "Call artifact_lifecycle with op \"reopen\" and artifact \"$ARGUMENTS\". Require one exact active ID, slug, contained path, or registered URL; do not guess.", + "description": "Reopen an exact artifact reference" + } + } + }, + { + "route": "config-array", + "hostVersion": "1.18.18", + "health": { + "healthy": true, + "version": "1.18.18" + }, + "toolIds": [ + "invalid", + "question", + "bash", + "read", + "glob", + "grep", + "edit", + "write", + "task", + "webfetch", + "todowrite", + "websearch", + "skill", + "apply_patch", + "artifact_publish", + "artifact_lifecycle", + "artifact_db", + "artifact_state", + "artifact_comments" + ], + "tools": [ + { + "id": "artifact_publish", + "description": "Render a Markdown document into a single self-contained interactive HTML artifact under .opencode/artifacts/ and return its path. The Markdown may carry frontmatter (title:, icon:) and fenced chart spec blocks (```vega-lite, ```vega, ```echarts). Use it for dashboards, PR walkthroughs, incident timelines, comparisons - anything easier to see as a page than to read as terminal text. Republish with the same title to update the page in place; pass version:true to keep numbered history.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "markdown": { + "description": "Full Markdown source of the artifact page", + "type": "string" + }, + "title": { + "description": "Title override; defaults to frontmatter title", + "type": "string" + }, + "open": { + "description": "Open the artifact in the system browser", + "type": "boolean" + }, + "version": { + "description": "Also keep a numbered version file next to the stable path", + "type": "boolean" + }, + "format": { + "description": "'html' embeds the input as raw trusted HTML instead of rendering Markdown", + "type": "string", + "enum": [ + "markdown", + "html" + ] + }, + "expectedHash": { + "description": "Hash from a previous publish result; publishing fails with a conflict if the artifact changed since", + "type": "string" + }, + "artifact": { + "description": "Exact artifact ID, active slug, contained path, or registered URL for an update", + "type": "string" + }, + "expectedRevision": { + "description": "Expected schema-2 head revision for an update", + "type": "number" + }, + "force": { + "description": "Publish even when the sensitive-content scan finds credential-looking strings", + "type": "boolean" + }, + "dataSources": { + "description": "Named read-only shell commands the served page may poll via opencodeArtifacts.data(name) (raw-HTML pages)", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "command": { + "type": "string" + }, + "args": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "name", + "command" + ] + } + }, + "deploy": { + "description": "Also push the artifact to a hosted site (requires repo or workerName)", + "type": "boolean" + }, + "repo": { + "description": "GitHub Pages target as owner/name; created public if missing", + "type": "string" + }, + "target": { + "description": "Deploy target; defaults to github when repo is set", + "type": "string", + "enum": [ + "github", + "cloudflare" + ] + }, + "workerName": { + "description": "Cloudflare Worker name (target cloudflare)", + "type": "string" + } + }, + "required": [ + "markdown" + ] + } + }, + { + "id": "artifact_lifecycle", + "description": "List, inspect, read, reopen, restore, archive/unarchive, export, or import schema-2 artifacts by exact identity/reference. Archive is recoverable and requires a preview-bound permission confirmation.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": [ + "list", + "status", + "read", + "reopen", + "restore", + "archive-preview", + "archive-confirm", + "unarchive", + "export", + "import" + ] + }, + "artifact": { + "description": "Exact artifact reference or opaque ID", + "type": "string" + }, + "revision": { + "description": "Revision to read or restore", + "type": "number" + }, + "expectedRevision": { + "description": "Expected current head for restore", + "type": "number" + }, + "token": { + "description": "One-use archive confirmation token", + "type": "string" + }, + "slug": { + "description": "Explicit non-conflicting slug for unarchive", + "type": "string" + }, + "path": { + "description": "Export destination or import bundle directory", + "type": "string" + } + }, + "required": [ + "op" + ] + } + }, + { + "id": "artifact_db", + "description": "Read or write an artifact's shared mini-database (collections of JSON documents, stored under .opencode/artifacts/.db/). Mirrors what the served page can do through the opencodeArtifacts.db bridge.", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "type": "string" + }, + "collection": { + "type": "string" + }, + "op": { + "type": "string", + "enum": [ + "get", + "list", + "set", + "delete" + ] + }, + "id": { + "description": "Document id (required for get/set/delete)", + "type": "string" + }, + "doc": { + "description": "Document body for set" + }, + "q": { + "description": "Equality filter for list, as field:value", + "type": "string" + }, + "expectedRevision": { + "description": "Current collection revision for set/delete", + "type": "number" + }, + "expectedDocumentHash": { + "description": "Current document SHA-256 for update/delete", + "type": "string" + }, + "createOnly": { + "description": "Require the document ID to be absent", + "type": "boolean" + }, + "operationId": { + "description": "UUID retained across retries of one mutation", + "type": "string" + } + }, + "required": [ + "slug", + "collection", + "op" + ] + } + }, + { + "id": "artifact_state", + "description": "Read the saved decision/state of an artifact published in this worktree (answers a reader gave on the served page, e.g. workshop decisions).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + }, + { + "id": "artifact_comments", + "description": "Read comment threads a reader left on a served artifact page, or resolve a thread after acting on it. Pass digest: true for a compact triage view (unresolved first, oldest unresolved at top).", + "parameters": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "slug": { + "description": "Artifact slug (the filename without .html)", + "type": "string" + }, + "resolveId": { + "description": "Thread id to mark resolved after you have acted on it", + "type": "string" + }, + "digest": { + "description": "Return a compact triage digest instead of raw threads", + "type": "boolean" + }, + "expectedRevision": { + "description": "Current comment-store revision required with resolveId", + "type": "number" + }, + "expectedHash": { + "description": "Current comment-store hash required with resolveId", + "type": "string" + }, + "operationId": { + "description": "UUID retained across retries of one resolve mutation", + "type": "string" + } + }, + "required": [ + "slug" + ] + } + } + ], + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-19T03:31:43.745Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:31:43.749Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:31:43.752Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-19T03:31:44.009Z level=INFO run=8bf4613a message=\"creating instance\" directory=/tmp/opencode-host-matrix-AICHqb/config-project-1-18-18\ntimestamp=2026-08-19T03:31:44.010Z level=INFO run=8bf4613a message=fromDirectory directory=/tmp/opencode-host-matrix-AICHqb/config-project-1-18-18\ntimestamp=2026-08-19T03:31:44.019Z level=INFO run=8bf4613a message=bootstrapping directory=/tmp/opencode-host-matrix-AICHqb/config-project-1-18-18\ntimestamp=2026-08-19T03:31:44.033Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:31:44.033Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:31:44.034Z level=INFO run=8bf4613a message=loading path=/tmp/opencode-host-matrix-AICHqb/config-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-19T03:31:46.799Z level=INFO run=8bf4613a message=\"all LSPs are disabled\"\ntimestamp=2026-08-19T03:31:46.801Z level=INFO run=8bf4613a message=\"all formatters are disabled\"\ntimestamp=2026-08-19T03:31:46.801Z level=INFO run=8bf4613a message=init\ntimestamp=2026-08-19T03:31:46.814Z level=INFO run=8bf4613a message=\"shell tool using shell\" shell=/bin/bash\n", + "config": { + "plugin": [ + "file:///tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts" + ], + "permission": { + "*": "allow", + "artifact_publish": "ask", + "artifact_datasource": "ask", + "artifact_deploy": "deny", + "artifact_audience": "deny" + } + }, + "effectivePermission": { + "*": "allow", + "artifact_publish": "ask", + "artifact_datasource": "ask", + "artifact_deploy": "deny", + "artifact_audience": "deny" + }, + "effectiveCommand": { + "artifact-reopen": { + "template": "Call artifact_lifecycle with op \"reopen\" and artifact \"$ARGUMENTS\". Require one exact active ID, slug, contained path, or registered URL; do not guess.", + "description": "Reopen an exact artifact reference" + } + } + } + ], + "smoke": { + "tool": "artifact_lifecycle", + "operation": "list", + "result": "{\n \"schemaVersion\": 1,\n \"artifacts\": []\n}", + "filesystemUnchanged": true, + "executionBoundary": "exact-packed-module" + }, + "skill": { + "install": { + "command": [ + "/opt/hostedtoolcache/node/24.19.0/x64/bin/node", + "/tmp/opencode-host-matrix-AICHqb/package/node_modules/opencode-artifacts/dist/cli.js", + "skill", + "install", + "--project" + ], + "exitCode": 0, + "output": "{\n \"schemaVersion\": 1,\n \"status\": \"installed\",\n \"destination\": \"/tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/skills/artifact-pages\",\n \"files\": [\n \"SKILL.md\",\n \"reference/components.md\",\n \"reference/visuals.md\"\n ],\n \"digest\": \"8ba7e0716799b90c5c576eb9c28ff26574de3c1c2c86ff60906761551d208e03\",\n \"removal\": \"Remove manually only after review: /tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/skills/artifact-pages\"\n}\n" + }, + "destination": "/tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/skills/artifact-pages", + "sourcePackageRemoved": true, + "files": [ + { + "path": "SKILL.md", + "sha256": "1f8a6377b312d4f7b3759d897e4901cdbc8c82aeb39c60e788913db3e961a117", + "bytes": 6454 + }, + { + "path": "reference/components.md", + "sha256": "b5f88df20e10709947c76a87724b2d922489b854c7da99d8bcb8cb383b2b598d", + "bytes": 5389 + }, + { + "path": "reference/visuals.md", + "sha256": "0228e8d7f496da912e9ba1102bcccb9a82584c1b1ae84f5a9d5734e4a8dd1b99", + "bytes": 2280 + } + ], + "hosts": [ + { + "hostVersion": "1.18.18", + "name": "artifact-pages", + "description": "Publish session output as interactive artifact pages via the artifact_publish tool (opencode-artifacts plugin). Use proactively whenever a deliverable has an audience — reports, plans, PR walkthroughs, dashboards, incident timelines, comparisons, checklists — or whenever output is easier to see as a page than to read as terminal text. Covers when to publish, page naming, and the comment loop; component schemas live in reference/components.md.", + "location": "/tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/skills/artifact-pages/SKILL.md", + "contentSha256": "86dcbaa3b04513cdc08cb60f715419acf1147f7d52cfddc3759f81b7c77589af", + "contentBytes": 5961, + "logs": "Warning: OPENCODE_SERVER_PASSWORD is not set; server is unsecured.\ntimestamp=2026-08-19T03:31:48.868Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:31:48.871Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:31:48.872Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/opencode.jsonc\nopencode server listening on http://127.0.0.1:4096\ntimestamp=2026-08-19T03:31:49.142Z level=INFO run=9d8190de message=\"creating instance\" directory=/tmp/opencode-host-matrix-AICHqb/skill-project\ntimestamp=2026-08-19T03:31:49.143Z level=INFO run=9d8190de message=fromDirectory directory=/tmp/opencode-host-matrix-AICHqb/skill-project\ntimestamp=2026-08-19T03:31:49.151Z level=INFO run=9d8190de message=bootstrapping directory=/tmp/opencode-host-matrix-AICHqb/skill-project\ntimestamp=2026-08-19T03:31:49.161Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/config.json\ntimestamp=2026-08-19T03:31:49.162Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/opencode.json\ntimestamp=2026-08-19T03:31:49.162Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-env-1-18-18/config/opencode/opencode.jsonc\ntimestamp=2026-08-19T03:31:49.183Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/opencode.json\ntimestamp=2026-08-19T03:31:49.348Z level=INFO run=9d8190de message=loading path=/tmp/opencode-host-matrix-AICHqb/skill-project/.opencode/opencode.jsonc\ntimestamp=2026-08-19T03:31:49.385Z level=INFO run=9d8190de message=\"all LSPs are disabled\"\ntimestamp=2026-08-19T03:31:49.387Z level=INFO run=9d8190de message=\"all formatters are disabled\"\ntimestamp=2026-08-19T03:31:49.387Z level=INFO run=9d8190de message=init\ntimestamp=2026-08-19T03:31:49.404Z level=INFO run=9d8190de message=init count=2\n" + } + ] + }, + "result": "pass" +} diff --git a/docs/evidence/releases/2026-08-19-local-core-performance-browser.json b/docs/evidence/releases/2026-08-19-local-core-performance-browser.json new file mode 100644 index 0000000..c27faf9 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-performance-browser.json @@ -0,0 +1,936 @@ +{ + "schemaVersion": 1, + "benchmark": "renderer-browser-v1", + "capturedAt": "2026-08-19T03:36:34.450Z", + "configPath": "benchmarks/renderer/v1/budgets.json", + "configSha256": "bf6961cdb235d513d4e604659bba19cd4fa4f8bbf059998552c5e65cb7a2b90e", + "percentileMethod": "nearest-rank", + "navigationState": "new WebDriver session and browser profile for every cold sample", + "excludedSetup": [ + "Selenium container start", + "artifact rendering", + "local server start", + "session creation before navigation", + "report serialization" + ], + "environment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "browserName": "chrome", + "browserMajor": 151, + "browserVersion": "151.0.7922.108", + "cgroupCpuMax": null, + "cgroupMemoryMax": null, + "constraintSource": "explicit Docker constraints; verify retained docker inspect values" + }, + "expectedEnvironment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "browserName": "chrome", + "browserMajor": 151 + }, + "environmentComparison": { + "comparable": true, + "mismatches": [] + }, + "sampling": { + "cliSamples": 12, + "browserSamples": 7, + "minimumSamples": 5, + "noiseFloorMs": 250, + "maxRelativeP95Spread": 1 + }, + "workloads": { + "no-runtime": { + "fixture": "no-runtime.md", + "fixtureSha256": "53068378d5707429795aaba82f864de5064f139364ec35490104e964bfb2d28b", + "finalBytes": 43589, + "byteBudget": { + "totalBytes": 43589, + "warningBytes": 131072, + "hardBytes": 196608, + "remainingToHardBytes": 153019, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 814.868242, + 825.4443170000004, + 816.1179900000006, + 792.9135920000008, + 717.0160919999998, + 800.4489919999996, + 887.8407189999998 + ], + "count": 7, + "minMs": 717.0160919999998, + "maxMs": 887.8407189999998, + "p50Ms": 814.868242, + "p95Ms": 887.8407189999998, + "relativeP95Spread": 0.08955125901200577, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 245.751761, + 293.46671100000003, + 392.29269999999997, + 136.22837499999878, + 125.61736900000142, + 187.02679999999964, + 187.2084790000008 + ], + "count": 7, + "minMs": 125.61736900000142, + "maxMs": 392.29269999999997, + "p50Ms": 187.2084790000008, + "p95Ms": 392.29269999999997, + "relativeP95Spread": 0.8203368839999966, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 1500, + "p95Ms": 887.8407189999998, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 392.29269999999997, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 814.868242, + "keyboardAdditionalMs": 245.751761, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 825.4443170000004, + "keyboardAdditionalMs": 293.46671100000003, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 816.1179900000006, + "keyboardAdditionalMs": 392.29269999999997, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 792.9135920000008, + "keyboardAdditionalMs": 136.22837499999878, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 717.0160919999998, + "keyboardAdditionalMs": 125.61736900000142, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 800.4489919999996, + "keyboardAdditionalMs": 187.02679999999964, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 887.8407189999998, + "keyboardAdditionalMs": 187.2084790000008, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 638.6008379999985, + 900.2056139999986, + 689.6672920000019, + 1102.2203850000005, + 808.7035920000017, + 891.6322230000005, + 807.8485670000009 + ], + "count": 7, + "minMs": 638.6008379999985, + "maxMs": 1102.2203850000005, + "p50Ms": 808.7035920000017, + "p95Ms": 1102.2203850000005, + "relativeP95Spread": 0.36294730962441213, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 189.60168700000213, + 114.82374100000015, + 203.91782800000146, + 224.68505500000174, + 128.19467900000018, + 139.24067800000194, + 129.44597000000067 + ], + "count": 7, + "minMs": 114.82374100000015, + "maxMs": 224.68505500000174, + "p50Ms": 139.24067800000194, + "p95Ms": 224.68505500000174, + "relativeP95Spread": 0.34177750799999923, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 3000, + "p95Ms": 1102.2203850000005, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 224.68505500000174, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 638.6008379999985, + "keyboardAdditionalMs": 189.60168700000213, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 900.2056139999986, + "keyboardAdditionalMs": 114.82374100000015, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 689.6672920000019, + "keyboardAdditionalMs": 203.91782800000146, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1102.2203850000005, + "keyboardAdditionalMs": 224.68505500000174, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 808.7035920000017, + "keyboardAdditionalMs": 128.19467900000018, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 891.6322230000005, + "keyboardAdditionalMs": 139.24067800000194, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 807.8485670000009, + "keyboardAdditionalMs": 129.44597000000067, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/no-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + }, + "one-chart": { + "fixture": "one-chart.md", + "fixtureSha256": "9e2db7cd4b9ffd197baeb1eb539df779dadca192002769fedbe361e934192e26", + "finalBytes": 625792, + "byteBudget": { + "totalBytes": 625792, + "warningBytes": 1048576, + "hardBytes": 1572864, + "remainingToHardBytes": 947072, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 983.506097999998, + 1090.2232360000053, + 908.0568800000037, + 1092.3162259999954, + 1005.1924389999986, + 892.5397840000005, + 984.7074560000037 + ], + "count": 7, + "minMs": 892.5397840000005, + "maxMs": 1092.3162259999954, + "p50Ms": 984.7074560000037, + "p95Ms": 1092.3162259999954, + "relativeP95Spread": 0.109279938264214, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 206.47585300000355, + 194.0020250000016, + 219.18885800000135, + 207.2545639999953, + 141.43628800000442, + 130.62385400000494, + 131.31818900000508 + ], + "count": 7, + "minMs": 130.62385400000494, + "maxMs": 219.18885800000135, + "p50Ms": 194.0020250000016, + "p95Ms": 219.18885800000135, + "relativeP95Spread": 0.10074733199999901, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 3000, + "p95Ms": 1092.3162259999954, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 219.18885800000135, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 983.506097999998, + "keyboardAdditionalMs": 206.47585300000355, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1090.2232360000053, + "keyboardAdditionalMs": 194.0020250000016, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 908.0568800000037, + "keyboardAdditionalMs": 219.18885800000135, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1092.3162259999954, + "keyboardAdditionalMs": 207.2545639999953, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1005.1924389999986, + "keyboardAdditionalMs": 141.43628800000442, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 892.5397840000005, + "keyboardAdditionalMs": 130.62385400000494, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 984.7074560000037, + "keyboardAdditionalMs": 131.31818900000508, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 1015.6137519999975, + 974.4916939999966, + 1127.8989629999996, + 989.0962230000005, + 977.3424860000086, + 916.0046670000011, + 1005.9680030000018 + ], + "count": 7, + "minMs": 916.0046670000011, + "maxMs": 1127.8989629999996, + "p50Ms": 989.0962230000005, + "p95Ms": 1127.8989629999996, + "relativeP95Spread": 0.14033289863245096, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 92.44454899999982, + 131.50906399999803, + 128.97129600000335, + 110.32120299999951, + 52.957697999998345, + 132.6656890000013, + 97.47543099999893 + ], + "count": 7, + "minMs": 52.957697999998345, + "maxMs": 132.6656890000013, + "p50Ms": 110.32120299999951, + "p95Ms": 132.6656890000013, + "relativeP95Spread": 0.08937794400000712, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 6000, + "p95Ms": 1127.8989629999996, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 132.6656890000013, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 1015.6137519999975, + "keyboardAdditionalMs": 92.44454899999982, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 974.4916939999966, + "keyboardAdditionalMs": 131.50906399999803, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1127.8989629999996, + "keyboardAdditionalMs": 128.97129600000335, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 989.0962230000005, + "keyboardAdditionalMs": 110.32120299999951, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 977.3424860000086, + "keyboardAdditionalMs": 52.957697999998345, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 916.0046670000011, + "keyboardAdditionalMs": 132.6656890000013, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1005.9680030000018, + "keyboardAdditionalMs": 97.47543099999893, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/one-chart.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + }, + "multi-runtime": { + "fixture": "multi-runtime.md", + "fixtureSha256": "bd9ed6d6f370ccc67fadb6b5d44b9a9c1d9c66107f4336de9be283d31b0e9b14", + "finalBytes": 5314981, + "byteBudget": { + "totalBytes": 5314981, + "warningBytes": 6291456, + "hardBytes": 8388608, + "remainingToHardBytes": 3073627, + "status": "pass" + }, + "cells": { + "desktop": { + "viewport": { + "width": 1440, + "height": 1200 + }, + "usefulContent": { + "samplesMs": [ + 2074.0292920000065, + 1844.337910999995, + 1889.595503000004, + 2028.8588239999954, + 2024.3135059999913, + 2117.014925999989, + 2415.0596680000017 + ], + "count": 7, + "minMs": 1844.337910999995, + "maxMs": 2415.0596680000017, + "p50Ms": 2028.8588239999954, + "p95Ms": 2415.0596680000017, + "relativeP95Spread": 0.19035372960972824, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 118.63073299999814, + 62.772813000003225, + 79.02929799999401, + 109.12392199999886, + 102.27991199999815, + 90.20690099999774, + 115.52422500000102 + ], + "count": 7, + "minMs": 62.772813000003225, + "maxMs": 118.63073299999814, + "p50Ms": 102.27991199999815, + "p95Ms": 118.63073299999814, + "relativeP95Spread": 0.06540328399999998, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 5000, + "p95Ms": 2415.0596680000017, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 1000, + "p95Ms": 118.63073299999814, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 2074.0292920000065, + "keyboardAdditionalMs": 118.63073299999814, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1844.337910999995, + "keyboardAdditionalMs": 62.772813000003225, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1889.595503000004, + "keyboardAdditionalMs": 79.02929799999401, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2028.8588239999954, + "keyboardAdditionalMs": 109.12392199999886, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2024.3135059999913, + "keyboardAdditionalMs": 102.27991199999815, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2117.014925999989, + "keyboardAdditionalMs": 90.20690099999774, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 2415.0596680000017, + "keyboardAdditionalMs": 115.52422500000102, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + }, + "mobile": { + "viewport": { + "width": 390, + "height": 844 + }, + "usefulContent": { + "samplesMs": [ + 1753.5495789999986, + 1702.8365150000027, + 1957.293807000009, + 1835.088533999995, + 1627.9324819999892, + 1839.1292200000025, + 1804.0480559999996 + ], + "count": 7, + "minMs": 1627.9324819999892, + "maxMs": 1957.293807000009, + "p50Ms": 1804.0480559999996, + "p95Ms": 1957.293807000009, + "relativeP95Spread": 0.08494549271586006, + "disposition": "stable", + "reasons": [] + }, + "keyboardAdditional": { + "samplesMs": [ + 82.18647900001088, + 61.83770999999251, + 53.71479700000782, + 88.01712000000407, + 80.24913000001106, + 51.205063000001246, + 65.61028800001077 + ], + "count": 7, + "minMs": 51.205063000001246, + "maxMs": 88.01712000000407, + "p50Ms": 65.61028800001077, + "p95Ms": 88.01712000000407, + "relativeP95Spread": 0.08962732799997321, + "disposition": "stable", + "reasons": [] + }, + "budget": { + "usefulContent": { + "limitMs": 10000, + "p95Ms": 1957.293807000009, + "pass": true, + "reasons": [] + }, + "keyboardAdditional": { + "limitMs": 2000, + "p95Ms": 88.01712000000407, + "pass": true, + "reasons": [] + }, + "hardFailures": [], + "pass": true + }, + "samples": [ + { + "usefulContentMs": 1753.5495789999986, + "keyboardAdditionalMs": 82.18647900001088, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1702.8365150000027, + "keyboardAdditionalMs": 61.83770999999251, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1957.293807000009, + "keyboardAdditionalMs": 53.71479700000782, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1835.088533999995, + "keyboardAdditionalMs": 88.01712000000407, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1627.9324819999892, + "keyboardAdditionalMs": 80.24913000001106, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1839.1292200000025, + "keyboardAdditionalMs": 51.205063000001246, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + }, + { + "usefulContentMs": 1804.0480559999996, + "keyboardAdditionalMs": 65.61028800001077, + "ready": true, + "keyboardReady": true, + "browserLogs": [], + "requestUrls": [ + "http://127.0.0.1:4173/multi-runtime.html" + ], + "hardFailures": [] + } + ], + "pass": true + } + } + } + }, + "pass": true +} diff --git a/docs/evidence/releases/2026-08-19-local-core-performance-cli.json b/docs/evidence/releases/2026-08-19-local-core-performance-cli.json new file mode 100644 index 0000000..7573a95 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-performance-cli.json @@ -0,0 +1,278 @@ +{ + "schemaVersion": 1, + "benchmark": "renderer-cli-v1", + "capturedAt": "2026-08-19T03:31:22.814Z", + "configPath": "benchmarks/renderer/v1/budgets.json", + "configSha256": "bf6961cdb235d513d4e604659bba19cd4fa4f8bbf059998552c5e65cb7a2b90e", + "percentileMethod": "nearest-rank", + "dependencyInstall": { + "includedInTiming": false, + "measured": false, + "durationMs": null, + "reason": "dependencies were preinstalled before the timed harness" + }, + "excludedSetup": [ + "dependency installation", + "TypeScript build", + "fixture discovery", + "report serialization" + ], + "environment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296, + "nodeVersion": "v24.19.0", + "cpuModel": "AMD EPYC Processor (with IBPB)", + "visibleLogicalCpus": 10, + "hostVisibleMemoryBytes": 37831692288, + "cgroupCpuMax": "200000 100000", + "cgroupMemoryMax": "4294967296" + }, + "expectedEnvironment": { + "profile": "renderer-linux-container-v1", + "platform": "linux", + "arch": "x64", + "nodeMajor": 24, + "cpuQuotaCores": 2, + "memoryLimitBytes": 4294967296 + }, + "environmentComparison": { + "comparable": true, + "mismatches": [] + }, + "sampling": { + "cliSamples": 12, + "browserSamples": 7, + "minimumSamples": 5, + "noiseFloorMs": 250, + "maxRelativeP95Spread": 1 + }, + "workloads": { + "no-runtime": { + "fixture": "no-runtime.md", + "fixtureSha256": "53068378d5707429795aaba82f864de5064f139364ec35490104e964bfb2d28b", + "sourceBytes": 655, + "coldSampleMs": 705.972562, + "allSamples": { + "samplesMs": [ + 705.972562, + 711.004126, + 658.905677, + 736.834527, + 814.037327, + 678.599977, + 727.815364, + 717.366399, + 700.89916, + 751.414901, + 808.634222, + 841.01106 + ], + "count": 12, + "minMs": 658.905677, + "maxMs": 841.01106, + "p50Ms": 717.366399, + "p95Ms": 841.01106, + "relativeP95Spread": 0.1723591475323617, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 711.004126, + 658.905677, + 736.834527, + 814.037327, + 678.599977, + 727.815364, + 717.366399, + 700.89916, + 751.414901, + 808.634222, + 841.01106 + ], + "count": 11, + "minMs": 658.905677, + "maxMs": 841.01106, + "p50Ms": 727.815364, + "p95Ms": 841.01106, + "relativeP95Spread": 0.15552803856446207, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 2000, + "p95Ms": 841.01106, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 43589, + "runtimeBytes": 0, + "assetBytes": 0, + "shellAndContentBytes": 43589, + "outputSha256": "ada851484b51bae5fe495c0d4d33f72ca3fcdd3faa6cee06c9d25347c72418e4", + "budget": { + "totalBytes": 43589, + "warningBytes": 131072, + "hardBytes": 196608, + "remainingToHardBytes": 153019, + "status": "pass" + } + }, + "pass": true + }, + "one-chart": { + "fixture": "one-chart.md", + "fixtureSha256": "9e2db7cd4b9ffd197baeb1eb539df779dadca192002769fedbe361e934192e26", + "sourceBytes": 664, + "coldSampleMs": 770.679741, + "allSamples": { + "samplesMs": [ + 770.679741, + 779.531452, + 785.314665, + 747.546784, + 794.803203, + 839.491337, + 716.260938, + 807.993265, + 724.687316, + 746.482942, + 735.555965, + 759.237563 + ], + "count": 12, + "minMs": 716.260938, + "maxMs": 839.491337, + "p50Ms": 759.237563, + "p95Ms": 839.491337, + "relativeP95Spread": 0.10570311311111989, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 779.531452, + 785.314665, + 747.546784, + 794.803203, + 839.491337, + 716.260938, + 807.993265, + 724.687316, + 746.482942, + 735.555965, + 759.237563 + ], + "count": 11, + "minMs": 716.260938, + "maxMs": 839.491337, + "p50Ms": 759.237563, + "p95Ms": 839.491337, + "relativeP95Spread": 0.10570311311111989, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 5000, + "p95Ms": 839.491337, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 625792, + "runtimeBytes": 581244, + "assetBytes": 0, + "shellAndContentBytes": 44548, + "outputSha256": "1d2e1462a17504db7be231759b04dcc36a92dd171333c43247f3fa718e72bfce", + "budget": { + "totalBytes": 625792, + "warningBytes": 1048576, + "hardBytes": 1572864, + "remainingToHardBytes": 947072, + "status": "pass" + } + }, + "pass": true + }, + "multi-runtime": { + "fixture": "multi-runtime.md", + "fixtureSha256": "bd9ed6d6f370ccc67fadb6b5d44b9a9c1d9c66107f4336de9be283d31b0e9b14", + "sourceBytes": 1120, + "coldSampleMs": 933.558896, + "allSamples": { + "samplesMs": [ + 933.558896, + 855.67667, + 854.782784, + 873.968357, + 899.392954, + 1002.595461, + 846.008819, + 984.282634, + 891.419368, + 946.232175, + 1286.465401, + 940.580466 + ], + "count": 12, + "minMs": 846.008819, + "maxMs": 1286.465401, + "p50Ms": 899.392954, + "p95Ms": 1286.465401, + "relativeP95Spread": 0.4303707798449129, + "disposition": "stable", + "reasons": [] + }, + "warmSamples": { + "samplesMs": [ + 855.67667, + 854.782784, + 873.968357, + 899.392954, + 1002.595461, + 846.008819, + 984.282634, + 891.419368, + 946.232175, + 1286.465401, + 940.580466 + ], + "count": 11, + "minMs": 846.008819, + "maxMs": 1286.465401, + "p50Ms": 899.392954, + "p95Ms": 1286.465401, + "relativeP95Spread": 0.4303707798449129, + "disposition": "stable", + "reasons": [] + }, + "timeBudget": { + "limitMs": 5000, + "p95Ms": 1286.465401, + "pass": true, + "reasons": [] + }, + "bytes": { + "finalBytes": 5314981, + "runtimeBytes": 5269229, + "assetBytes": 0, + "shellAndContentBytes": 45752, + "outputSha256": "82cae547e5005e9c3fdaa4dec385e1b62f155203550b668bcdab6316a87bf665", + "budget": { + "totalBytes": 5314981, + "warningBytes": 6291456, + "hardBytes": 8388608, + "remainingToHardBytes": 3073627, + "status": "pass" + } + }, + "pass": true + } + }, + "pass": true +} diff --git a/docs/evidence/releases/2026-08-19-local-core-performance.md b/docs/evidence/releases/2026-08-19-local-core-performance.md new file mode 100644 index 0000000..369b348 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-performance.md @@ -0,0 +1,29 @@ +# Local-core renderer performance — 2026-08-19 + +Status: **pass for the refrozen exact renderer bytes on the reference Linux profile** + +Candidate SHA-256: `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd` + +Environment: Node 24.19.0, Chromium 151.0.7922.108, Linux x64, two CPU cores, and +4 GiB memory. The CLI and browser runners used `node:24-bookworm`; Chromium used +`selenium/standalone-chromium`. Docker constraints were `NanoCpus=2000000000`, +`Memory=4294967296`, and `ShmSize=2147483648`. The browser harness created seven new +WebDriver profiles per cell and declared the same two-core/4-GiB limits because rootless +container cgroup files were not visible inside the runner. + +| Workload | CLI p95 | Final bytes | Desktop useful/keyboard p95 | Mobile useful/keyboard p95 | +|---|---:|---:|---:|---:| +| no runtime | 841.0 ms | 43,589 | 887.8 / 392.3 ms | 1,102.2 / 224.7 ms | +| one chart | 839.5 ms | 625,792 | 1,092.3 / 219.2 ms | 1,127.9 / 132.7 ms | +| multiple runtimes | 1,286.5 ms | 5,314,981 | 2,415.1 / 118.6 ms | 1,957.3 / 88.0 ms | + +All 36 CLI samples and 42 browser samples passed their time and byte budgets. Every browser +sample reached useful content and keyboard-ready state with no hard failure, severe console +entry, runtime error, or external request. + +- [`CLI distribution`](2026-08-19-local-core-performance-cli.json), SHA-256 + `d4a03ac112c78671a31bbf7df8cd7588b655f813019757d79d09242dc0be9322`. +- [`Browser distribution`](2026-08-19-local-core-performance-browser.json), SHA-256 + `b2667091d8d877261e131e63a4e7ef9f3452f4816461080fc859f96dde9918ab`. + +This is one constrained Linux/Chromium reference profile, not a supported-platform matrix. diff --git a/docs/evidence/releases/2026-08-19-local-core-platform-linux.json b/docs/evidence/releases/2026-08-19-local-core-platform-linux.json new file mode 100644 index 0000000..0899f1b --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-platform-linux.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-19T03:32:28.102Z", + "result": "pass", + "environment": { + "platform": "linux", + "release": "6.17.0-1022-azure", + "architecture": "x64", + "node": "v24.19.0", + "runnerOs": "Linux", + "runnerArchitecture": "X64", + "runnerImageOs": "ubuntu24", + "runnerImageVersion": "20260810.271.1" + }, + "candidate": { + "filename": "opencode-artifacts-0.15.0.tgz", + "integrity": "sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==", + "shasum": "9dbcd2f6d05e5984c698c97ec9524f0f21a237ef", + "size": 126504, + "unpackedSize": 543313, + "entryCount": 69, + "sha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd" + }, + "install": { + "source": "exact-local-tarball", + "lifecycleScripts": false, + "repositorySourceImported": false + }, + "portableOutput": { + "sourceFixture": "benchmarks/renderer/v1/no-runtime.md", + "bytes": 43589, + "sha256": "ada851484b51bae5fe495c0d4d33f72ca3fcdd3faa6cee06c9d25347c72418e4", + "strictOfflineCsp": true, + "packageTreeRemovedBeforeReopen": true, + "byteIdenticalAfterRemoval": true + }, + "claimBoundary": { + "technicalObservationOnly": true, + "supportedPlatform": false, + "browserCoverage": false, + "representativeUserEvidence": false + } +} diff --git a/docs/evidence/releases/2026-08-19-local-core-platform-macos.json b/docs/evidence/releases/2026-08-19-local-core-platform-macos.json new file mode 100644 index 0000000..98512c1 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-platform-macos.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-19T03:32:34.462Z", + "result": "pass", + "environment": { + "platform": "darwin", + "release": "25.5.0", + "architecture": "arm64", + "node": "v24.18.0", + "runnerOs": "macOS", + "runnerArchitecture": "ARM64", + "runnerImageOs": "macos26", + "runnerImageVersion": "20260728.0273.1" + }, + "candidate": { + "filename": "opencode-artifacts-0.15.0.tgz", + "integrity": "sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==", + "shasum": "9dbcd2f6d05e5984c698c97ec9524f0f21a237ef", + "size": 126504, + "unpackedSize": 543313, + "entryCount": 69, + "sha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd" + }, + "install": { + "source": "exact-local-tarball", + "lifecycleScripts": false, + "repositorySourceImported": false + }, + "portableOutput": { + "sourceFixture": "benchmarks/renderer/v1/no-runtime.md", + "bytes": 43589, + "sha256": "ada851484b51bae5fe495c0d4d33f72ca3fcdd3faa6cee06c9d25347c72418e4", + "strictOfflineCsp": true, + "packageTreeRemovedBeforeReopen": true, + "byteIdenticalAfterRemoval": true + }, + "claimBoundary": { + "technicalObservationOnly": true, + "supportedPlatform": false, + "browserCoverage": false, + "representativeUserEvidence": false + } +} diff --git a/docs/evidence/releases/2026-08-19-local-core-platform-windows.json b/docs/evidence/releases/2026-08-19-local-core-platform-windows.json new file mode 100644 index 0000000..4c98d52 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-platform-windows.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-08-19T03:35:42.993Z", + "result": "pass", + "environment": { + "platform": "win32", + "release": "10.0.26100", + "architecture": "x64", + "node": "v24.19.0", + "runnerOs": "Windows", + "runnerArchitecture": "X64", + "runnerImageOs": "win25-vs2026", + "runnerImageVersion": "20260810.198.2" + }, + "candidate": { + "filename": "opencode-artifacts-0.15.0.tgz", + "integrity": "sha512-X9IgvKCHvs1Q5QegNWoBcdId/tIkqlrF0dNSlfSPCCj2Lehrfbl+j4HF2QSVHUNtGFRpGMw6OhDYHLKKARDXfQ==", + "shasum": "9dbcd2f6d05e5984c698c97ec9524f0f21a237ef", + "size": 126504, + "unpackedSize": 543313, + "entryCount": 69, + "sha256": "6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd" + }, + "install": { + "source": "exact-local-tarball", + "lifecycleScripts": false, + "repositorySourceImported": false + }, + "portableOutput": { + "sourceFixture": "benchmarks/renderer/v1/no-runtime.md", + "bytes": 43589, + "sha256": "ada851484b51bae5fe495c0d4d33f72ca3fcdd3faa6cee06c9d25347c72418e4", + "strictOfflineCsp": true, + "packageTreeRemovedBeforeReopen": true, + "byteIdenticalAfterRemoval": true + }, + "claimBoundary": { + "technicalObservationOnly": true, + "supportedPlatform": false, + "browserCoverage": false, + "representativeUserEvidence": false + } +} diff --git a/docs/evidence/releases/2026-08-19-local-core-status.md b/docs/evidence/releases/2026-08-19-local-core-status.md new file mode 100644 index 0000000..382eff6 --- /dev/null +++ b/docs/evidence/releases/2026-08-19-local-core-status.md @@ -0,0 +1,34 @@ +# Local artifact core certification status — 2026-08-19 + +Decision status: **not eligible; fail-closed record reports refusal** + +The first frozen candidate exposed an installed npm-bin entrypoint defect during the new +macOS platform job. The defect was fixed, a new exact candidate was frozen, and every +automated candidate-bound gate was rerun rather than inherited. + +Current candidate: + +- product commit `2908d52ef94bb0f8cf74d7f2be021ada78a0d661`; +- archive `opencode-artifacts-0.15.0.tgz`; +- SHA-256 `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd`; +- GitHub Actions run `32212321648`: 258 tests, build/check, exact pack, audit, + licenses, SBOM, dependency signatures/attestations, provenance subject, and stable + OpenCode host matrix pass; and +- the same tarball passes clean npm-bin render/removal/reopen observations on Linux, macOS, + and Windows, while constrained CLI/browser performance and 16-cell composition also pass. + +Remaining mandatory prerequisites: + +- independent ambiguity, neutrality, privacy, and redistribution review of the frozen corpus; +- fresh manual screen-reader review of the Goal 5 composition; +- authorized current Claude same-input generations and three independent reviewers; +- an authorized study owner and at least 10 eligible representative users; +- the complete release-time OS/browser/device matrix, including required Safari, Firefox, + Windows 11/WSL, previous macOS, and mobile cells; +- separately authorized candidate publication followed by npm integrity, signature, and + provenance readback; and +- accountable release, security, and support sign-offs. + +The exact claim/rollback audit passes, but certification, equal-or-better, and supported- +platform claims remain disabled. No tag, npm publish, deployment, or provider mutation was +performed. diff --git a/docs/evidence/releases/local-core-certification.template.json b/docs/evidence/releases/local-core-certification.template.json new file mode 100644 index 0000000..0681ba0 --- /dev/null +++ b/docs/evidence/releases/local-core-certification.template.json @@ -0,0 +1,47 @@ +{ + "schemaVersion": 1, + "recordId": "local-core-pending", + "level": "local-artifact-core", + "decision": "pending", + "candidate": { + "status": "unfrozen", + "commit": null, + "version": null, + "tarball": null, + "sha256": null, + "sri": null, + "corpusVersions": null + }, + "requirements": [ + { "id": "code-tests", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "package-host", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "browser-accessibility", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "migrations", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "security-privacy", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "performance", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "support-matrix", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "page-quality", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "first-use", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "comprehension", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "audit-license-sbom", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "integrity-provenance", "status": "pending", "applicabilityReason": null, "evidence": [] }, + { "id": "claims-rollback", "status": "pending", "applicabilityReason": null, "evidence": [] } + ], + "signoffs": [ + { "role": "release", "status": "pending", "by": null, "at": null }, + { "role": "security", "status": "pending", "by": null, "at": null }, + { "role": "support", "status": "pending", "by": null, "at": null } + ], + "blockers": [ + "Authorized current Claude same-input runs and retention disposition are absent.", + "Three independent benchmark reviewers are absent.", + "Representative-user first-use and comprehension records are absent.", + "The exact claimed support matrix is incomplete." + ], + "claims": { + "certification": false, + "equalOrBetter": false, + "supportedPlatformIds": [] + }, + "providerMutationCount": 0 +} diff --git a/docs/journeys/README.md b/docs/journeys/README.md index 6f38e9b..7a6b1cf 100644 --- a/docs/journeys/README.md +++ b/docs/journeys/README.md @@ -2,6 +2,9 @@ Status: approved protocol; participant baseline not yet run +The candidate-bound owner/recruitment handoff is ordered in the Goal 5 +[external-evidence execution kit](../evidence/page-quality/goal-5-external-evidence-kit-2026-08-19.md). + This directory owns the versioned create/revise/review/share corpus and the Phase 0 protocols for `OUT-02`, `OUT-03`, `OUT-05`, and `UX-01`. It does not contain participant identities or claim a result before a consented study is complete. diff --git a/docs/journeys/goal-5-participant-materials.md b/docs/journeys/goal-5-participant-materials.md new file mode 100644 index 0000000..3a558d7 --- /dev/null +++ b/docs/journeys/goal-5-participant-materials.md @@ -0,0 +1,62 @@ +# Goal 5 first-use and comprehension materials + +Materials version: 1 + +Status: frozen protocol; do not recruit until an authorized study owner and exact candidate are +recorded + +## Consent script + +“We are evaluating whether a first-time terminal-agent user can create and reopen an offline +artifact from the README, and whether a reader can identify a synthetic artifact's purpose, +primary finding, provenance, and next action. Participation is voluntary. We retain only a +random participant code, categorical eligibility and platform fields, bounded timing/outcome +data, your short answers about synthetic content, rubric scores, and consent/withdrawal times. +We do not collect your name, email, account ID, private repository content, telemetry, audio, +video, or screen recording. Raw JSON stays access-controlled and is deleted 30 days after the +aggregate is accepted, or immediately after a timely withdrawal. Only a non-identifying +aggregate and raw-file digest enter the repository. You may decline or withdraw without losing +product access. Do you consent?” + +A decline creates no record. The facilitator records affirmative consent before exposing the +candidate, README, or timed task. + +## Eligibility and conflict screen + +- Uses a terminal coding agent as an individual developer. +- Has not maintained, contributed to, or reviewed this repository, its corpus, or rubric. +- Has not previously installed or used `opencode-artifacts` when assigned the first-use task. +- Has no other disclosed conflict that would bias the acceptance denominator. + +Ineligible, conflicted, assisted, duplicate, synthetic, and withdrawn records remain visible as +exclusion counts and never enter the denominator. + +## First-use prompt + +“Using only the supplied README and this clean machine, install the exact named package, +create one offline artifact, and reopen the generated file. Tell the facilitator when the +artifact is open.” + +Start timing immediately before showing the README. Stop at success, 600 seconds, repository +checkout, hosting-account use, maintainer help, or an unrecoverable failure. Do not coach or +diagnose during timing. Record the first failed step even when the run does not pass. + +## Comprehension prompt + +Open one assigned, hash-bound synthetic fixture and start timing when useful content is visible. +Ask exactly: + +“What is this page for? What is its primary finding or current state? Where did its information +come from? What should happen next?” + +Stop at 60 seconds. Record the concise answers verbatim in the private file, then score all four +fields against `docs/journeys/corpus.json`. Do not offer examples or hints. + +## Acceptance and handling + +Every claimed install-capable support cell needs an eligible README-only first-use pass. The +comprehension denominator requires at least ten eligible representative primary participants; +at least 90% must pass all four fields without assistance within 60 seconds. Do not round, +impute, discard failures, or treat synthetic runs as evidence. Validate private records with +`npm run study -- validate` and retain only the redacted aggregate, raw-file digest, owner, +candidate identity, exclusions, failures, and deletion disposition. diff --git a/docs/page-quality-benchmark.md b/docs/page-quality-benchmark.md index 29d3def..80420d6 100644 --- a/docs/page-quality-benchmark.md +++ b/docs/page-quality-benchmark.md @@ -155,16 +155,20 @@ Artifact design change. ## Current status -The repository has a useful renderer regression corpus under `examples/patterns/` and -`docs/evidence/patterns/`; official visual material remains link-only. The local corpus proves -pattern coverage and catches regressions; it is not a same-input comparative benchmark. - -A visual audit of the closest dashboard pair found a concrete shortfall: the current -OpenCode example renders a small fixed-size chart inside a wide card with a large unintended -blank region, while the official reference uses the canvas for a denser narrative composition -with large product mockups, an annotated chart, metric hierarchy, and varied card treatments. -Our strengths are consistency, readable components, data honesty, themes, offline operation, -and predictable safety. Bespoke composition, responsive visual sizing, and demonstrated -same-task preference remain roadmap work. +`benchmarks/page-quality/v1/` now owns the hash-bound eight-task corpus, frozen benchmark +manifest, and an empty external-run template. The refrozen candidate's +[dated Chromium report](evidence/page-quality/2026-08-19-local-composition.md) covers all 16 +desktop/mobile/mode cells: responsive chart/diagram sizing, task-aware composition, keyboard +traces, runtime errors, overflow, clipping, external requests, useful-content timing, and layout +shift pass the local gate. This resolves the previously observed fixed-size dashboard chart +defect for the normalized fixture. + +That local result is regression evidence, not a same-input comparison. Authorized current +Claude generations, their account/settings/retention protocol, all 24 unselected pairs, and at +least three independent reviewer distributions remain absent. Official visual material stays +link-only and no synthetic test score enters the benchmark denominator. The candidate-bound +[external-evidence execution kit](evidence/page-quality/goal-5-external-evidence-kit-2026-08-19.md) +defines the non-secret authorization record, independent corpus review, seed-bound blinded +packet preparation, reviewer scoring, screen-reader run, study, and support-browser handoff. Current verdict: **equal-or-better page quality is a target, not yet a verified capability.** diff --git a/docs/redistribution-inventory.json b/docs/redistribution-inventory.json index b0787a0..50a8661 100644 --- a/docs/redistribution-inventory.json +++ b/docs/redistribution-inventory.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "reviewedAt": "2026-08-17", + "reviewedAt": "2026-08-18", "repositoryLicense": "MIT", "authoredScopes": [ { "path": ".github/", "provenance": "repository-authored", "license": "MIT" }, @@ -102,6 +102,150 @@ "attribution": "opencode-artifacts contributors", "status": "approved" }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/compare-desktop.png", + "sha256": "e4090b465cb7f55084217071b644cbbd7d542614b13db665f06326f48d6e6f16", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/compare-mobile.png", + "sha256": "c09c66f31d998147defe68930f8ff593c85c2fde85297f147fc17f58329dfd6a", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/dashboard-desktop.png", + "sha256": "2295021a8ec58b533c3b7a70cc2cd9ff6bbd8c3c7c11779f714e565d2f4e01d2", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/dashboard-mobile.png", + "sha256": "bde8a54bba502c6d72fede5efc8287ccde81e2998ae1fe54a8d6bb1c09153e95", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/findings-table-desktop.png", + "sha256": "3e34b09ca4d63a957243a8fdf403beda4c342694086e3b4c85f67ffeb8658135", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/findings-table-mobile.png", + "sha256": "c0157f607aef3309c77b43253c8ef30e08f43fbd3e7578345adc3939edfcea94", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/incident-desktop.png", + "sha256": "e8e449a72bfef2001db37ee4b9a199f0edb899417e5f737cbcfb812bf9b8e438", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/incident-mobile.png", + "sha256": "111a7b582b55b847e4f466916dacf9a9b3f9906b4c3b060e7bf4b098ead8c884", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/interactive-decision-desktop.png", + "sha256": "0c9306f3558a0a831bdc7c1f7964fd079d7c4860eda0110e4bb25506028072bf", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/interactive-decision-mobile.png", + "sha256": "e459fc1e7c953ea02f2f5d81771ff25d1113219acc44a8cbfe54a03c432135ad", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/plan-checklist-desktop.png", + "sha256": "3ec11697f5622a771d81839bb4a4f9cf37b502ec2b34ce7c6139054378982147", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/plan-checklist-mobile.png", + "sha256": "c58b6327c849f6555af59a8c46527cdd3690b8efa014d90ace206cfef0eba3e4", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/pr-walkthrough-desktop.png", + "sha256": "dc5415d05de3b42740721b9a90aba60f4b371a0e96d0a0a68b8613fb7779933a", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/pr-walkthrough-mobile.png", + "sha256": "42fe01fcc0ebce9636377ab30c514b915c66ce5136b4a904a6d49e29a9678266", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/system-explainer-desktop.png", + "sha256": "258fb47572447071d47e5c9070bf4ca5259a5ab1fddd5a3ab6ab7f2b7e840dc1", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, + { + "path": "docs/evidence/page-quality/2026-08-18-local-composition/system-explainer-mobile.png", + "sha256": "67ae7867700e0d2b0a4d3be96590d515a0516378769f037ea1b04109f8bd8c33", + "provenance": "repository-generated browser capture", + "source": "page-quality-v1 repository-authored synthetic fixture rendered in Chromium 151", + "license": "MIT", + "attribution": "opencode-artifacts contributors", + "status": "approved" + }, { "path": "docs/evidence/patterns/compare-layouts.png", "sha256": "8d4847f887462c6953067b0dd4bd2ca5faf692e9ac5d7224f0188225a8639784", diff --git a/docs/roadmap.md b/docs/roadmap.md index e7b0c3a..5568eee 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -47,7 +47,7 @@ Evidence from the repository and clean local host probes: | Stable OpenCode plugin registration | Shipped | [Local host probe](evidence/opencode-host-verification.md): clean OpenCode processes loaded this checkout and the published 0.14.3 package and returned all four artifact tools | | Claude reference audit | Verified with auth boundary | [Claude host probe](evidence/claude-code-host-verification.md): current official Artifact references were audited and native 2.1.233 was installed/inspected; claude.ai publishing remains untested because the host is not signed in | | Markdown/declarative rendering | Shipped | component, Markdown, render, CSP, and browser evidence suites | -| Equal-or-better page quality | Unverified | pattern screenshots exist, but there is no authenticated same-input Claude corpus or blinded benchmark; the current dashboard also exposes fixed-size chart/dead-space composition gaps | +| Equal-or-better page quality | Unverified | the refrozen local renderer track passes all 16 responsive composition cells, but there is no authorized same-input Claude corpus or blinded reviewer benchmark | | Offline self-contained page | Shipped | conditional inlined runtimes, strict on-disk CSP, 15 MiB final-write cap | | Local gallery/live preview | Shipped | manifest/gallery tests and `serve` SSE tests | | Identity and immutable lifecycle | Partial | slug is title-derived; history is opt-in; lock is process-local; writes are not one crash-safe transaction | @@ -213,9 +213,10 @@ OpenCode meets or exceeds Claude's median in every rubric dimension while scorin Status: **Goal 3's Phase 2 correctness track passed on 2026-08-18**. The portable asset, declarative preflight, bounded design-token, accessibility/internationalization, and renderer performance packets are archived with offline browser, keyboard/mobile-width, manual -screen-reader, byte-boundary, and percentile evidence. Phase 2 as a whole remains incomplete: -the corpus expansion and comparative-quality work in items 8–12, including authorized Claude -runs and independent reviewers, belongs to Goal 5 and retains its external-input blockers. +screen-reader, byte-boundary, and percentile evidence. Goal 5's refrozen candidate now passes +the eight-task local responsive-composition track, resolving the prior fixed-size/dead-space +defect. Phase 2 as a whole remains incomplete: independent corpus review, a fresh composition +screen-reader run, authorized Claude runs, and independent benchmark reviewers remain external inputs. ## Phase 3 — Native OpenCode lifecycle diff --git a/docs/support-policy.md b/docs/support-policy.md index 80d2878..245b872 100644 --- a/docs/support-policy.md +++ b/docs/support-policy.md @@ -1,6 +1,6 @@ # Support and release policy -Policy version: 1. Last reviewed: 2026-08-18. +Policy version: 1. Last reviewed: 2026-08-19. This document owns the Phase 0 D-01 platform decision and D-06 release/supply-chain decision. The machine-readable status record is [`governance-policy.json`](governance-policy.json). @@ -10,9 +10,10 @@ The machine-readable status record is [`governance-policy.json`](governance-poli There are currently **no fully supported platform/browser cells**. A dated Ubuntu observation installed an exact CI tarball, discovered all tools through real OpenCode, rendered and reopened an offline page in one Chromium build, and repeated the reopen after package-tree -removal. It did not exercise Firefox, the latest two stable browser generations, or a -consented first-time user following only the README. The target matrix therefore remains -unverified and blocks a production-readiness claim. +removal. A newer Goal 5 diagnostic installed one shared exact tarball and exercised its npm-bin +offline render/removal/reopen path on one Linux, macOS, and Windows CI runner. Neither track +exercised the complete latest-two browser/device matrix or a consented first-time user following +only the README. The target matrix therefore remains unverified and blocks a production-readiness claim. Public preview is nevertheless distributable after its closed technical, security, privacy, package-integrity, and supply-chain gates pass. Public preview is explicitly unsupported and @@ -23,6 +24,7 @@ later release still requires the complete target matrix and OUT-02/OUT-03 eviden | Cell | Target | Current status | Evidence or missing scope | |---|---|---|---| | Ubuntu technical observation | Ubuntu 24.04.4, Node 24.19.0, OpenCode 1.18.18, Chrome for Testing 145.0.7632.6 | Tested, not supported | [Exact packed observation](evidence/governance/ubuntu-packed-observation-2026-08-16.md); Firefox/latest-two/human first-use absent | +| Goal 5 cross-OS CLI observation | Ubuntu 24 runner, macOS 25.5 arm64 runner, Windows Server 2025 runner; Node 24; exact candidate SHA-256 `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd` | Tested, not supported | [Exact CI report](evidence/releases/2026-08-19-local-core-ci.md); browsers/devices, Windows 11/WSL, previous macOS, and human first-use absent | | Ubuntu desktop | Ubuntu 24.04 LTS, Node 24, stable OpenCode 1.x, latest two Chromium/Firefox | Unverified | Exact combined run absent | | macOS desktop | Current and previous macOS, Node 24, stable OpenCode 1.x, latest two Safari/Chromium/Firefox | Unverified | Machines/runs absent | | Windows desktop | Windows 11 native and WSL 2, Node 24, stable OpenCode 1.x, latest two Chromium/Firefox | Unverified | Machines/runs absent | diff --git a/examples/patterns/compare-layouts.md b/examples/patterns/compare-layouts.md index 8689b6d..b8031e0 100644 --- a/examples/patterns/compare-layouts.md +++ b/examples/patterns/compare-layouts.md @@ -1,6 +1,8 @@ --- title: Settings panel — four layout directions icon: 🧭 +composition: split +source: Synthetic design workshop, captured 2026-08-18 --- Four distinctly different layouts for the settings panel, varying density and grouping. Tradeoff under each. diff --git a/examples/patterns/dashboard.md b/examples/patterns/dashboard.md index e413444..251a106 100644 --- a/examples/patterns/dashboard.md +++ b/examples/patterns/dashboard.md @@ -1,6 +1,8 @@ --- title: Deploy failures by service — last 7 days icon: 📊 +composition: dashboard +source: Synthetic deployment telemetry, captured 2026-08-18 --- A week of deploy failures across all services, with the two hotspots called out. @@ -27,7 +29,7 @@ A week of deploy failures across all services, with the two hotspots called out. ]}, "mark": { "type": "bar", "color": "#6d6bd6", "cornerRadiusTopLeft": 4, "cornerRadiusTopRight": 4 }, "encoding": { - "x": { "field": "day", "type": "ordinal", "title": null }, + "x": { "field": "day", "type": "ordinal", "title": null, "sort": null }, "y": { "field": "failures", "type": "quantitative", "title": "failed deploys" } } } diff --git a/examples/patterns/incident.md b/examples/patterns/incident.md index 55ac92b..8030ace 100644 --- a/examples/patterns/incident.md +++ b/examples/patterns/incident.md @@ -1,6 +1,8 @@ --- title: Incident 4172 — Checkout latency spike icon: 🚨 +composition: narrative +source: Synthetic incident log, captured 2026-08-18 --- ```stats diff --git a/examples/patterns/license-audit.md b/examples/patterns/license-audit.md index e3c8c53..4091aac 100644 --- a/examples/patterns/license-audit.md +++ b/examples/patterns/license-audit.md @@ -3,6 +3,7 @@ title: Dependency license audit icon: ⚖️ description: Every production dependency and its license, copyleft flagged source: package-lock.json, 2026-08-15 +composition: dense --- ```stats diff --git a/examples/patterns/pr-walkthrough.md b/examples/patterns/pr-walkthrough.md index 9dc071e..cca1824 100644 --- a/examples/patterns/pr-walkthrough.md +++ b/examples/patterns/pr-walkthrough.md @@ -1,6 +1,8 @@ --- title: PR #412 walkthrough — cache TTL config icon: 🔀 +composition: split +source: Synthetic PR diff, captured 2026-08-18 --- One config change, one default flipped, one test gap. Reviewers should read the findings top to bottom, then the diff. @@ -34,3 +36,7 @@ One config change, one default flipped, one test gap. Reviewers should read the > [!WARNING] > Safe to merge only after the default goes back to 300 and `validate` rejects ttl < 1. The feature flag itself is fine. + +```copy +{ "label": "Copy review verdict", "text": "Request changes on PR #412: restore DEFAULT_TTL_SECONDS to 300 and reject ttlSeconds values below 1. The feature flag itself is acceptable after those fixes." } +``` diff --git a/examples/patterns/release-checklist.md b/examples/patterns/release-checklist.md index e40b511..ac31e89 100644 --- a/examples/patterns/release-checklist.md +++ b/examples/patterns/release-checklist.md @@ -1,6 +1,8 @@ --- title: Release 2.14 checklist icon: ✅ +composition: quiet +source: Synthetic release plan, captured 2026-08-18 --- ```progress diff --git a/examples/patterns/system-explainer.md b/examples/patterns/system-explainer.md new file mode 100644 index 0000000..04bf731 --- /dev/null +++ b/examples/patterns/system-explainer.md @@ -0,0 +1,62 @@ +--- +title: Artifact publication — trust-boundary explainer +icon: 🧭 +description: How source becomes an offline page without granting view-time authority +source: Synthetic architecture bundle, captured 2026-08-18 +composition: full +--- + +The publication path separates authoring input, validation, permission, durable storage, and +viewing. The decisive property is that opening the final HTML grants no network or filesystem +authority. + +## Data flow and boundaries + +```mermaid +%% summary: Markdown and declared local assets pass through bounded preflight before a permission decision; approved content is committed as an immutable revision and rendered into offline HTML that performs no view-time requests. +flowchart TB + A["Markdown + assets"] --> B["Bounded preflight"] + A --> C["Explicit permission"] + B --> D["Atomic revision"] + C --> D + D --> E["Offline HTML"] +``` + +## What crosses each boundary + +```table +{ + "caption": "Publication boundary inventory", + "columns": [ + { "key": "boundary", "label": "Boundary" }, + { "key": "allowed", "label": "Allowed data" }, + { "key": "authority", "label": "Authority added" } + ], + "rows": [ + { "boundary": "Input → preflight", "allowed": "Markdown and explicitly declared contained assets", "authority": "None" }, + { "boundary": "Preflight → permission", "allowed": "Bounded diagnostics and mutation summary", "authority": "None" }, + { "boundary": "Permission → commit", "allowed": "Approved exact artifact mutation", "authority": "Local write only" }, + { "boundary": "Datasource → loopback service", "allowed": "Registered fixed command and bounded captured result", "authority": "Separate execution permission" }, + { "boundary": "Portable HTML → viewer", "allowed": "Embedded page bytes", "authority": "None; connect-src is none" } + ] +} +``` + +## Offline viewer frame + +```frame +{ + "kind": "mockup", + "title": "portable-artifact.html", + "caption": "The viewer receives one embedded HTML file and no network or filesystem authority.", + "content": "Incident 4172\n\nPrimary finding\nCheckout p99 reached 2.6 s after a synchronous fraud check.\n\nData: synthetic incident log · captured 2026-08-18\nNetwork requests: 0", + "annotations": [ + "Finding and provenance remain readable offline", + "The strict CSP leaves connect-src set to none" + ] +} +``` + +```callout +{ "tone": "info", "title": "Reader decision", "body": "Use the portable HTML for offline review. Start the loopback service only when comments, decisions, or an explicitly registered datasource are required." } +``` diff --git a/examples/patterns/tune-controls.md b/examples/patterns/tune-controls.md index cbd7dd4..c4656e7 100644 --- a/examples/patterns/tune-controls.md +++ b/examples/patterns/tune-controls.md @@ -1,6 +1,8 @@ --- title: Tuning playground — wave parameters icon: 🎛️ +composition: full +source: Synthetic tuning inputs, captured 2026-08-18 --- Drag the sliders: the charts update live. When the shape looks right, copy the values back into the session. diff --git a/package.json b/package.json index 0ef00fb..7487cbb 100644 --- a/package.json +++ b/package.json @@ -36,13 +36,17 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "test": "node --test test/*.test.ts", + "test": "node --test --test-concurrency=1 test/*.test.ts", "check": "node scripts/check-repo.ts", "check:links": "node scripts/check-links.ts", "check:governance": "node scripts/governance-policy.ts", "check:licenses": "node scripts/release-integrity.ts licenses package-lock.json docs/license-dispositions.json", "study": "node scripts/journey-study.ts", "spec": "node scripts/spec-workflow.ts", + "quality:corpus": "node scripts/page-quality-corpus.ts", + "quality:local": "node scripts/page-quality-local-report.ts", + "quality:benchmark": "node scripts/page-quality-benchmark.ts", + "quality:certification": "node scripts/local-core-certification.ts", "prepare": "git config core.hooksPath .githooks || true", "prepublishOnly": "npm run build" }, diff --git a/scripts/candidate-platform-smoke.ts b/scripts/candidate-platform-smoke.ts new file mode 100644 index 0000000..0051493 --- /dev/null +++ b/scripts/candidate-platform-smoke.ts @@ -0,0 +1,252 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { arch, platform, release, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +interface PackEntry { + filename: string; + integrity: string; + shasum: string; + size: number; + unpackedSize: number; + entryCount: number; +} + +export interface CandidatePlatformEvidence { + schemaVersion: 1; + generatedAt: string; + result: "pass"; + environment: { + platform: string; + release: string; + architecture: string; + node: string; + runnerOs: string | null; + runnerArchitecture: string | null; + runnerImageOs: string | null; + runnerImageVersion: string | null; + }; + candidate: PackEntry & { sha256: string }; + install: { + source: "exact-local-tarball"; + lifecycleScripts: false; + repositorySourceImported: false; + }; + portableOutput: { + sourceFixture: string; + bytes: number; + sha256: string; + strictOfflineCsp: true; + packageTreeRemovedBeforeReopen: true; + byteIdenticalAfterRemoval: true; + }; + claimBoundary: { + technicalObservationOnly: true; + supportedPlatform: false; + browserCoverage: false; + representativeUserEvidence: false; + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function parsePackEntry(output: string): PackEntry { + const value: unknown = JSON.parse(output); + const item = Array.isArray(value) && value.length === 1 + ? value[0] + : isRecord(value) && Object.values(value).length === 1 + ? Object.values(value)[0] + : undefined; + if (!isRecord(item)) throw new Error("npm pack must return exactly one entry"); + const fields = ["filename", "integrity", "shasum"] as const; + for (const field of fields) { + if (typeof item[field] !== "string" || item[field].length === 0) { + throw new Error(`npm pack entry is missing ${field}`); + } + } + const numbers = ["size", "unpackedSize", "entryCount"] as const; + for (const field of numbers) { + if (!Number.isInteger(item[field]) || Number(item[field]) <= 0) { + throw new Error(`npm pack entry has invalid ${field}`); + } + } + return { + filename: String(item["filename"]), + integrity: String(item["integrity"]), + shasum: String(item["shasum"]), + size: Number(item["size"]), + unpackedSize: Number(item["unpackedSize"]), + entryCount: Number(item["entryCount"]), + }; +} + +export function assertPortableHtml(html: string): void { + const required = [ + "", + "Renderer no-runtime benchmark", + "Content-Security-Policy", + "connect-src 'none'", + "Build results", + "Benchmark interaction", + ]; + for (const marker of required) { + if (!html.includes(marker)) throw new Error(`portable output is missing ${marker}`); + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +function run(command: string, args: string[], cwd: string, env?: NodeJS.ProcessEnv): string { + const result = spawnSync(command, args, { + cwd, + env: { ...process.env, ...env }, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + windowsHide: true, + shell: process.platform === "win32" && command.toLowerCase().endsWith(".cmd"), + }); + if (result.error) throw result.error; + if (result.status !== 0) { + const detail = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); + throw new Error(`${command} ${args.join(" ")} exited ${result.status}: ${detail}`); + } + return result.stdout; +} + +function argument(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +export async function runCandidatePlatformSmoke( + repositoryRoot: string, + outputPath: string, + expectedSha256?: string, + existingCandidate?: { tarballPath: string; packJsonPath: string }, +): Promise { + const work = await mkdtemp(join(tmpdir(), "opencode-platform-smoke-")); + const npm = process.platform === "win32" ? "npm.cmd" : "npm"; + try { + const cache = join(work, "npm-cache"); + const pack = existingCandidate === undefined + ? parsePackEntry(run( + npm, + ["pack", "--json", "--pack-destination", work], + repositoryRoot, + { NPM_CONFIG_CACHE: cache }, + )) + : parsePackEntry(await readFile(existingCandidate.packJsonPath, "utf8")); + const tarball = existingCandidate?.tarballPath ?? join(work, pack.filename); + const tarballBytes = await readFile(tarball); + const candidateSha256 = sha256(tarballBytes); + if (expectedSha256 !== undefined && candidateSha256 !== expectedSha256) { + throw new Error(`candidate SHA-256 ${candidateSha256} does not match ${expectedSha256}`); + } + + const installRoot = join(work, "consumer"); + await mkdir(installRoot); + run( + npm, + ["install", "--prefix", installRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], + work, + { NPM_CONFIG_CACHE: cache }, + ); + + const fixture = resolve(repositoryRoot, "benchmarks/renderer/v1/no-runtime.md"); + const htmlPath = join(work, "portable.html"); + const cli = join( + installRoot, + "node_modules", + ".bin", + process.platform === "win32" ? "opencode-artifacts.cmd" : "opencode-artifacts", + ); + run(cli, ["render", fixture, "-o", htmlPath], work); + + const beforeRemoval = await readFile(htmlPath); + assertPortableHtml(beforeRemoval.toString("utf8")); + const htmlSha256 = sha256(beforeRemoval); + await rm(installRoot, { recursive: true, force: true }); + const afterRemoval = await readFile(htmlPath); + if (sha256(afterRemoval) !== htmlSha256) throw new Error("portable output changed after package removal"); + + const evidence: CandidatePlatformEvidence = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + result: "pass", + environment: { + platform: platform(), + release: release(), + architecture: arch(), + node: process.version, + runnerOs: process.env["RUNNER_OS"] ?? null, + runnerArchitecture: process.env["RUNNER_ARCH"] ?? null, + runnerImageOs: process.env["ImageOS"] ?? null, + runnerImageVersion: process.env["ImageVersion"] ?? null, + }, + candidate: { ...pack, sha256: candidateSha256 }, + install: { + source: "exact-local-tarball", + lifecycleScripts: false, + repositorySourceImported: false, + }, + portableOutput: { + sourceFixture: "benchmarks/renderer/v1/no-runtime.md", + bytes: beforeRemoval.byteLength, + sha256: htmlSha256, + strictOfflineCsp: true, + packageTreeRemovedBeforeReopen: true, + byteIdenticalAfterRemoval: true, + }, + claimBoundary: { + technicalObservationOnly: true, + supportedPlatform: false, + browserCoverage: false, + representativeUserEvidence: false, + }, + }; + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + return evidence; + } finally { + await rm(work, { recursive: true, force: true }); + } +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : ""; +if (invokedPath === resolve(fileURLToPath(import.meta.url))) { + const output = argument("--output"); + const tarball = argument("--tarball"); + const packJson = argument("--pack-json"); + const candidateDirectory = argument("--candidate-dir"); + if (!output || (tarball === undefined) !== (packJson === undefined) || + (candidateDirectory !== undefined && tarball !== undefined)) { + throw new Error("Usage: node scripts/candidate-platform-smoke.ts --output [--expected-sha256 ] [--candidate-dir | --tarball --pack-json ]"); + } + let existingCandidate: { tarballPath: string; packJsonPath: string } | undefined; + if (candidateDirectory !== undefined) { + const directory = resolve(candidateDirectory); + const downloadedPackJson = join(directory, "release-evidence", "pack.json"); + const downloadedPack = parsePackEntry(await readFile(downloadedPackJson, "utf8")); + existingCandidate = { + tarballPath: join(directory, downloadedPack.filename), + packJsonPath: downloadedPackJson, + }; + } else if (tarball !== undefined && packJson !== undefined) { + existingCandidate = { tarballPath: resolve(tarball), packJsonPath: resolve(packJson) }; + } + const evidence = await runCandidatePlatformSmoke( + process.cwd(), + resolve(output), + argument("--expected-sha256"), + existingCandidate, + ); + console.log(JSON.stringify(evidence, null, 2)); +} diff --git a/scripts/local-core-certification-lib.ts b/scripts/local-core-certification-lib.ts new file mode 100644 index 0000000..d50dbdc --- /dev/null +++ b/scripts/local-core-certification-lib.ts @@ -0,0 +1,173 @@ +export const LOCAL_CORE_REQUIREMENTS = [ + "code-tests", + "package-host", + "browser-accessibility", + "migrations", + "security-privacy", + "performance", + "support-matrix", + "page-quality", + "first-use", + "comprehension", + "audit-license-sbom", + "integrity-provenance", + "claims-rollback", +] as const; + +export const LOCAL_CORE_SIGNOFFS = ["release", "security", "support"] as const; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function exactKeys(value: Record, keys: readonly string[], path: string, errors: string[]): void { + for (const key of keys) if (!(key in value)) errors.push(`${path} is missing ${key}`); + for (const key of Object.keys(value)) if (!keys.includes(key)) errors.push(`${path} has unexpected field ${key}`); +} + +function text(value: unknown, path: string, errors: string[], max = 500): value is string { + if (typeof value !== "string" || value.trim() === "" || value.length > max) { + errors.push(`${path} must be non-empty text of at most ${max} characters`); + return false; + } + return true; +} + +function sha256(value: unknown, path: string, errors: string[]): value is string { + if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) { + errors.push(`${path} must be 64 lowercase hexadecimal characters`); + return false; + } + return true; +} + +function timestamp(value: unknown, path: string, errors: string[]): value is string { + if (typeof value !== "string" || Number.isNaN(Date.parse(value))) { + errors.push(`${path} must be an ISO timestamp`); + return false; + } + return true; +} + +const RECORD_KEYS = ["schemaVersion", "recordId", "level", "decision", "candidate", "requirements", "signoffs", "blockers", "claims", "providerMutationCount"]; + +export function validateCertificationRecord(value: unknown): string[] { + const errors: string[] = []; + if (!isRecord(value)) return ["record must be an object"]; + exactKeys(value, RECORD_KEYS, "record", errors); + if (value["schemaVersion"] !== 1) errors.push("record.schemaVersion must be 1"); + text(value["recordId"], "record.recordId", errors, 120); + if (value["level"] !== "local-artifact-core") errors.push("record.level must be local-artifact-core"); + if (!['pending', 'refused', 'certified'].includes(String(value["decision"]))) errors.push("record.decision is invalid"); + if (value["providerMutationCount"] !== 0) errors.push("record.providerMutationCount must remain zero"); + + const candidate = value["candidate"]; + if (!isRecord(candidate)) errors.push("record.candidate must be an object"); + else { + const fields = ["status", "commit", "version", "tarball", "sha256", "sri", "corpusVersions"]; + exactKeys(candidate, fields, "record.candidate", errors); + if (candidate["status"] !== "unfrozen" && candidate["status"] !== "frozen") errors.push("record.candidate.status is invalid"); + if (candidate["status"] === "unfrozen") { + for (const field of fields.slice(1)) if (candidate[field] !== null) errors.push(`record.candidate.${field} must be null while unfrozen`); + } else { + if (typeof candidate["commit"] !== "string" || !/^[a-f0-9]{40}$/.test(candidate["commit"])) errors.push("record.candidate.commit must be a full Git SHA"); + text(candidate["version"], "record.candidate.version", errors, 80); + text(candidate["tarball"], "record.candidate.tarball", errors, 200); + sha256(candidate["sha256"], "record.candidate.sha256", errors); + if (typeof candidate["sri"] !== "string" || !/^sha512-[A-Za-z0-9+/]+=*$/.test(candidate["sri"])) errors.push("record.candidate.sri must be SHA-512 SRI"); + const corpusVersions = candidate["corpusVersions"]; + if (!isRecord(corpusVersions)) errors.push("record.candidate.corpusVersions must be an object"); + else { + exactKeys(corpusVersions, ["pageQuality", "journey"], "record.candidate.corpusVersions", errors); + if (corpusVersions["pageQuality"] !== "page-quality-v1") errors.push("record.candidate.corpusVersions.pageQuality must be page-quality-v1"); + if (corpusVersions["journey"] !== 1) errors.push("record.candidate.corpusVersions.journey must be 1"); + } + } + } + + const requirementIds = new Set(); + if (!Array.isArray(value["requirements"]) || value["requirements"].length !== LOCAL_CORE_REQUIREMENTS.length) errors.push("record.requirements must contain every canonical row exactly once"); + else for (let index = 0; index < value["requirements"].length; index++) { + const row = value["requirements"][index]; + const path = `record.requirements[${index}]`; + if (!isRecord(row)) { errors.push(`${path} must be an object`); continue; } + exactKeys(row, ["id", "status", "applicabilityReason", "evidence"], path, errors); + if (typeof row["id"] !== "string" || !LOCAL_CORE_REQUIREMENTS.includes(row["id"] as typeof LOCAL_CORE_REQUIREMENTS[number])) errors.push(`${path}.id is not canonical`); + else if (requirementIds.has(row["id"])) errors.push(`${path}.id is duplicated`); + else requirementIds.add(row["id"]); + if (!['pending', 'pass', 'fail', 'not-applicable'].includes(String(row["status"]))) errors.push(`${path}.status is invalid`); + if (row["status"] === "not-applicable") errors.push(`${path}.status cannot be not-applicable for Local artifact core`); + if (row["applicabilityReason"] !== null) text(row["applicabilityReason"], `${path}.applicabilityReason`, errors); + if (!Array.isArray(row["evidence"]) || row["evidence"].length > 30) errors.push(`${path}.evidence must be an array of at most 30 records`); + else { + if (row["status"] === "pass" && row["evidence"].length === 0) errors.push(`${path}.evidence is required for pass`); + for (let evidenceIndex = 0; evidenceIndex < row["evidence"].length; evidenceIndex++) { + const evidence = row["evidence"][evidenceIndex]; + const evidencePath = `${path}.evidence[${evidenceIndex}]`; + if (!isRecord(evidence)) { errors.push(`${evidencePath} must be an object`); continue; } + exactKeys(evidence, ["path", "date", "owner", "result", "candidateSha256", "scope", "environment"], evidencePath, errors); + for (const field of ["path", "owner", "scope", "environment"]) text(evidence[field], `${evidencePath}.${field}`, errors); + if (typeof evidence["date"] !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(evidence["date"])) errors.push(`${evidencePath}.date must be an ISO date`); + if (evidence["result"] !== "pass" && evidence["result"] !== "fail") errors.push(`${evidencePath}.result must be pass or fail`); + if (evidence["candidateSha256"] !== "non-byte-bound") sha256(evidence["candidateSha256"], `${evidencePath}.candidateSha256`, errors); + } + } + } + for (const id of LOCAL_CORE_REQUIREMENTS) if (!requirementIds.has(id)) errors.push(`record.requirements is missing ${id}`); + + const signoffRoles = new Set(); + if (!Array.isArray(value["signoffs"]) || value["signoffs"].length !== LOCAL_CORE_SIGNOFFS.length) errors.push("record.signoffs must contain release, security, and support"); + else for (let index = 0; index < value["signoffs"].length; index++) { + const signoff = value["signoffs"][index]; + const path = `record.signoffs[${index}]`; + if (!isRecord(signoff)) { errors.push(`${path} must be an object`); continue; } + exactKeys(signoff, ["role", "status", "by", "at"], path, errors); + if (typeof signoff["role"] !== "string" || !LOCAL_CORE_SIGNOFFS.includes(signoff["role"] as typeof LOCAL_CORE_SIGNOFFS[number])) errors.push(`${path}.role is invalid`); + else if (signoffRoles.has(signoff["role"])) errors.push(`${path}.role is duplicated`); + else signoffRoles.add(signoff["role"]); + if (signoff["status"] !== "pending" && signoff["status"] !== "approved") errors.push(`${path}.status is invalid`); + if (signoff["status"] === "approved") { text(signoff["by"], `${path}.by`, errors, 120); timestamp(signoff["at"], `${path}.at`, errors); } + else if (signoff["by"] !== null || signoff["at"] !== null) errors.push(`${path} pending signoff must not name an approver or time`); + } + + if (!Array.isArray(value["blockers"]) || !value["blockers"].every((item) => typeof item === "string" && item.trim() !== "") || value["blockers"].length > 50) errors.push("record.blockers must be at most 50 non-empty strings"); + const claims = value["claims"]; + if (!isRecord(claims)) errors.push("record.claims must be an object"); + else { + exactKeys(claims, ["certification", "equalOrBetter", "supportedPlatformIds"], "record.claims", errors); + for (const field of ["certification", "equalOrBetter"]) if (typeof claims[field] !== "boolean") errors.push(`record.claims.${field} must be boolean`); + if (!Array.isArray(claims["supportedPlatformIds"]) || !claims["supportedPlatformIds"].every((id) => typeof id === "string" && id.trim() !== "") || new Set(claims["supportedPlatformIds"]).size !== claims["supportedPlatformIds"].length) errors.push("record.claims.supportedPlatformIds must be unique non-empty strings"); + } + return errors; +} + +export function certificationFailures(value: unknown): string[] { + const validation = validateCertificationRecord(value); + if (validation.length > 0) return validation.map((error) => `invalid record: ${error}`); + if (!isRecord(value)) return ["invalid record"]; + const failures: string[] = []; + const candidate = value["candidate"]; + if (!isRecord(candidate) || candidate["status"] !== "frozen") failures.push("candidate is not frozen"); + const candidateSha = isRecord(candidate) && typeof candidate["sha256"] === "string" ? candidate["sha256"] : null; + for (const row of Array.isArray(value["requirements"]) ? value["requirements"].filter(isRecord) : []) { + if (row["status"] !== "pass") failures.push(`requirement is not pass: ${row["id"]}`); + for (const evidence of Array.isArray(row["evidence"]) ? row["evidence"].filter(isRecord) : []) { + if (evidence["result"] !== "pass") failures.push(`evidence is not pass: ${row["id"]}`); + if (evidence["candidateSha256"] !== "non-byte-bound" && evidence["candidateSha256"] !== candidateSha) failures.push(`evidence candidate mismatch: ${row["id"]}`); + } + } + for (const signoff of Array.isArray(value["signoffs"]) ? value["signoffs"].filter(isRecord) : []) if (signoff["status"] !== "approved") failures.push(`signoff is not approved: ${signoff["role"]}`); + if (Array.isArray(value["blockers"]) && value["blockers"].length > 0) failures.push("unresolved blockers remain"); + const claims = value["claims"]; + if (!isRecord(claims) || claims["certification"] !== true) failures.push("certification claim is not enabled"); + if (!isRecord(claims) || claims["equalOrBetter"] !== true) failures.push("equal-or-better claim is not backed by the required page-quality row"); + if (!isRecord(claims) || !Array.isArray(claims["supportedPlatformIds"]) || claims["supportedPlatformIds"].length === 0) failures.push("no supported platform is named"); + if (value["providerMutationCount"] !== 0) failures.push("provider mutation is forbidden"); + return failures; +} + +export function transitionCertification(value: unknown, target: "refused" | "certified"): { decision: "refused" | "certified"; failures: string[]; providerMutations: 0 } { + const failures = certificationFailures(value); + if (target === "certified" && failures.length > 0) return { decision: "refused", failures, providerMutations: 0 }; + return { decision: target, failures: target === "refused" ? failures : [], providerMutations: 0 }; +} diff --git a/scripts/local-core-certification.ts b/scripts/local-core-certification.ts new file mode 100644 index 0000000..299047c --- /dev/null +++ b/scripts/local-core-certification.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { certificationFailures, validateCertificationRecord } from "./local-core-certification-lib.ts"; + +const input = process.argv[2]; +if (!input) { + console.error("Usage: npm run quality:certification -- "); + process.exit(2); +} +const record = JSON.parse(await readFile(resolve(process.cwd(), input), "utf8")) as unknown; +const errors = validateCertificationRecord(record); +if (errors.length > 0) { + for (const error of errors) console.error(`FAIL - ${error}`); + process.exit(1); +} +const failures = certificationFailures(record); +console.log(JSON.stringify({ status: failures.length === 0 ? "eligible" : "refused", failures }, null, 2)); diff --git a/scripts/opencode-host-matrix.ts b/scripts/opencode-host-matrix.ts index a047820..ac84b27 100644 --- a/scripts/opencode-host-matrix.ts +++ b/scripts/opencode-host-matrix.ts @@ -23,6 +23,7 @@ export const OPENCODE_PERMISSION_POLICY = { artifact_deploy: "deny", artifact_audience: "deny", } as const; +export const OPENCODE_HOST_INSTALL_FLAGS = ["--ignore-scripts", "--no-audit", "--no-fund"] as const; interface CommandResult { command: string[]; @@ -76,7 +77,7 @@ export interface MatrixEvidence { install: { package: CommandResult; currentResolution: CommandResult; - hosts: Array<{ version: string; command: CommandResult }>; + hosts: Array<{ version: string; command: CommandResult; postinstall: CommandResult }>; cliPlugins: Array<{ version: string; command: CommandResult }>; }; routes: ServerResult[]; @@ -125,6 +126,18 @@ export function exactStableMatrix(currentStable: string, oldestTested: string): return { versions, deduplicated: versions.length === 1 }; } +export function stableVersionFromNpm(value: unknown): string { + const version = typeof value === "string" + ? value + : Array.isArray(value) && value.length === 1 && typeof value[0] === "string" + ? value[0] + : undefined; + if (version === undefined || !/^\d+\.\d+\.\d+$/.test(version)) { + throw new Error("npm must resolve exactly one stable OpenCode version"); + } + return version; +} + export function parseServerUrl(output: string): string | undefined { return output.match(/opencode server listening on (http:\/\/127\.0\.0\.1:\d+)/)?.[1]; } @@ -436,16 +449,13 @@ export async function runMatrix(tarballInput: string, outputInput: string): Prom await mkdir(packageRoot, { recursive: true }); const packageInstall = await runCommand("npm", ["install", "--prefix", packageRoot, "--ignore-scripts", "--no-audit", "--no-fund", tarball], { cwd: work }); const currentResolution = await runCommand("npm", ["view", "opencode-ai", "version", "--json"], { cwd: work }); - const currentStable = JSON.parse(currentResolution.output.trim()) as unknown; - if (typeof currentStable !== "string" || !/^\d+\.\d+\.\d+$/.test(currentStable)) { - throw new Error(`npm resolved a non-stable OpenCode version: ${currentResolution.output}`); - } + const currentStable = stableVersionFromNpm(JSON.parse(currentResolution.output.trim()) as unknown); const pluginDirectory = join(packageRoot, "node_modules", "opencode-artifacts"); const manifest = JSON.parse(await readFile(join(pluginDirectory, "package.json"), "utf8")) as { version?: unknown }; if (typeof manifest.version !== "string") throw new Error("packed package version is missing"); const pluginUrl = pathToFileURL(pluginDirectory).href; const matrix = exactStableMatrix(currentStable, OLDEST_TESTED_OPENCODE_VERSION); - const hosts: Array<{ version: string; command: CommandResult }> = []; + const hosts: Array<{ version: string; command: CommandResult; postinstall: CommandResult }> = []; const hostBinaries: Array<{ version: string; binary: string }> = []; const cliPlugins: Array<{ version: string; command: CommandResult }> = []; const routes: ServerResult[] = []; @@ -455,8 +465,10 @@ export async function runMatrix(tarballInput: string, outputInput: string): Prom const cliProject = join(work, `cli-project-${versionKey}`); const configProject = join(work, `config-project-${versionKey}`); await Promise.all([hostRoot, cliProject, configProject].map((path) => mkdir(path, { recursive: true }))); - const hostInstall = await runCommand("npm", ["install", "--prefix", hostRoot, "--no-audit", "--no-fund", `opencode-ai@${version}`], { cwd: work }); - hosts.push({ version, command: hostInstall }); + const hostInstall = await runCommand("npm", ["install", "--prefix", hostRoot, ...OPENCODE_HOST_INSTALL_FLAGS, `opencode-ai@${version}`], { cwd: work }); + const hostPackage = join(hostRoot, "node_modules", "opencode-ai"); + const hostPostinstall = await runCommand(process.execPath, ["postinstall.mjs"], { cwd: hostPackage }); + hosts.push({ version, command: hostInstall, postinstall: hostPostinstall }); const hostBinary = join(hostRoot, "node_modules", ".bin", process.platform === "win32" ? "opencode.cmd" : "opencode"); hostBinaries.push({ version, binary: hostBinary }); const cliEnvRoot = join(work, `cli-env-${versionKey}`); diff --git a/scripts/page-quality-benchmark-lib.ts b/scripts/page-quality-benchmark-lib.ts new file mode 100644 index 0000000..71cc094 --- /dev/null +++ b/scripts/page-quality-benchmark-lib.ts @@ -0,0 +1,493 @@ +import { createHash, createHmac } from "node:crypto"; + +export const SYSTEMS = ["opencode", "claude"] as const; +export const DIMENSIONS = [ + "taskOrientation", + "informationHierarchy", + "compositionDensity", + "typographyReadability", + "visualEncoding", + "coherenceCraft", + "interactionQuality", + "responsiveAdaptation", +] as const; +export const HARD_GATES = [ + "facts", + "interactions", + "layout", + "composition", + "accessibility", + "dataHonesty", + "offlineBudget", +] as const; + +type System = typeof SYSTEMS[number]; +type Dimension = typeof DIMENSIONS[number]; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function exactKeys(value: Record, keys: readonly string[], path: string, errors: string[]): void { + for (const key of keys) if (!(key in value)) errors.push(`${path} is missing ${key}`); + for (const key of Object.keys(value)) if (!keys.includes(key)) errors.push(`${path} has unexpected field ${key}`); +} + +function text(value: unknown, path: string, errors: string[], max = 500): value is string { + if (typeof value !== "string" || value.trim() === "" || value.length > max) { + errors.push(`${path} must be non-empty text of at most ${max} characters`); + return false; + } + return true; +} + +function sha256(value: unknown, path: string, errors: string[]): value is string { + if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) { + errors.push(`${path} must be 64 lowercase hexadecimal characters`); + return false; + } + return true; +} + +function isoTimestamp(value: unknown, path: string, errors: string[]): value is string { + if (typeof value !== "string" || Number.isNaN(Date.parse(value))) { + errors.push(`${path} must be an ISO timestamp`); + return false; + } + return true; +} + +function manifestTaskIds(manifest: unknown): string[] { + if (!isRecord(manifest) || !Array.isArray(manifest["taskIds"])) return []; + return manifest["taskIds"].filter((item): item is string => typeof item === "string"); +} + +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (!isRecord(value)) return JSON.stringify(value); + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; +} + +export function manifestDigest(manifest: unknown): string { + return createHash("sha256").update(canonicalJson(manifest)).digest("hex"); +} + +export function validateBenchmarkManifest(value: unknown): string[] { + const errors: string[] = []; + if (!isRecord(value)) return ["manifest must be an object"]; + exactKeys(value, ["schemaVersion", "corpusId", "systems", "taskIds", "minimumRunsPerSystemTask", "captureModes", "hardGates", "dimensions", "thresholds"], "manifest", errors); + if (value["schemaVersion"] !== 1) errors.push("manifest.schemaVersion must be 1"); + if (value["corpusId"] !== "page-quality-v1") errors.push("manifest.corpusId must be page-quality-v1"); + if (JSON.stringify(value["systems"]) !== JSON.stringify(SYSTEMS)) errors.push("manifest.systems must preserve the canonical systems"); + if (JSON.stringify(value["dimensions"]) !== JSON.stringify(DIMENSIONS)) errors.push("manifest.dimensions must preserve the canonical rubric"); + if (JSON.stringify(value["hardGates"]) !== JSON.stringify(HARD_GATES)) errors.push("manifest.hardGates must preserve every hard gate"); + const tasks = manifestTaskIds(value); + if (tasks.length !== 8 || new Set(tasks).size !== 8) errors.push("manifest.taskIds must contain eight unique tasks"); + if (value["minimumRunsPerSystemTask"] !== 3) errors.push("manifest.minimumRunsPerSystemTask must be 3"); + const modes = value["captureModes"]; + if (!Array.isArray(modes) || modes.length !== 2 || !modes.every(isRecord)) { + errors.push("manifest.captureModes must contain desktop and mobile modes"); + } else { + const expected = [ + { id: "desktop-light", width: 1440, height: 900, colorScheme: "light" }, + { id: "mobile-dark", width: 390, height: 844, colorScheme: "dark" }, + ]; + if (JSON.stringify(modes) !== JSON.stringify(expected)) errors.push("manifest.captureModes must preserve the canonical captures"); + } + const thresholds = value["thresholds"]; + if (!isRecord(thresholds)) errors.push("manifest.thresholds must be an object"); + else { + exactKeys(thresholds, ["openCodeBetterOrEquivalent", "openCodeDimensionMedian", "minimumReviewers", "requiredReviewerRoles"], "manifest.thresholds", errors); + if (thresholds["openCodeBetterOrEquivalent"] !== 0.8) errors.push("manifest parity threshold must be 0.8"); + if (thresholds["openCodeDimensionMedian"] !== 4) errors.push("manifest absolute median must be 4"); + if (thresholds["minimumReviewers"] !== 3) errors.push("manifest minimumReviewers must be 3"); + if (JSON.stringify(thresholds["requiredReviewerRoles"]) !== JSON.stringify(["design-ux", "technical"])) errors.push("manifest required reviewer roles changed"); + } + return errors; +} + +const RUN_KEYS = ["schemaVersion", "runId", "corpusId", "manifestSha256", "authorization", "generations", "reviewers", "pairs"]; +const GENERATION_KEYS = ["id", "system", "taskId", "runIndex", "promptSha256", "fixtureSha256", "outputSha256", "capturedAt", "environment", "desktopCapture", "mobileCapture", "interactionTrace", "hardGates"]; + +export function validateBenchmarkRun(value: unknown, manifest: unknown): string[] { + const errors = validateBenchmarkManifest(manifest).map((error) => `invalid manifest: ${error}`); + if (!isRecord(value)) return [...errors, "run must be an object"]; + exactKeys(value, RUN_KEYS, "run", errors); + if (value["schemaVersion"] !== 1) errors.push("run.schemaVersion must be 1"); + text(value["runId"], "run.runId", errors, 120); + if (value["corpusId"] !== "page-quality-v1") errors.push("run.corpusId must be page-quality-v1"); + sha256(value["manifestSha256"], "run.manifestSha256", errors); + if (typeof value["manifestSha256"] === "string" && value["manifestSha256"] !== manifestDigest(manifest)) errors.push("run.manifestSha256 does not match the canonical manifest"); + + const authorization = value["authorization"]; + if (!isRecord(authorization)) errors.push("run.authorization must be an object"); + else { + const fields = ["status", "authorizedBy", "authorizedAt", "claudeAccountScope", "modelProtocol", "retentionDisposition"]; + exactKeys(authorization, fields, "run.authorization", errors); + if (authorization["status"] !== "pending" && authorization["status"] !== "approved") errors.push("run.authorization.status must be pending or approved"); + if (authorization["status"] === "approved") { + text(authorization["authorizedBy"], "run.authorization.authorizedBy", errors, 120); + isoTimestamp(authorization["authorizedAt"], "run.authorization.authorizedAt", errors); + for (const field of ["claudeAccountScope", "modelProtocol", "retentionDisposition"]) text(authorization[field], `run.authorization.${field}`, errors); + } else { + for (const field of fields.slice(1)) if (authorization[field] !== null) errors.push(`run.authorization.${field} must remain null while pending`); + } + } + + const tasks = new Set(manifestTaskIds(manifest)); + const generationIds = new Set(); + const generationsById = new Map>(); + if (!Array.isArray(value["generations"]) || value["generations"].length > 200) errors.push("run.generations must be an array of at most 200 entries"); + else for (let index = 0; index < value["generations"].length; index++) { + const item = value["generations"][index]; + const path = `run.generations[${index}]`; + if (!isRecord(item)) { errors.push(`${path} must be an object`); continue; } + exactKeys(item, GENERATION_KEYS, path, errors); + if (text(item["id"], `${path}.id`, errors, 120)) { + if (generationIds.has(item["id"])) errors.push(`${path}.id is duplicated`); + generationIds.add(item["id"]); + generationsById.set(item["id"], item); + } + if (!SYSTEMS.includes(String(item["system"]) as System)) errors.push(`${path}.system is invalid`); + if (typeof item["taskId"] !== "string" || !tasks.has(item["taskId"])) errors.push(`${path}.taskId is not canonical`); + if (!Number.isInteger(item["runIndex"]) || Number(item["runIndex"]) < 1 || Number(item["runIndex"]) > 20) errors.push(`${path}.runIndex must be an integer from 1 to 20`); + for (const field of ["promptSha256", "fixtureSha256", "outputSha256"]) sha256(item[field], `${path}.${field}`, errors); + isoTimestamp(item["capturedAt"], `${path}.capturedAt`, errors); + for (const field of ["environment", "desktopCapture", "mobileCapture", "interactionTrace"]) text(item[field], `${path}.${field}`, errors, 300); + const gates = item["hardGates"]; + if (!isRecord(gates)) errors.push(`${path}.hardGates must be an object`); + else { + exactKeys(gates, HARD_GATES, `${path}.hardGates`, errors); + for (const gate of HARD_GATES) if (typeof gates[gate] !== "boolean") errors.push(`${path}.hardGates.${gate} must be boolean`); + } + } + + const reviewerIds = new Set(); + if (!Array.isArray(value["reviewers"]) || value["reviewers"].length > 20) errors.push("run.reviewers must be an array of at most 20 entries"); + else for (let index = 0; index < value["reviewers"].length; index++) { + const item = value["reviewers"][index]; + const path = `run.reviewers[${index}]`; + if (!isRecord(item)) { errors.push(`${path} must be an object`); continue; } + exactKeys(item, ["id", "roles", "independent", "conflicts", "scoresRetainedWithConsent"], path, errors); + if (text(item["id"], `${path}.id`, errors, 80)) { + if (!/^r-[a-z0-9]{6,}$/.test(item["id"])) errors.push(`${path}.id must be pseudonymous`); + if (reviewerIds.has(item["id"])) errors.push(`${path}.id is duplicated`); + reviewerIds.add(item["id"]); + } + if (!Array.isArray(item["roles"]) || item["roles"].length === 0 || !item["roles"].every((role) => role === "design-ux" || role === "technical" || role === "reader")) errors.push(`${path}.roles are invalid`); + if (typeof item["independent"] !== "boolean") errors.push(`${path}.independent must be boolean`); + if (!Array.isArray(item["conflicts"]) || !item["conflicts"].every((conflict) => typeof conflict === "string")) errors.push(`${path}.conflicts must be a string array`); + if (typeof item["scoresRetainedWithConsent"] !== "boolean") errors.push(`${path}.scoresRetainedWithConsent must be boolean`); + } + + const pairIds = new Set(); + if (!Array.isArray(value["pairs"]) || value["pairs"].length > 100) errors.push("run.pairs must be an array of at most 100 entries"); + else for (let index = 0; index < value["pairs"].length; index++) { + const item = value["pairs"][index]; + const path = `run.pairs[${index}]`; + if (!isRecord(item)) { errors.push(`${path} must be an object`); continue; } + exactKeys(item, ["id", "taskId", "runIndex", "labelA", "labelB", "generationA", "generationB", "systemA", "systemB", "randomizationSha256", "scores"], path, errors); + if (text(item["id"], `${path}.id`, errors, 120)) { + if (pairIds.has(item["id"])) errors.push(`${path}.id is duplicated`); + pairIds.add(item["id"]); + } + if (typeof item["taskId"] !== "string" || !tasks.has(item["taskId"])) errors.push(`${path}.taskId is not canonical`); + if (!Number.isInteger(item["runIndex"]) || Number(item["runIndex"]) < 1) errors.push(`${path}.runIndex must be positive`); + for (const field of ["labelA", "labelB"]) if (typeof item[field] !== "string" || !/^[A-Z][0-9]{3}$/.test(item[field])) errors.push(`${path}.${field} must be a neutral blinded label`); + if (item["labelA"] === item["labelB"]) errors.push(`${path} labels must differ`); + for (const field of ["generationA", "generationB"]) if (typeof item[field] !== "string" || !generationIds.has(item[field])) errors.push(`${path}.${field} does not reference a generation`); + if (!SYSTEMS.includes(String(item["systemA"]) as System) || !SYSTEMS.includes(String(item["systemB"]) as System) || item["systemA"] === item["systemB"]) errors.push(`${path} must map one generation from each system`); + for (const side of ["A", "B"] as const) { + const generation = typeof item[`generation${side}`] === "string" ? generationsById.get(item[`generation${side}`]) : undefined; + if (generation !== undefined && (generation["system"] !== item[`system${side}`] || generation["taskId"] !== item["taskId"] || generation["runIndex"] !== item["runIndex"])) { + errors.push(`${path}.generation${side} does not match its system, task, and run mapping`); + } + } + sha256(item["randomizationSha256"], `${path}.randomizationSha256`, errors); + if (!Array.isArray(item["scores"]) || item["scores"].length > 20) errors.push(`${path}.scores must be an array of at most 20 entries`); + else { + const scored = new Set(); + for (let scoreIndex = 0; scoreIndex < item["scores"].length; scoreIndex++) { + const score = item["scores"][scoreIndex]; + const scorePath = `${path}.scores[${scoreIndex}]`; + if (!isRecord(score)) { errors.push(`${scorePath} must be an object`); continue; } + exactKeys(score, ["reviewerId", "a", "b", "overall", "reason"], scorePath, errors); + if (typeof score["reviewerId"] !== "string" || !reviewerIds.has(score["reviewerId"])) errors.push(`${scorePath}.reviewerId is unknown`); + else if (scored.has(score["reviewerId"])) errors.push(`${scorePath}.reviewerId is duplicated in the pair`); + else scored.add(score["reviewerId"]); + for (const side of ["a", "b"] as const) { + const dimensions = score[side]; + if (!isRecord(dimensions)) errors.push(`${scorePath}.${side} must be an object`); + else { + exactKeys(dimensions, DIMENSIONS, `${scorePath}.${side}`, errors); + for (const dimension of DIMENSIONS) if (!Number.isInteger(dimensions[dimension]) || Number(dimensions[dimension]) < 1 || Number(dimensions[dimension]) > 5) errors.push(`${scorePath}.${side}.${dimension} must be an integer from 1 to 5`); + } + } + if (score["overall"] !== "a" && score["overall"] !== "equivalent" && score["overall"] !== "b") errors.push(`${scorePath}.overall is invalid`); + text(score["reason"], `${scorePath}.reason`, errors, 500); + } + } + } + return errors; +} + +export interface BlindedReviewPacket { + schemaVersion: 1; + runId: string; + corpusId: "page-quality-v1"; + manifestSha256: string; + dimensions: readonly Dimension[]; + scoring: { + minimum: 1; + maximum: 5; + overall: readonly ["a", "equivalent", "b"]; + }; + pairs: Array<{ + id: string; + taskId: string; + runIndex: number; + labelA: string; + labelB: string; + resources: { + a: { desktop: string; mobile: string; interaction: string }; + b: { desktop: string; mobile: string; interaction: string }; + }; + scoreTemplate: { + a: Record; + b: Record; + overall: null; + reason: null; + }; + }>; +} + +export interface PreparedBlindedReview { + privateRun: Record; + reviewerPacket: BlindedReviewPacket; +} + +function nullScores(): Record { + return Object.fromEntries(DIMENSIONS.map((dimension) => [dimension, null])) as Record; +} + +function completeGenerationMatrix( + value: Record, + manifest: unknown, +): Map> { + const generations = Array.isArray(value["generations"]) + ? value["generations"].filter(isRecord) + : []; + const byKey = new Map>(); + for (const generation of generations) { + const key = `${generation["system"]}:${generation["taskId"]}:${generation["runIndex"]}`; + if (byKey.has(key)) throw new Error(`duplicate benchmark generation ${key}`); + byKey.set(key, generation); + } + const expected = new Set(); + for (const taskId of manifestTaskIds(manifest)) for (const system of SYSTEMS) { + for (let runIndex = 1; runIndex <= 3; runIndex++) expected.add(`${system}:${taskId}:${runIndex}`); + } + for (const key of expected) if (!byKey.has(key)) throw new Error(`missing benchmark generation ${key}`); + for (const key of byKey.keys()) if (!expected.has(key)) throw new Error(`unexpected benchmark generation ${key}`); + return byKey; +} + +export function prepareBlindedReview( + value: unknown, + manifest: unknown, + seed: Uint8Array, +): PreparedBlindedReview { + const errors = validateBenchmarkRun(value, manifest); + if (errors.length > 0) throw new Error(errors.join("\n")); + if (!isRecord(value)) throw new Error("validated benchmark run was lost"); + if (seed.byteLength < 32) throw new Error("blinding seed must contain at least 32 bytes"); + if (!Array.isArray(value["pairs"]) || value["pairs"].length !== 0) { + throw new Error("blinding preparation requires an unpaired run and never replaces an existing mapping"); + } + const authorization = value["authorization"]; + if (!isRecord(authorization) || authorization["status"] !== "approved") { + throw new Error("blinding preparation requires approved account, settings, and retention authority"); + } + const generations = completeGenerationMatrix(value, manifest); + const pairs: Record[] = []; + const reviewerPairs: BlindedReviewPacket["pairs"] = []; + let ordinal = 0; + for (const taskId of manifestTaskIds(manifest)) for (let runIndex = 1; runIndex <= 3; runIndex++) { + ordinal++; + const opencode = generations.get(`opencode:${taskId}:${runIndex}`); + const claude = generations.get(`claude:${taskId}:${runIndex}`); + if (!opencode || !claude) throw new Error(`benchmark generation matrix changed for ${taskId}/${runIndex}`); + const randomizationInput = canonicalJson({ + corpusId: value["corpusId"], + manifestSha256: value["manifestSha256"], + runId: value["runId"], + taskId, + runIndex, + opencode: opencode["id"], + claude: claude["id"], + }); + const randomization = createHmac("sha256", seed).update(randomizationInput).digest(); + const openCodeIsA = (randomization[0] ?? 0) < 128; + const generationA = openCodeIsA ? opencode : claude; + const generationB = openCodeIsA ? claude : opencode; + const labelA = `A${String(ordinal).padStart(3, "0")}`; + const labelB = `B${String(ordinal).padStart(3, "0")}`; + const id = `${taskId}-${runIndex}`; + pairs.push({ + id, + taskId, + runIndex, + labelA, + labelB, + generationA: generationA["id"], + generationB: generationB["id"], + systemA: generationA["system"], + systemB: generationB["system"], + randomizationSha256: randomization.toString("hex"), + scores: [], + }); + reviewerPairs.push({ + id, + taskId, + runIndex, + labelA, + labelB, + resources: { + a: { + desktop: `blinded://${labelA}/desktop`, + mobile: `blinded://${labelA}/mobile`, + interaction: `blinded://${labelA}/interaction`, + }, + b: { + desktop: `blinded://${labelB}/desktop`, + mobile: `blinded://${labelB}/mobile`, + interaction: `blinded://${labelB}/interaction`, + }, + }, + scoreTemplate: { a: nullScores(), b: nullScores(), overall: null, reason: null }, + }); + } + const privateRun = structuredClone(value); + privateRun["pairs"] = pairs; + const reviewerRunId = `review-${createHmac("sha256", seed).update(`reviewer-packet:${String(value["runId"])}`).digest("hex").slice(0, 16)}`; + return { + privateRun, + reviewerPacket: { + schemaVersion: 1, + runId: reviewerRunId, + corpusId: "page-quality-v1", + manifestSha256: String(value["manifestSha256"]), + dimensions: DIMENSIONS, + scoring: { minimum: 1, maximum: 5, overall: ["a", "equivalent", "b"] }, + pairs: reviewerPairs, + }, + }; +} + +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[middle] ?? null : ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2; +} + +function wilson(successes: number, total: number): { lower: number | null; upper: number | null } { + if (total === 0) return { lower: null, upper: null }; + const z = 1.96; + const p = successes / total; + const denominator = 1 + z * z / total; + const center = (p + z * z / (2 * total)) / denominator; + const margin = z * Math.sqrt((p * (1 - p) + z * z / (4 * total)) / total) / denominator; + return { lower: Math.max(0, center - margin), upper: Math.min(1, center + margin) }; +} + +export interface BenchmarkSummary { + status: "pass" | "fail" | "incomplete"; + missing: string[]; + pairOutcomes: { openCodeBetter: number; equivalent: number; claudeBetter: number; qualifyingRate: number | null; confidence95: { lower: number | null; upper: number | null } }; + taskClaudeMajorities: string[]; + dimensionMedians: Record; + openCodeHardGateFailures: string[]; + thresholds: Record; +} + +export function summarizeBenchmarkRun(value: unknown, manifest: unknown): BenchmarkSummary { + const errors = validateBenchmarkRun(value, manifest); + if (errors.length > 0) throw new Error(errors.join("\n")); + if (!isRecord(value) || !Array.isArray(value["generations"]) || !Array.isArray(value["reviewers"]) || !Array.isArray(value["pairs"])) throw new Error("validated benchmark shape was lost"); + const generations = value["generations"].filter(isRecord); + const reviewers = value["reviewers"].filter(isRecord); + const pairs = value["pairs"].filter(isRecord); + const tasks = manifestTaskIds(manifest); + const missing: string[] = []; + const authorization = value["authorization"]; + if (!isRecord(authorization) || authorization["status"] !== "approved") missing.push("authorized Claude account/settings/retention protocol"); + for (const task of tasks) for (const system of SYSTEMS) { + const indexes = generations.filter((item) => item["taskId"] === task && item["system"] === system).map((item) => Number(item["runIndex"])).sort((a, b) => a - b); + if (indexes.length < 3 || indexes.some((index, position) => index !== position + 1)) missing.push(`${system}/${task}: three consecutive unselected generations`); + } + const eligibleReviewers = reviewers.filter((item) => item["independent"] === true && item["scoresRetainedWithConsent"] === true && Array.isArray(item["conflicts"]) && item["conflicts"].length === 0); + const roles = new Set(eligibleReviewers.flatMap((item) => Array.isArray(item["roles"]) ? item["roles"].filter((role): role is string => typeof role === "string") : [])); + if (eligibleReviewers.length < 3) missing.push("three eligible independent reviewers"); + for (const role of ["design-ux", "technical"]) if (!roles.has(role)) missing.push(`${role} reviewer role`); + const expectedPairKeys = new Set(generations.map((item) => `${item["taskId"]}:${item["runIndex"]}`)); + const actualPairKeys = new Set(pairs.map((item) => `${item["taskId"]}:${item["runIndex"]}`)); + for (const key of expectedPairKeys) if (!actualPairKeys.has(key)) missing.push(`pair ${key}`); + for (const pair of pairs) { + const scores = Array.isArray(pair["scores"]) ? pair["scores"].filter(isRecord) : []; + const scoredIds = new Set(scores.map((score) => score["reviewerId"])); + for (const reviewer of eligibleReviewers) if (!scoredIds.has(reviewer["id"])) missing.push(`pair ${pair["id"]}: score from ${reviewer["id"]}`); + } + + const outcomes = { openCodeBetter: 0, equivalent: 0, claudeBetter: 0 }; + const taskJudgments = new Map(); + const dimensionValues = Object.fromEntries(DIMENSIONS.map((dimension) => [dimension, { opencode: [] as number[], claude: [] as number[] }])) as Record; + for (const pair of pairs) { + const systemA = pair["systemA"] as System; + let openCodeVotes = 0; + let claudeVotes = 0; + const task = String(pair["taskId"]); + const taskCounts = taskJudgments.get(task) ?? { opencode: 0, claude: 0 }; + for (const score of Array.isArray(pair["scores"]) ? pair["scores"].filter(isRecord) : []) { + const overall = score["overall"]; + if (overall !== "equivalent") { + const chosen = overall === "a" ? systemA : systemA === "opencode" ? "claude" : "opencode"; + if (chosen === "opencode") { openCodeVotes++; taskCounts.opencode++; } + else { claudeVotes++; taskCounts.claude++; } + } + for (const dimension of DIMENSIONS) { + const a = isRecord(score["a"]) ? Number(score["a"][dimension]) : Number.NaN; + const b = isRecord(score["b"]) ? Number(score["b"][dimension]) : Number.NaN; + if (systemA === "opencode") { dimensionValues[dimension].opencode.push(a); dimensionValues[dimension].claude.push(b); } + else { dimensionValues[dimension].opencode.push(b); dimensionValues[dimension].claude.push(a); } + } + } + taskJudgments.set(task, taskCounts); + if (openCodeVotes > claudeVotes) outcomes.openCodeBetter++; + else if (claudeVotes > openCodeVotes) outcomes.claudeBetter++; + else outcomes.equivalent++; + } + const totalPairs = pairs.length; + const qualifying = outcomes.openCodeBetter + outcomes.equivalent; + const qualifyingRate = totalPairs === 0 ? null : qualifying / totalPairs; + const taskClaudeMajorities = [...taskJudgments].filter(([, counts]) => counts.claude > counts.opencode).map(([task]) => task).sort(); + const dimensionMedians = Object.fromEntries(DIMENSIONS.map((dimension) => [dimension, { opencode: median(dimensionValues[dimension].opencode), claude: median(dimensionValues[dimension].claude) }])) as BenchmarkSummary["dimensionMedians"]; + const openCodeHardGateFailures = generations.filter((item) => item["system"] === "opencode" && isRecord(item["hardGates"]) && HARD_GATES.some((gate) => item["hardGates"][gate] !== true)).map((item) => String(item["id"])); + const thresholds = { + eightyPercent: qualifyingRate !== null && qualifyingRate >= 0.8, + noTaskClaudeMajority: taskClaudeMajorities.length === 0, + dimensionParity: DIMENSIONS.every((dimension) => dimensionMedians[dimension].opencode !== null && dimensionMedians[dimension].claude !== null && Number(dimensionMedians[dimension].opencode) >= Number(dimensionMedians[dimension].claude)), + absoluteQuality: DIMENSIONS.every((dimension) => dimensionMedians[dimension].opencode !== null && Number(dimensionMedians[dimension].opencode) >= 4), + hardGates: openCodeHardGateFailures.length === 0 && generations.some((item) => item["system"] === "opencode"), + }; + const status = missing.length > 0 ? "incomplete" : Object.values(thresholds).every(Boolean) ? "pass" : "fail"; + return { + status, + missing: [...new Set(missing)].sort(), + pairOutcomes: { ...outcomes, qualifyingRate, confidence95: wilson(qualifying, totalPairs) }, + taskClaudeMajorities, + dimensionMedians, + openCodeHardGateFailures, + thresholds, + }; +} diff --git a/scripts/page-quality-benchmark.ts b/scripts/page-quality-benchmark.ts new file mode 100644 index 0000000..9cf6f0f --- /dev/null +++ b/scripts/page-quality-benchmark.ts @@ -0,0 +1,50 @@ +#!/usr/bin/env node + +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { prepareBlindedReview, summarizeBenchmarkRun, validateBenchmarkRun } from "./page-quality-benchmark-lib.ts"; + +const root = resolve(import.meta.dirname, ".."); +async function main(): Promise { + const [commandOrInput, ...args] = process.argv.slice(2); + if (!commandOrInput) { + console.error("Usage: npm run quality:benchmark -- \n npm run quality:benchmark -- prepare "); + process.exitCode = 2; + return; + } + const manifest = JSON.parse(await readFile(resolve(root, "benchmarks/page-quality/v1/benchmark-manifest.json"), "utf8")) as unknown; + if (commandOrInput === "prepare") { + const [input, seedPath, privateOutput, reviewerOutput] = args; + if (!input || !seedPath || !privateOutput || !reviewerOutput) { + console.error("Usage: npm run quality:benchmark -- prepare "); + process.exitCode = 2; + return; + } + const privatePath = resolve(process.cwd(), privateOutput); + const reviewerPath = resolve(process.cwd(), reviewerOutput); + if (privatePath === reviewerPath) throw new Error("private and reviewer outputs must be different files"); + const run = JSON.parse(await readFile(resolve(process.cwd(), input), "utf8")) as unknown; + const seed = await readFile(resolve(process.cwd(), seedPath)); + const prepared = prepareBlindedReview(run, manifest, seed); + await writeFile(privatePath, `${JSON.stringify(prepared.privateRun, null, 2)}\n`, { encoding: "utf8", flag: "wx", mode: 0o600 }); + try { + await writeFile(reviewerPath, `${JSON.stringify(prepared.reviewerPacket, null, 2)}\n`, { encoding: "utf8", flag: "wx", mode: 0o600 }); + } catch (error) { + console.error(`reviewer packet was not written; private mapping remains at ${privatePath}`); + throw error; + } + console.log(JSON.stringify({ privateOutput: privatePath, reviewerOutput: reviewerPath, pairs: prepared.reviewerPacket.pairs.length }, null, 2)); + return; + } + + const run = JSON.parse(await readFile(resolve(process.cwd(), commandOrInput), "utf8")) as unknown; + const errors = validateBenchmarkRun(run, manifest); + if (errors.length > 0) { + for (const error of errors) console.error(`FAIL - ${error}`); + process.exitCode = 1; + return; + } + console.log(JSON.stringify(summarizeBenchmarkRun(run, manifest), null, 2)); +} + +await main(); diff --git a/scripts/page-quality-browser-evidence.ts b/scripts/page-quality-browser-evidence.ts new file mode 100644 index 0000000..54a0142 --- /dev/null +++ b/scripts/page-quality-browser-evidence.ts @@ -0,0 +1,166 @@ +import { writeFile } from "node:fs/promises"; + +interface WebDriverResponse { value: T } +interface BrowserLog { level: string; message: string; timestamp: number } +interface PerformanceLog { message: string } +interface AxNode { role?: { value?: string }; name?: { value?: string }; ignored?: boolean } + +const endpoint = process.argv[2] ?? "http://127.0.0.1:4444"; +const pageUrl = process.argv[3]; +const screenshotPath = process.argv[4]; +const reportPath = process.argv[5]; +const width = Number(process.argv[6]); +const height = Number(process.argv[7]); +const colorScheme = process.argv[8] ?? "light"; +if (!pageUrl || !screenshotPath || !reportPath || !Number.isInteger(width) || !Number.isInteger(height)) { + throw new Error("usage: page-quality-browser-evidence [light|dark]"); +} +if (width < 320 || height < 480) throw new Error("viewport is below the supported evidence floor"); +if (colorScheme !== "light" && colorScheme !== "dark") throw new Error("color scheme must be light or dark"); + +async function request(method: string, path: string, body?: unknown): Promise { + const response = await fetch(`${endpoint}${path}`, { + method, + headers: body === undefined ? undefined : { "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const responseText = await response.text(); + if (!response.ok) throw new Error(`WebDriver ${method} ${path} failed (${response.status}): ${responseText.slice(0, 1000)}`); + return (JSON.parse(responseText) as WebDriverResponse).value; +} + +const session = await request<{ sessionId: string }>("POST", "/session", { + capabilities: { alwaysMatch: { + browserName: "chrome", + "goog:loggingPrefs": { browser: "ALL", performance: "ALL" }, + "goog:chromeOptions": { args: ["--headless=new", "--no-sandbox", "--disable-dev-shm-usage"] }, + } }, +}); +const route = `/session/${session.sessionId}`; + +async function script(source: string): Promise { + return request("POST", `${route}/execute/sync`, { script: source, args: [] }); +} + +async function key(value: string): Promise { + await request("POST", `${route}/actions`, { actions: [{ + type: "key", id: "keyboard", actions: [{ type: "keyDown", value }, { type: "keyUp", value }], + }] }); +} + +async function focus(selector: string): Promise { + return script(`const n=document.querySelector(${JSON.stringify(selector)});if(n)n.focus();return !!n;`); +} + +try { + await request("POST", `${route}/window/rect`, { width, height, x: 0, y: 0 }); + await request("POST", `${route}/goog/cdp/execute`, { cmd: "Network.enable", params: {} }); + await request("POST", `${route}/goog/cdp/execute`, { + cmd: "Page.addScriptToEvaluateOnNewDocument", + params: { source: "window.__ARTIFACT_LAYOUT_SHIFT__=0;new PerformanceObserver(function(list){list.getEntries().forEach(function(entry){if(!entry.hadRecentInput)window.__ARTIFACT_LAYOUT_SHIFT__+=entry.value;});}).observe({type:'layout-shift',buffered:true});" }, + }); + await request("POST", `${route}/goog/cdp/execute`, { + cmd: "Emulation.setEmulatedMedia", + params: { media: "screen", features: [ + { name: "prefers-color-scheme", value: colorScheme }, + { name: "prefers-reduced-motion", value: width <= 390 ? "reduce" : "no-preference" }, + ] }, + }); + const startedAt = Date.now(); + await request("POST", `${route}/url`, { url: pageUrl }); + let ready = false; + for (let attempt = 0; attempt < 160; attempt++) { + ready = await script(`return document.readyState==='complete'&&!document.querySelector('.component[data-component-index]')&&Array.from(document.querySelectorAll('.chart')).every(n=>n.querySelector('canvas,svg'))&&Array.from(document.querySelectorAll('pre.mermaid')).every(n=>n.querySelector('svg'));`); + if (ready) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + const usefulContentMs = Date.now() - startedAt; + await new Promise((resolve) => setTimeout(resolve, 500)); + const initialScreenshot = await request("GET", `${route}/screenshot`); + + await script("document.activeElement&&document.activeElement.blur();return true;"); + await key("\uE004"); + const firstTab = await script>(`return {tag:document.activeElement?.tagName||null,className:document.activeElement?.className||null,text:document.activeElement?.textContent?.trim()||null};`); + const keyboard: Record = { firstTab }; + if (await focus(".decision-opt")) { + await key("\uE014"); + keyboard["decision"] = await script(`const n=document.querySelector('.decision-opt[aria-checked="true"]');return {selected:n?.dataset.option||null,active:document.activeElement?.dataset.option||null};`); + } + if (await focus(".th-sort")) { + await key("\uE007"); + keyboard["table"] = await script(`const n=document.querySelector('th[aria-sort]:not([aria-sort="none"])');return {direction:n?.getAttribute('aria-sort')||null,label:n?.textContent?.trim()||null};`); + } + if (await focus('input[type="range"]')) { + const before = await script("return document.activeElement?.value||'';"); + await key("\uE014"); + const after = await script("return document.activeElement?.value||'';"); + keyboard["range"] = { before, after }; + } + if (await focus(".copy-btn")) { + await key("\uE007"); + await new Promise((resolve) => setTimeout(resolve, 100)); + keyboard["copy"] = await script("return document.querySelector('.copy-note')?.textContent?.trim()||'';"); + } + + const observations = await script>(` + const main=document.querySelector('main'); + const sections=Array.from(document.querySelectorAll('.section-card')).map((section)=>{ + const visual=section.querySelector('.chart-frame,.diagram-frame,.visual-frame'); + const sr=section.getBoundingClientRect();const vr=visual?.getBoundingClientRect(); + return {heading:section.querySelector('h2')?.textContent?.trim()||null,width:Math.round(sr.width),visualWidth:vr?Math.round(vr.width):null,utilization:vr&&sr.width?Number((vr.width/sr.width).toFixed(3)):null,classes:section.className}; + }); + const clipped=Array.from(document.querySelectorAll('h1,h2,h3,.stat-label,.finding-title,.pill,.decision-label,figcaption,th,td')).flatMap((node)=>{ + const element=/** @type {HTMLElement} */(node);const style=getComputedStyle(element); + return element.scrollWidth>element.clientWidth+1&&style.overflow!=='visible'?[element.textContent?.trim()?.slice(0,120)||element.tagName]:[]; + }); + const errors=Array.from(document.querySelectorAll('.chart-error,.asset-error')).map((n)=>n.textContent?.trim()); + return { + readyState:document.readyState, + viewport:{width:innerWidth,height:innerHeight,devicePixelRatio}, + composition:Array.from(main?.classList||[]).find((name)=>name.startsWith('composition-'))||'standard', + documentHorizontalOverflow:document.documentElement.scrollWidth>document.documentElement.clientWidth, + mainHorizontalOverflow:!!main&&main.scrollWidth>main.clientWidth, + reducedMotion:matchMedia('(prefers-reduced-motion: reduce)').matches, + colorScheme:matchMedia('(prefers-color-scheme: dark)').matches?'dark':'light', + sections,clippedText:clipped,renderErrors:errors, + headings:Array.from(document.querySelectorAll('h1,h2,h3')).map((n)=>n.textContent?.trim()), + interactive:{buttons:document.querySelectorAll('button').length,ranges:document.querySelectorAll('input[type="range"]').length,radios:document.querySelectorAll('[role="radio"]').length}, + chartGeometry:Array.from(document.querySelectorAll('.chart')).map((chart)=>{const own=chart.getBoundingClientRect();const child=chart.querySelector('canvas,svg')?.getBoundingClientRect();return {width:own.width,height:own.height,childWidth:child?.width||0,childHeight:child?.height||0,markup:chart.innerHTML.slice(0,160)};}), + diagramGeometry:Array.from(document.querySelectorAll('.diagram-frame svg')).map((svg)=>{const own=svg.getBoundingClientRect();const parent=svg.parentElement?.getBoundingClientRect();return {width:own.width,height:own.height,parentWidth:parent?.width||0,viewBox:svg.getAttribute('viewBox'),widthAttribute:svg.getAttribute('width'),style:svg.getAttribute('style')};}), + layoutShift:window.__ARTIFACT_LAYOUT_SHIFT__||0 + }; + `); + + const ax = await request<{ nodes?: AxNode[] }>("POST", `${route}/goog/cdp/execute`, { cmd: "Accessibility.getFullAXTree", params: {} }); + const roles = new Set(["RootWebArea", "banner", "main", "contentinfo", "heading", "figure", "image", "table", "caption", "button", "slider", "radiogroup", "radio", "status"]); + const accessibilityTree = (ax.nodes ?? []).filter((node) => !node.ignored && roles.has(node.role?.value ?? "")) + .map((node) => ({ role: node.role?.value ?? "", name: node.name?.value ?? "" })).slice(0, 160); + await writeFile(screenshotPath, Buffer.from(initialScreenshot, "base64")); + const browserLogs = await request("POST", `${route}/log`, { type: "browser" }); + const performanceLogs = await request("POST", `${route}/log`, { type: "performance" }); + const requestUrls = performanceLogs.flatMap((entry) => { + const parsed = JSON.parse(entry.message) as { message?: { method?: string; params?: { request?: { url?: string } } } }; + const url = parsed.message?.method === "Network.requestWillBeSent" ? parsed.message.params?.request?.url : undefined; + return url === undefined ? [] : [url]; + }); + const origin = new URL(pageUrl).origin; + const report = { + schemaVersion: 1, + capturedAt: new Date().toISOString(), + browser: "Chromium 151 via selenium/standalone-chromium", + fixture: pageUrl, + requested: { width, height, colorScheme, reducedMotion: width <= 390 }, + usefulContentMs, + ready, + keyboard, + observations, + accessibilityTree, + browserLogs, + requestUrls, + externalHttpRequests: requestUrls.filter((url) => /^https?:/i.test(url) && new URL(url).origin !== origin), + }; + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + console.log(JSON.stringify({ ready, usefulContentMs, observations, browserLogCount: browserLogs.length, externalHttpRequests: report.externalHttpRequests }, null, 2)); +} finally { + await request("DELETE", route); +} diff --git a/scripts/page-quality-corpus.ts b/scripts/page-quality-corpus.ts new file mode 100644 index 0000000..b4370bb --- /dev/null +++ b/scripts/page-quality-corpus.ts @@ -0,0 +1,171 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { lstat, readFile, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const PAGE_QUALITY_TASK_IDS = [ + "dashboard", + "incident", + "pr-walkthrough", + "system-explainer", + "compare", + "plan-checklist", + "findings-table", + "interactive-decision", +] as const; + +const STRESS_CASES = ["long-label", "missing-value", "dense-data", "narrow-viewport"]; +const BUNDLE_KEYS = [ + "id", "task", "fixture", "fixtureSha256", "primaryReader", "readerDecision", "prompt", + "requiredFacts", "forbiddenClaims", "sourceBundle", "interactions", "stressCases", +]; +const SOURCE_KEYS = ["license", "provenance", "missingValues", "longLabel"]; +const INTERACTION_KEYS = ["id", "action", "expectedState"]; + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : undefined; +} + +function exactKeys(value: Record, expected: readonly string[], path: string, errors: string[]): void { + for (const key of expected) if (!(key in value)) errors.push(`${path} is missing ${key}`); + for (const key of Object.keys(value)) if (!expected.includes(key)) errors.push(`${path} has unexpected field ${key}`); +} + +function text(value: unknown, path: string, errors: string[], min = 1): value is string { + if (typeof value !== "string" || value.trim().length < min) { + errors.push(`${path} must be a string of at least ${min} characters`); + return false; + } + return true; +} + +function stringList(value: unknown, path: string, errors: string[], min: number): string[] { + if (!Array.isArray(value) || value.length < min || value.some((item) => typeof item !== "string" || item.trim() === "")) { + errors.push(`${path} must contain at least ${min} non-empty strings`); + return []; + } + return value as string[]; +} + +function safeRelativePath(value: string): boolean { + return !isAbsolute(value) && value !== "" && !value.split(/[\\/]/).includes(".."); +} + +export interface CorpusValidation { + schemaVersion: number | undefined; + corpusId: string | undefined; + bundleCount: number; + errors: string[]; +} + +export async function validatePageQualityCorpus( + repositoryRoot: string, + corpusPath = "benchmarks/page-quality/v1/corpus.json", +): Promise { + const errors: string[] = []; + const root = await realpath(repositoryRoot); + const corpusFile = resolve(root, corpusPath); + const corpusRelative = relative(root, corpusFile); + if (corpusRelative === "" || corpusRelative === ".." || corpusRelative.startsWith(`..${sep}`) || isAbsolute(corpusRelative)) { + return { schemaVersion: undefined, corpusId: undefined, bundleCount: 0, errors: ["corpus path escapes the repository"] }; + } + let parsed: unknown; + try { + const bytes = await readFile(corpusFile); + if (bytes.byteLength > 128 * 1024) errors.push("corpus exceeds the 128 KiB limit"); + parsed = JSON.parse(bytes.toString("utf8")) as unknown; + } catch (error) { + return { schemaVersion: undefined, corpusId: undefined, bundleCount: 0, errors: [`cannot read corpus: ${error instanceof Error ? error.message : String(error)}`] }; + } + const corpus = record(parsed); + if (!corpus) return { schemaVersion: undefined, corpusId: undefined, bundleCount: 0, errors: ["corpus must be an object"] }; + exactKeys(corpus, ["schemaVersion", "corpusId", "license", "provenance", "captures", "bundles"], "corpus", errors); + if (corpus["schemaVersion"] !== 1) errors.push("corpus.schemaVersion must be 1"); + if (corpus["corpusId"] !== "page-quality-v1") errors.push("corpus.corpusId must be page-quality-v1"); + if (corpus["license"] !== "CC0-1.0") errors.push("corpus.license must be CC0-1.0"); + text(corpus["provenance"], "corpus.provenance", errors, 20); + const captures = corpus["captures"]; + if (!Array.isArray(captures) || captures.length !== 2) errors.push("corpus.captures must contain desktop and mobile"); + else { + const expected = [{ id: "desktop", width: 1440, height: 900 }, { id: "mobile", width: 390, height: 844 }]; + if (JSON.stringify(captures) !== JSON.stringify(expected)) errors.push("corpus.captures must use the canonical viewports"); + } + const bundles = corpus["bundles"]; + if (!Array.isArray(bundles)) { + errors.push("corpus.bundles must be an array"); + return { schemaVersion: corpus["schemaVersion"] as number | undefined, corpusId: corpus["corpusId"] as string | undefined, bundleCount: 0, errors }; + } + const ids: string[] = []; + for (const [index, value] of bundles.entries()) { + const path = `corpus.bundles[${index}]`; + const bundle = record(value); + if (!bundle) { errors.push(`${path} must be an object`); continue; } + exactKeys(bundle, BUNDLE_KEYS, path, errors); + if (text(bundle["id"], `${path}.id`, errors)) ids.push(bundle["id"]); + for (const key of ["task", "primaryReader", "readerDecision"] as const) text(bundle[key], `${path}.${key}`, errors, 8); + text(bundle["prompt"], `${path}.prompt`, errors, 80); + stringList(bundle["requiredFacts"], `${path}.requiredFacts`, errors, 3); + stringList(bundle["forbiddenClaims"], `${path}.forbiddenClaims`, errors, 2); + const stresses = stringList(bundle["stressCases"], `${path}.stressCases`, errors, 4); + if (JSON.stringify([...new Set(stresses)].sort()) !== JSON.stringify([...STRESS_CASES].sort())) { + errors.push(`${path}.stressCases must contain each canonical stress case exactly once`); + } + const source = record(bundle["sourceBundle"]); + if (!source) errors.push(`${path}.sourceBundle must be an object`); + else { + exactKeys(source, SOURCE_KEYS, `${path}.sourceBundle`, errors); + if (source["license"] !== "CC0-1.0") errors.push(`${path}.sourceBundle.license must be CC0-1.0`); + text(source["provenance"], `${path}.sourceBundle.provenance`, errors, 8); + stringList(source["missingValues"], `${path}.sourceBundle.missingValues`, errors, 1); + text(source["longLabel"], `${path}.sourceBundle.longLabel`, errors, 30); + } + const interactions = bundle["interactions"]; + if (!Array.isArray(interactions) || interactions.length === 0) errors.push(`${path}.interactions must not be empty`); + else for (const [interactionIndex, interactionValue] of interactions.entries()) { + const interactionPath = `${path}.interactions[${interactionIndex}]`; + const interaction = record(interactionValue); + if (!interaction) { errors.push(`${interactionPath} must be an object`); continue; } + exactKeys(interaction, INTERACTION_KEYS, interactionPath, errors); + for (const key of INTERACTION_KEYS) text(interaction[key], `${interactionPath}.${key}`, errors, key === "id" ? 2 : 12); + } + const fixture = bundle["fixture"]; + const digest = bundle["fixtureSha256"]; + if (!text(fixture, `${path}.fixture`, errors) || !safeRelativePath(fixture)) errors.push(`${path}.fixture must be a safe relative path`); + if (typeof digest !== "string" || !/^[a-f0-9]{64}$/.test(digest)) errors.push(`${path}.fixtureSha256 must be lowercase SHA-256`); + if (typeof fixture === "string" && safeRelativePath(fixture)) { + const fixturePath = resolve(root, fixture); + const fixtureRelative = relative(root, fixturePath); + if (fixtureRelative.startsWith(`..${sep}`) || isAbsolute(fixtureRelative)) errors.push(`${path}.fixture escapes the repository`); + else try { + const stat = await lstat(fixturePath); + if (!stat.isFile() || stat.isSymbolicLink()) errors.push(`${path}.fixture must be a regular non-symlink file`); + else { + const bytes = await readFile(fixturePath); + if (bytes.byteLength > 1024 * 1024) errors.push(`${path}.fixture exceeds 1 MiB`); + const actual = createHash("sha256").update(bytes).digest("hex"); + if (actual !== digest) errors.push(`${path}.fixtureSha256 does not match ${fixture}`); + } + } catch (error) { + errors.push(`${path}.fixture cannot be read: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + if (JSON.stringify(ids) !== JSON.stringify(PAGE_QUALITY_TASK_IDS)) errors.push("corpus bundle IDs/order must match the canonical eight-task corpus"); + if (new Set(ids).size !== ids.length) errors.push("corpus bundle IDs must be unique"); + return { schemaVersion: corpus["schemaVersion"] as number | undefined, corpusId: corpus["corpusId"] as string | undefined, bundleCount: bundles.length, errors }; +} + +async function main(): Promise { + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const result = await validatePageQualityCorpus(root, process.argv[2]); + if (result.errors.length > 0) { + for (const error of result.errors) console.error(`FAIL - ${error}`); + process.exitCode = 1; + } else console.log(`ok - ${result.corpusId}: ${result.bundleCount} validated permission-safe bundles`); +} + +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) await main(); diff --git a/scripts/page-quality-local-report.ts b/scripts/page-quality-local-report.ts new file mode 100644 index 0000000..91dc98c --- /dev/null +++ b/scripts/page-quality-local-report.ts @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readdir, readFile, writeFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; +import { PAGE_QUALITY_TASK_IDS } from "./page-quality-corpus.ts"; + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : undefined; +} + +function list(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } + +export interface LocalCompositionSummary { + schemaVersion: 1; + corpusId: "page-quality-v1"; + cells: number; + passed: number; + failed: number; + maxUsefulContentMs: number; + maxLayoutShift: number; + minPrimaryVisualUtilization: number | null; + minChartFill: number | null; + failures: string[]; + evidence: Array<{ report: string; sha256: string }>; +} + +export function summarizeLocalCompositionReports(reports: Array<{ name: string; bytes: Buffer; value: unknown }>): LocalCompositionSummary { + const failures: string[] = []; + let maxUsefulContentMs = 0; + let maxLayoutShift = 0; + const visualUtilizations: number[] = []; + const chartFills: number[] = []; + const expected = PAGE_QUALITY_TASK_IDS.flatMap((id) => [`${id}-desktop.json`, `${id}-mobile.json`]); + const byName = new Map(reports.map((report) => [report.name, report])); + for (const name of expected) { + const report = byName.get(name); + if (!report) { failures.push(`${name}: missing report`); continue; } + const value = record(report.value); + const requested = record(value?.["requested"]); + const observations = record(value?.["observations"]); + const keyboard = record(value?.["keyboard"]); + if (value?.["schemaVersion"] !== 1) failures.push(`${name}: schema version is not 1`); + if (value?.["ready"] !== true) failures.push(`${name}: page did not settle`); + const useful = value?.["usefulContentMs"]; + if (typeof useful !== "number" || !Number.isFinite(useful)) failures.push(`${name}: useful-content time is missing`); + else maxUsefulContentMs = Math.max(maxUsefulContentMs, useful); + const mobile = name.endsWith("-mobile.json"); + if (requested?.["width"] !== (mobile ? 390 : 1440) || requested?.["height"] !== (mobile ? 844 : 900)) failures.push(`${name}: viewport is not canonical`); + if (requested?.["colorScheme"] !== (mobile ? "dark" : "light")) failures.push(`${name}: color mode is not canonical`); + if (observations?.["reducedMotion"] !== mobile) failures.push(`${name}: reduced-motion observation is incorrect`); + for (const field of ["documentHorizontalOverflow", "mainHorizontalOverflow"] as const) { + if (observations?.[field] !== false) failures.push(`${name}: ${field}`); + } + for (const [field, valueList] of [["clippedText", observations?.["clippedText"]], ["renderErrors", observations?.["renderErrors"]], ["browserLogs", value?.["browserLogs"]], ["externalHttpRequests", value?.["externalHttpRequests"]]] as const) { + if (!Array.isArray(valueList) || valueList.length !== 0) failures.push(`${name}: ${field} is not empty`); + } + const shift = observations?.["layoutShift"]; + if (typeof shift !== "number" || !Number.isFinite(shift)) failures.push(`${name}: layout shift is missing`); + else { maxLayoutShift = Math.max(maxLayoutShift, shift); if (shift > 0.1) failures.push(`${name}: layout shift ${shift} exceeds 0.1`); } + for (const sectionValue of list(observations?.["sections"])) { + const section = record(sectionValue); + const utilization = section?.["utilization"]; + if (typeof utilization === "number") { + visualUtilizations.push(utilization); + if (utilization < 0.5) failures.push(`${name}: primary visual uses only ${utilization} of its card`); + } + } + for (const geometryValue of list(observations?.["chartGeometry"])) { + const geometry = record(geometryValue); + const width = geometry?.["width"]; + const childWidth = geometry?.["childWidth"]; + if (typeof width !== "number" || typeof childWidth !== "number" || width <= 0) failures.push(`${name}: chart geometry is invalid`); + else { + const fill = childWidth / width; + chartFills.push(fill); + if (fill < 0.5) failures.push(`${name}: chart fills only ${fill.toFixed(3)} of its container`); + } + } + if (keyboard?.["firstTab"] === undefined) failures.push(`${name}: keyboard trace is missing`); + const id = name.replace(/-(?:desktop|mobile)\.json$/, ""); + if ((id === "system-explainer" || id === "findings-table") && keyboard?.["table"] === undefined) failures.push(`${name}: table keyboard trace is missing`); + if (id === "interactive-decision") { + const range = record(keyboard?.["range"]); + if (range?.["before"] === range?.["after"] || keyboard?.["copy"] !== "Copied") failures.push(`${name}: tune/copy keyboard trace failed`); + } + if (id === "pr-walkthrough" && keyboard?.["copy"] !== "Copied") failures.push(`${name}: verdict copy keyboard trace failed`); + } + const evidence = reports.filter((report) => expected.includes(report.name)).map((report) => ({ + report: report.name, + sha256: createHash("sha256").update(report.bytes).digest("hex"), + })).sort((a, b) => a.report.localeCompare(b.report)); + return { + schemaVersion: 1, + corpusId: "page-quality-v1", + cells: expected.length, + passed: Math.max(0, expected.length - new Set(failures.map((failure) => failure.split(":")[0])).size), + failed: new Set(failures.map((failure) => failure.split(":")[0])).size, + maxUsefulContentMs, + maxLayoutShift, + minPrimaryVisualUtilization: visualUtilizations.length === 0 ? null : Math.min(...visualUtilizations), + minChartFill: chartFills.length === 0 ? null : Math.min(...chartFills), + failures, + evidence, + }; +} + +async function main(): Promise { + const directory = resolve(process.argv[2] ?? "docs/evidence/page-quality/2026-08-18-local-composition"); + const output = resolve(process.argv[3] ?? `${directory}/summary.json`); + const names = (await readdir(directory)).filter((name) => name.endsWith(".json") && name !== basename(output)); + const reports = await Promise.all(names.map(async (name) => { + const bytes = await readFile(resolve(directory, name)); + return { name, bytes, value: JSON.parse(bytes.toString("utf8")) as unknown }; + })); + const summary = summarizeLocalCompositionReports(reports); + await writeFile(output, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + if (summary.failures.length > 0) { + for (const failure of summary.failures) console.error(`FAIL - ${failure}`); + process.exitCode = 1; + } else console.log(`ok - ${summary.passed}/${summary.cells} local composition browser cells passed`); +} + +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) await main(); diff --git a/scripts/support-browser-smoke.ts b/scripts/support-browser-smoke.ts new file mode 100644 index 0000000..ad8f8a5 --- /dev/null +++ b/scripts/support-browser-smoke.ts @@ -0,0 +1,272 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { lstat, readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +interface WebDriverResponse { value: T } + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : undefined; +} + +export interface BrowserRequest { + browserName: "chrome" | "firefox" | "safari"; + browserVersion?: string; + platformName?: string; + headless: boolean; +} + +function assertSafeExtraCapabilities(value: unknown, path = "extra capabilities"): asserts value is Record { + const detail = record(value); + if (!detail) throw new Error(`${path} must be an object`); + for (const [key, item] of Object.entries(detail)) { + if (/(?:access.?key|api.?key|token|secret|password|credential|username)/i.test(key)) { + throw new Error(`${path}.${key} looks credential-bearing; put provider authentication in WEBDRIVER_ENDPOINT`); + } + if (record(item)) assertSafeExtraCapabilities(item, `${path}.${key}`); + else if (Array.isArray(item)) { + for (let index = 0; index < item.length; index++) if (record(item[index])) { + assertSafeExtraCapabilities(item[index], `${path}.${key}[${index}]`); + } + } + } +} + +export function buildWebDriverCapabilities( + request: BrowserRequest, + extraCapabilities: Record = {}, +): Record { + if (request.headless && request.browserName === "safari") { + throw new Error("Safari does not expose a standard headless WebDriver mode"); + } + assertSafeExtraCapabilities(extraCapabilities); + for (const key of ["browserName", "browserVersion", "platformName", "acceptInsecureCerts", "goog:chromeOptions", "moz:firefoxOptions"]) { + if (key in extraCapabilities) throw new Error(`extra capabilities cannot override ${key}`); + } + const alwaysMatch: Record = { + ...extraCapabilities, + browserName: request.browserName, + acceptInsecureCerts: false, + }; + if (request.browserVersion !== undefined) alwaysMatch["browserVersion"] = request.browserVersion; + if (request.platformName !== undefined) alwaysMatch["platformName"] = request.platformName; + if (request.headless && request.browserName === "chrome") { + alwaysMatch["goog:chromeOptions"] = { args: ["--headless=new", "--disable-dev-shm-usage"] }; + } + if (request.headless && request.browserName === "firefox") { + alwaysMatch["moz:firefoxOptions"] = { args: ["-headless"] }; + } + return { capabilities: { alwaysMatch } }; +} + +export function redactedTarget(value: string): string { + const url = new URL(value); + if (url.protocol === "data:" || url.protocol === "blob:") return `${url.protocol}`; + if (url.protocol === "file:") return "file://"; + url.username = ""; + url.password = ""; + if (url.search !== "") url.search = "?redacted"; + url.hash = ""; + return url.href; +} + +export function supportBrowserFailures(value: unknown): string[] { + const failures: string[] = []; + const report = record(value); + if (!report) return ["report must be an object"]; + if (report["schemaVersion"] !== 1) failures.push("schemaVersion must be 1"); + if (report["ready"] !== true) failures.push("page did not settle"); + const observations = record(report["observations"]); + if (!observations) failures.push("observations are missing"); + else { + for (const field of ["documentHorizontalOverflow", "mainHorizontalOverflow"] as const) { + if (observations[field] !== false) failures.push(field); + } + for (const field of ["clippedText", "renderErrors"] as const) { + if (!Array.isArray(observations[field]) || observations[field].length !== 0) failures.push(`${field} is not empty`); + } + } + const keyboard = record(report["keyboard"]); + if (!keyboard || !record(keyboard["firstTab"])) failures.push("first keyboard target is missing"); + if (!Array.isArray(report["externalHttpRequests"]) || report["externalHttpRequests"].length !== 0) { + failures.push("externalHttpRequests is not empty"); + } + const screenshot = record(report["screenshot"]); + if (!screenshot || typeof screenshot["sha256"] !== "string" || !/^[a-f0-9]{64}$/.test(screenshot["sha256"])) { + failures.push("screenshot digest is missing"); + } + return failures; +} + +function argument(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +async function absent(path: string): Promise { + try { + await lstat(path); + throw new Error(`refusing to overwrite evidence file ${path}`); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return; + throw error; + } +} + +async function main(): Promise { + const endpoint = process.env["WEBDRIVER_ENDPOINT"] ?? "http://127.0.0.1:4444"; + const pageUrl = argument("--url"); + const reportPath = argument("--report"); + const screenshotPath = argument("--screenshot"); + const browserName = argument("--browser"); + const browserVersion = argument("--browser-version"); + const platformName = argument("--platform"); + const width = Number(argument("--width")); + const height = Number(argument("--height")); + const headless = process.argv.includes("--headless"); + if (!pageUrl || !reportPath || !screenshotPath || + (browserName !== "chrome" && browserName !== "firefox" && browserName !== "safari") || + !Number.isInteger(width) || !Number.isInteger(height) || width < 320 || height < 480) { + throw new Error("usage: support-browser-smoke --url --report --screenshot --browser --width --height [--browser-version ] [--platform ] [--headless]"); + } + const resolvedReport = resolve(reportPath); + const resolvedScreenshot = resolve(screenshotPath); + await absent(resolvedReport); + await absent(resolvedScreenshot); + let extraCapabilities: Record = {}; + const extraCapabilitiesPath = process.env["WEBDRIVER_CAPABILITIES_FILE"]; + if (extraCapabilitiesPath !== undefined) { + const bytes = await readFile(resolve(extraCapabilitiesPath)); + if (bytes.byteLength > 64 * 1024) throw new Error("WebDriver capabilities file exceeds 64 KiB"); + const parsed = JSON.parse(bytes.toString("utf8")) as unknown; + assertSafeExtraCapabilities(parsed); + extraCapabilities = parsed; + } + const base = endpoint.replace(/\/$/, ""); + async function request(method: string, path: string, body?: unknown): Promise { + const response = await fetch(`${base}${path}`, { + method, + headers: body === undefined ? undefined : { "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const responseText = await response.text(); + if (!response.ok) throw new Error(`WebDriver ${method} ${path} failed (${response.status}): ${responseText.slice(0, 1000)}`); + return (JSON.parse(responseText) as WebDriverResponse).value; + } + + const session = await request<{ sessionId: string; capabilities?: Record }>( + "POST", + "/session", + buildWebDriverCapabilities({ browserName, browserVersion, platformName, headless }, extraCapabilities), + ); + const route = `/session/${session.sessionId}`; + async function script(source: string): Promise { + return request("POST", `${route}/execute/sync`, { script: source, args: [] }); + } + async function key(value: string): Promise { + await request("POST", `${route}/actions`, { actions: [{ + type: "key", id: "keyboard", actions: [{ type: "keyDown", value }, { type: "keyUp", value }], + }] }); + } + async function focus(selector: string): Promise { + return script(`const n=document.querySelector(${JSON.stringify(selector)});if(n)n.focus();return !!n;`); + } + + try { + await request("POST", `${route}/window/rect`, { width, height, x: 0, y: 0 }); + const startedAt = Date.now(); + await request("POST", `${route}/url`, { url: pageUrl }); + let ready = false; + for (let attempt = 0; attempt < 160; attempt++) { + ready = await script("return document.readyState==='complete'&&!document.querySelector('.component[data-component-index]')&&Array.from(document.querySelectorAll('.chart')).every(n=>n.querySelector('canvas,svg'))&&Array.from(document.querySelectorAll('pre.mermaid')).every(n=>n.querySelector('svg'));"); + if (ready) break; + await new Promise((resolveWait) => setTimeout(resolveWait, 50)); + } + const usefulContentMs = Date.now() - startedAt; + await new Promise((resolveWait) => setTimeout(resolveWait, 300)); + await script("document.activeElement&&document.activeElement.blur();return true;"); + await key("\uE004"); + const keyboard: Record = { + firstTab: await script("return {tag:document.activeElement?.tagName||null,className:document.activeElement?.className||null,text:document.activeElement?.textContent?.trim()||null};"), + }; + if (await focus(".decision-opt")) { + await key("\uE014"); + keyboard["decision"] = await script("const n=document.querySelector('.decision-opt[aria-checked=\"true\"]');return {selected:n?.dataset.option||null,active:document.activeElement?.dataset.option||null};"); + } + if (await focus(".th-sort")) { + await key("\uE007"); + keyboard["table"] = await script("const n=document.querySelector('th[aria-sort]:not([aria-sort=\"none\"])');return {direction:n?.getAttribute('aria-sort')||null,label:n?.textContent?.trim()||null};"); + } + if (await focus('input[type="range"]')) { + const before = await script("return document.activeElement?.value||'';"); + await key("\uE014"); + const after = await script("return document.activeElement?.value||'';"); + keyboard["range"] = { before, after }; + } + if (await focus(".copy-btn")) { + await key("\uE007"); + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); + keyboard["copy"] = await script("return document.querySelector('.copy-note')?.textContent?.trim()||'';"); + } + const observations = await script>(` + const main=document.querySelector('main'); + return { + documentHorizontalOverflow:document.documentElement.scrollWidth>document.documentElement.clientWidth, + mainHorizontalOverflow:!!main&&main.scrollWidth>main.clientWidth, + clippedText:Array.from(document.querySelectorAll('h1,h2,h3,.stat-label,.finding-title,.pill,.decision-label,figcaption,th,td')).flatMap((node)=>{const style=getComputedStyle(node);return node.scrollWidth>node.clientWidth+1&&style.overflow!=='visible'?[node.textContent?.trim()?.slice(0,120)||node.tagName]:[];}), + renderErrors:Array.from(document.querySelectorAll('.chart-error,.asset-error')).map((node)=>node.textContent?.trim()), + headings:Array.from(document.querySelectorAll('h1,h2,h3')).map((node)=>node.textContent?.trim()), + landmarks:{header:document.querySelectorAll('header').length,main:document.querySelectorAll('main').length,footer:document.querySelectorAll('footer').length}, + interactive:{buttons:document.querySelectorAll('button').length,ranges:document.querySelectorAll('input[type="range"]').length,radios:document.querySelectorAll('[role="radio"]').length}, + resourceUrls:performance.getEntriesByType('resource').map((entry)=>entry.name), + userAgent:navigator.userAgent + }; + `); + const screenshotBytes = Buffer.from(await request("GET", `${route}/screenshot`), "base64"); + const resourceUrls = Array.isArray(observations["resourceUrls"]) + ? observations["resourceUrls"].filter((item): item is string => typeof item === "string") + : []; + const pageOrigin = new URL(pageUrl).origin; + const externalHttpRequests = resourceUrls.filter((url) => /^https?:/i.test(url) && new URL(url).origin !== pageOrigin).map(redactedTarget); + const safeCapabilities = { + browserName: session.capabilities?.["browserName"] ?? browserName, + browserVersion: session.capabilities?.["browserVersion"] ?? browserVersion ?? "provider-selected", + platformName: session.capabilities?.["platformName"] ?? platformName ?? "provider-selected", + }; + const report: Record = { + schemaVersion: 1, + capturedAt: new Date().toISOString(), + result: "pending-assessment", + request: { browserName, browserVersion: browserVersion ?? null, platformName: platformName ?? null, width, height, headless }, + environment: safeCapabilities, + target: redactedTarget(pageUrl), + ready, + usefulContentMs, + keyboard, + observations: { ...observations, resourceUrls: resourceUrls.map(redactedTarget) }, + externalHttpRequests, + screenshot: { path: screenshotPath, sha256: createHash("sha256").update(screenshotBytes).digest("hex") }, + limitations: [ + "Standard WebDriver does not provide a portable accessibility tree or console-log endpoint.", + "This automated observation is not a support claim and requires the declared manual, assistive-technology, and human first-use evidence.", + ], + }; + const failures = supportBrowserFailures(report); + report["result"] = failures.length === 0 ? "pass" : "fail"; + report["failures"] = failures; + await writeFile(resolvedScreenshot, screenshotBytes, { flag: "wx", mode: 0o600 }); + await writeFile(resolvedReport, `${JSON.stringify(report, null, 2)}\n`, { encoding: "utf8", flag: "wx", mode: 0o600 }); + if (failures.length > 0) { + for (const failure of failures) console.error(`FAIL - ${failure}`); + process.exitCode = 1; + } else console.log(`ok - ${browserName} portable-page browser smoke passed`); + } finally { + await request("DELETE", route); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) await main(); diff --git a/skills/artifact-pages/reference/components.md b/skills/artifact-pages/reference/components.md index bb0dfb7..b77d707 100644 --- a/skills/artifact-pages/reference/components.md +++ b/skills/artifact-pages/reference/components.md @@ -4,7 +4,9 @@ The `artifact_publish` tool takes `markdown`. Frontmatter (`---` fences) sets `t `icon:` (emoji favicon), `description:` (gallery subtitle), `lang:` (BCP 47), `dir:` (`ltr|rtl`), `locale:` (Intl locale), and `timezone:` (IANA zone). Locale defaults to `en-US`, time zone to `UTC`, and direction is inferred from the language when omitted. `##` -sections become white cards on the page. Republish with the same title to update in place; `version: true` keeps +sections become cards on the page. Optional `composition:` is one of `standard`, `narrative`, +`dashboard`, `split`, `dense`, `quiet`, or `full`; it changes bounded layout only and never +reading order. Republish with the same title to update in place; `version: true` keeps numbered history; `open: true` opens the browser; `expectedHash` guards against overwriting unseen changes; `force: true` overrides the credential scan. @@ -23,6 +25,7 @@ unseen changes; `force: true` overrides the credential scan. | ```` ```mermaid ```` | first line `%% summary: meaningful equivalent`, then raw Mermaid | diagrams: graph/sequence/ER/... | | ```` ```decisions ```` | `{title?, questions: [{id, question, options: [{id, label, note?}]}]}` | workshop rows; answers read back via `artifact_state` | | ```` ```table ```` | `{caption, columns: [{key, label, type?: num\|date\|datetime}], rows: [{...}]}` | captioned sortable/filterable tables; dates require zoned ISO values | +| ```` ```frame ```` | `{kind: mockup\|media\|code, title, caption, content, annotations?: string[]}`; content max 8 KiB | bounded mockup/media/code framing with a text equivalent | ## Data honesty (non-negotiable) diff --git a/specs/changes/blinded-page-quality-benchmark/change.json b/specs/changes/blinded-page-quality-benchmark/change.json new file mode 100644 index 0000000..09141e9 --- /dev/null +++ b/specs/changes/blinded-page-quality-benchmark/change.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 1, + "id": "blinded-page-quality-benchmark", + "title": "Run the blinded page-quality benchmark", + "lane": "standard", + "status": "approved", + "affectedRequirements": [ + "RENDER-12", + "QUAL-07" + ], + "currentSpecs": [ + "specs/current/portable-renderer.spec.md" + ], + "currentSpecsUpdated": false, + "approval": { + "by": "aaron.zeng", + "at": "2026-08-18T15:22:56Z" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-18", + "archivedAt": null +} diff --git a/specs/changes/blinded-page-quality-benchmark/delta.md b/specs/changes/blinded-page-quality-benchmark/delta.md new file mode 100644 index 0000000..8630172 --- /dev/null +++ b/specs/changes/blinded-page-quality-benchmark/delta.md @@ -0,0 +1,45 @@ +# Specification delta: Run the blinded page-quality benchmark + +## MODIFIED + +### Requirement: RENDER-12 + +An equal-or-better claim requires a current, authorized, same-input run of all eight frozen tasks +with at least three independent generations per system/task, all hard gates passing, and a blinded +panel of at least three eligible reviewers meeting every absolute and comparative threshold. + +#### Scenario: Normal behavior +- **Given:** complete authorized captures and eligible blinded reviews +- **When:** the canonical aggregation runs +- **Then:** at least 80% pairs are OpenCode equivalent/better, no family loses a reviewer majority, and every OpenCode dimension median is at least Claude and 4/5 + +#### Scenario: Failure or refusal +- **Given:** any hard-gate failure, threshold miss, cherry-picked run, missing reviewer role, or unblinded pair +- **When:** claim status is computed +- **Then:** equal-or-better is refused and the full failure distribution remains visible + +#### Scenario: Relevant boundary +- **Given:** a complete local corpus but absent current authorized Claude output or reviewers +- **When:** status is reported +- **Then:** the comparative claim remains blocked/unverified without substituting synthetic judgments + +### Requirement: QUAL-07 + +The dated benchmark retains prompts, hashes, all required runs, environment and authorization +metadata, interaction traces, randomization, hard gates, pair choices, dimension distributions, +reviewer eligibility, failures, and confidence limits without exposing forbidden private content. + +#### Scenario: Normal behavior +- **Given:** a benchmark run and reviewer panel +- **When:** evidence validation completes +- **Then:** every required generation and score maps exactly once to the frozen corpus and report denominator + +#### Scenario: Failure or refusal +- **Given:** missing runs, duplicate scores, unverifiable settings, label leakage, or unauthorized retention +- **When:** the evidence record is checked +- **Then:** the run/panel is invalid, the issue is reported, and no claim is emitted + +#### Scenario: Relevant boundary +- **Given:** reference contents cannot be committed but digest/aggregate retention is authorized +- **When:** the report is produced +- **Then:** protected content stays access-controlled while git retains sufficient authorized metadata and hashes to audit the decision diff --git a/specs/changes/blinded-page-quality-benchmark/evidence.md b/specs/changes/blinded-page-quality-benchmark/evidence.md new file mode 100644 index 0000000..23b697f --- /dev/null +++ b/specs/changes/blinded-page-quality-benchmark/evidence.md @@ -0,0 +1,21 @@ +# Evidence: Run the blinded page-quality benchmark + +## Requirement: RENDER-12 + +- Validation: the equal-or-better claim requires 24 unselected outputs per system and at least + three eligible blinded reviewers across all eight task families. +- Verification: aggregation, hard-gate, threshold, denominator, blinding, and mutation tests pass; + seed-bound preparation refuses short seeds, existing mappings, incomplete matrices, and output + overwrite while emitting a separate identity-free reviewer packet. The dated status reports + zero authorized Claude runs, zero comparison OpenCode runs, and zero reviewers. +- Result: incomplete; equal-or-better remains disabled without substituting generated judgments. +- Evidence: [@test](test/page-quality-benchmark.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-benchmark-status.md) + +## Requirement: QUAL-07 + +- Validation: settings, authority, retention, hashes, traces, labels, scores, and failures must map + exactly once to the frozen corpus. +- Verification: the fail-closed validator rejects missing, duplicate, leaked, unauthorized, or + incomplete inputs and accepts the empty external-run template only as incomplete status. +- Result: harness pass; evidence collection is not authorized and the comparative report is absent. +- Evidence: [@test](test/page-quality-benchmark.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-benchmark-status.md) diff --git a/specs/changes/blinded-page-quality-benchmark/proposal.md b/specs/changes/blinded-page-quality-benchmark/proposal.md new file mode 100644 index 0000000..7b01813 --- /dev/null +++ b/specs/changes/blinded-page-quality-benchmark/proposal.md @@ -0,0 +1,40 @@ +# Proposal: Run the blinded page-quality benchmark + +## Outcome + +A dated report preserves every required same-input generation, hard-gate result, blinded score +distribution, confidence interval, failure, and authorization fact, and permits an equal-or- +better claim only when every canonical threshold passes. + +## Context + +Renderer regression screenshots cannot establish comparative quality. The normative benchmark +requires current authenticated Claude Code Artifact references, at least three independent +generations per system and task, randomized/blinded pairs, at least three reviewers including +design/UX and technical perspectives, and both absolute and comparative thresholds. + +## Scope + +- In scope: capture/run manifests, authorization and retention metadata, deterministic + randomization, reviewer packets, interaction traces, hard-gate automation, complete score + aggregation, confidence limits, report generation, losing-archetype iteration, and claim + gating against the frozen corpus. +- Out of scope: acquiring or using an account without explicit authority, committing private + reference contents without retention permission, CI credentials, selecting best runs, + automated judges as deciding reviewers, or lowering a failed threshold. + +## Risks and rollback + +- Risk: labels may leak, settings may be incomparable, reviewers may be conflicted, or retained + outputs may violate subscription or redistribution terms. +- Rollback: invalidate the affected run or panel, retain the failure visibly, withdraw any + claim, and rerun only after the authorization/settings/retention problem is resolved. + +## Validation plan + +Synthetic fixtures prove manifest completeness, blinding, deterministic randomization, +denominators, medians, majority logic, confidence calculations, and fail-closed claims. Real +verification remains blocked until a human supplies scoped Claude execution authority, +plan/model/settings metadata, output-retention permission, and at least three eligible +independent reviewers. All 48 generations per system (eight tasks times three runs across the +two required viewports/modes as captured states) and all failures remain in the dated record. diff --git a/specs/changes/blinded-page-quality-benchmark/tasks.md b/specs/changes/blinded-page-quality-benchmark/tasks.md new file mode 100644 index 0000000..a6d3be4 --- /dev/null +++ b/specs/changes/blinded-page-quality-benchmark/tasks.md @@ -0,0 +1,10 @@ +# Tasks: Run the blinded page-quality benchmark + +- [x] Confirm proposal validation and human approval. +- [ ] Obtain explicit Claude account execution authority, settings protocol, and retention disposition. +- [x] Freeze manifests, deterministic blinding/randomization, aggregation, confidence, and claim checks. +- [ ] Capture every required OpenCode and Claude generation and interaction trace without cherry-picking. +- [ ] Run automated/manual hard gates and retain all failures. +- [ ] Recruit at least three eligible independent reviewers and complete blinded scoring. +- [ ] Fix losing archetypes and rerun the complete affected benchmark until every threshold passes. +- [ ] Publish the dated distribution report, update current truth, and archive. diff --git a/specs/changes/first-use-comprehension-certification/change.json b/specs/changes/first-use-comprehension-certification/change.json new file mode 100644 index 0000000..9398290 --- /dev/null +++ b/specs/changes/first-use-comprehension-certification/change.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "id": "first-use-comprehension-certification", + "title": "Certify first-use comprehension", + "lane": "standard", + "status": "approved", + "affectedRequirements": [ + "OUT-02", + "OUT-03", + "OUT-05", + "COMPAT-01", + "QUAL-08" + ], + "currentSpecs": [ + "specs/current/journey-evidence.spec.md", + "specs/current/governance-policy.spec.md" + ], + "currentSpecsUpdated": false, + "approval": { + "by": "aaron.zeng", + "at": "2026-08-18T15:22:56Z" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-18", + "archivedAt": null +} diff --git a/specs/changes/first-use-comprehension-certification/delta.md b/specs/changes/first-use-comprehension-certification/delta.md new file mode 100644 index 0000000..330bed2 --- /dev/null +++ b/specs/changes/first-use-comprehension-certification/delta.md @@ -0,0 +1,106 @@ +# Specification delta: Certify first-use comprehension + +## MODIFIED + +### Requirement: OUT-02 + +A local-artifact-core candidate passes first use only when at least one eligible first-time +participant on every exact claimed supported clean-machine cell installs the exact package, +creates an offline artifact, and reopens it using only the README within 600 seconds. + +#### Scenario: Normal behavior +- **Given:** a claimed cell and an eligible unassisted participant +- **When:** install/create/reopen finishes within 600 seconds on the exact candidate +- **Then:** that participant supplies the cell's first-use pass + +#### Scenario: Failure or refusal +- **Given:** assistance, checkout/account use, wrong bytes, timeout, or failed creation/reopen +- **When:** the result is summarized +- **Then:** it is retained as a failure/exclusion and cannot cover the cell + +#### Scenario: Relevant boundary +- **Given:** one or more target claimed cells have no eligible passing participant +- **When:** certification is evaluated +- **Then:** OUT-02 is incomplete or failed and local artifact core is not certified + +### Requirement: OUT-03 + +At least ten eligible representative primary users independently identify purpose, primary +finding/state, provenance, and next action from frozen corpus pages within 60 seconds; at least +90% must pass all four fields without assistance. + +#### Scenario: Normal behavior +- **Given:** ten or more eligible complete participant records +- **When:** at least 90% pass all four rubric fields within the time limit +- **Then:** the comprehension outcome passes with exact counts and distribution + +#### Scenario: Failure or refusal +- **Given:** fewer than 90% pass, an answer is missing, or assistance occurred +- **When:** aggregation runs +- **Then:** the affected record fails/excludes as specified and the result is not rounded or imputed + +#### Scenario: Relevant boundary +- **Given:** fewer than ten eligible complete primary records +- **When:** status is computed +- **Then:** OUT-03 is incomplete rather than pass, fail, or not applicable + +### Requirement: OUT-05 + +The study uses affirmative consent and minimized private records with pseudonymous codes, +bounded outcome fields, withdrawal, access control, aggregate-only git retention, and no +effect on product functionality for declining participants. + +#### Scenario: Normal behavior +- **Given:** an informed participant consents before timing +- **When:** the study completes +- **Then:** the private record and anonymous aggregate follow the approved schema and retention policy + +#### Scenario: Failure or refusal +- **Given:** consent is absent/withdrawn or a direct identity/private artifact appears +- **When:** validation or study operation encounters it +- **Then:** collection stops or the record is rejected/deleted without entering any denominator + +#### Scenario: Relevant boundary +- **Given:** an aggregate has been accepted and the raw retention deadline arrives +- **When:** the study owner performs deletion +- **Then:** raw records are deleted while the non-identifying aggregate and digest remain + +### Requirement: COMPAT-01 + +Every local-artifact-core support cell records exact OS, Node, OpenCode, browser/device, +candidate digest, technical scope, and human first-use result; missing or stale cells stay unverified. + +#### Scenario: Normal behavior +- **Given:** the complete target support matrix +- **When:** exact technical and human evidence passes for a cell +- **Then:** only that dated exact cell may be promoted to supported + +#### Scenario: Failure or refusal +- **Given:** a cell fails or combines evidence from different candidate bytes +- **When:** support status is resolved +- **Then:** it is unsupported and blocks a broader certification claim + +#### Scenario: Relevant boundary +- **Given:** Goal 2 made macOS/Windows write-filesystem evidence optional for an earlier gate +- **When:** local-artifact-core certification is evaluated +- **Then:** that exception is not inherited and all cells required by COMPAT-01 remain mandatory + +### Requirement: QUAL-08 + +The dated aggregate and support report display participant failures, exclusions, withdrawals, +missing cells, stale evidence, and uncollected evidence as prominently as successful results. + +#### Scenario: Normal behavior +- **Given:** completed study and platform records +- **When:** the public aggregate is generated +- **Then:** exact denominators, passes, failures, exclusions, cells, versions, and gaps are visible + +#### Scenario: Failure or refusal +- **Given:** a report omits a failure or labels missing evidence not applicable without reason +- **When:** claim validation runs +- **Then:** the report and certification decision fail + +#### Scenario: Relevant boundary +- **Given:** no human study owner, participants, or platform access is available +- **When:** status is published +- **Then:** evidence remains explicitly uncollected/incomplete and no human or support claim is made diff --git a/specs/changes/first-use-comprehension-certification/evidence.md b/specs/changes/first-use-comprehension-certification/evidence.md new file mode 100644 index 0000000..b67a429 --- /dev/null +++ b/specs/changes/first-use-comprehension-certification/evidence.md @@ -0,0 +1,42 @@ +# Evidence: Certify first-use comprehension + +## Requirement: OUT-02 + +- Validation: every claimed support cell needs one eligible unassisted first-time README-only pass. +- Verification: validator and aggregate tests enforce exact bytes/cell, eligibility, no assistance, + 600-second create/reopen, failure retention, and incomplete missing cells. +- Result: incomplete: zero eligible sessions and every target cell is missing. +- Evidence: [@test](test/journey-study.test.ts) [@manual](docs/evidence/journeys/goal-5-status-2026-08-19.md) + +## Requirement: OUT-03 + +- Validation: at least ten eligible representative-primary users and a 90% all-four-fields pass rate are mandatory. +- Verification: aggregation tests reject rounding, imputation, incomplete fields, assistance, + ineligible records, and denominators below ten. +- Result: incomplete: zero of ten required eligible participants. +- Evidence: [@test](test/journey-study.test.ts) [@manual](docs/evidence/journeys/goal-5-status-2026-08-19.md) + +## Requirement: OUT-05 + +- Validation: consent, minimization, withdrawal, access control, and aggregate-only git retention + are prerequisites to collection. +- Verification: schema tests reject direct identity, unknown/unbounded fields, missing consent, + withdrawn records, and private data in aggregates; participant materials are frozen. +- Result: protocol pass; collection has not started because no study owner/recruitment is authorized. +- Evidence: [@test](test/journey-study.test.ts) [@manual](docs/journeys/goal-5-participant-materials.md) + +## Requirement: COMPAT-01 + +- Validation: technical and human evidence must bind exact candidate bytes in every claimed cell. +- Verification: three exact-tarball OS command-line observations are retained and explicitly + excluded from support; Windows 11/WSL, browser, mobile, and first-use cells remain absent. +- Result: incomplete with zero supported complete cells. +- Evidence: [@manual](docs/evidence/releases/2026-08-19-local-core-ci.md) [@manual](docs/support-policy.md) + +## Requirement: QUAL-08 + +- Validation: missing humans, cells, failures, exclusions, and withdrawals must remain visible. +- Verification: the dated status records exact zero denominators and distinguishes automated + technical observations from participants. +- Result: pass for fail-closed disclosure; the underlying outcomes remain incomplete. +- Evidence: [@test](test/journey-study.test.ts) [@manual](docs/evidence/journeys/goal-5-status-2026-08-19.md) diff --git a/specs/changes/first-use-comprehension-certification/proposal.md b/specs/changes/first-use-comprehension-certification/proposal.md new file mode 100644 index 0000000..d331d61 --- /dev/null +++ b/specs/changes/first-use-comprehension-certification/proposal.md @@ -0,0 +1,41 @@ +# Proposal: Certify first-use comprehension + +## Outcome + +Consented representative-user evidence demonstrates both README-only first use on every exact +claimed support cell and one-minute artifact comprehension at the normative denominator and +threshold, while raw records remain private, minimized, withdrawable, and time-bounded. + +## Context + +The approved journey protocol and validator exist, but the retained baseline has zero real +participants. Synthetic agent probes cannot satisfy OUT-02 or OUT-03. Local artifact core also +cannot inherit Goal 2's optional macOS/Windows evidence decision: every platform claimed by a +certified release needs an exact dated covered pass. + +## Scope + +- In scope: freeze the candidate and corpus; recruit eligible primary users; consent, execute, + validate, summarize, and retain non-identifying aggregates; resolve exact OS/Node/OpenCode/ + browser cells; record every failure/exclusion/withdrawal; update support and outcome status. +- Out of scope: default telemetry, identities or recordings in git, maintainer/model records in + denominators, coaching, rounded/imputed scores, declaring missing cells not applicable, or + recruiting participants without an authorized study owner. + +## Risks and rollback + +- Risk: privacy breach, biased recruitment, stale candidate bytes, incomplete platform + coverage, or a failed comprehension/first-use outcome. +- Rollback: stop collection, honor withdrawal and deletion immediately, mark the affected + result incomplete/failed, narrow the claimed support set only if the normative target still + permits it, and keep the release uncertified until a valid rerun passes. + +## Validation plan + +The existing strict schema receives boundary tests for consent, eligibility, duplicate and +conflicted users, exact release/platform hashes, all four rubric answers, timing, withdrawal, +and aggregate redaction. Human evidence requires at least ten eligible representative primary +users with at least 90% passing all four comprehension fields in 60 seconds, plus at least one +eligible first-time README-only install/create/reopen pass within 600 seconds for every exact +claimed support cell. Raw records stay access-controlled and are deleted under the approved +30-day policy. diff --git a/specs/changes/first-use-comprehension-certification/tasks.md b/specs/changes/first-use-comprehension-certification/tasks.md new file mode 100644 index 0000000..ed166fd --- /dev/null +++ b/specs/changes/first-use-comprehension-certification/tasks.md @@ -0,0 +1,9 @@ +# Tasks: Certify first-use comprehension + +- [x] Confirm proposal validation and human approval. +- [ ] Name the authorized study owner, exact candidate/corpus, target cells, and recruitment plan. +- [x] Extend validator/aggregate boundary tests and freeze participant-facing materials. +- [ ] Run consented README-only first-use sessions for every exact claimed support cell. +- [ ] Run at least ten eligible representative-primary comprehension sessions. +- [ ] Validate private records, honor withdrawals, and publish only the non-identifying aggregate/digest. +- [ ] Update outcome/support status with all failures, exclusions, and missing cells, then archive. diff --git a/specs/changes/local-artifact-core-release/change.json b/specs/changes/local-artifact-core-release/change.json new file mode 100644 index 0000000..68119aa --- /dev/null +++ b/specs/changes/local-artifact-core-release/change.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "id": "local-artifact-core-release", + "title": "Certify the local artifact core release", + "lane": "high-risk", + "status": "approved", + "affectedRequirements": [ + "OUT-04", + "COMPAT-01", + "DIST-03", + "DIST-04", + "QUAL-08" + ], + "currentSpecs": [ + "specs/current/contract-integrity.spec.md", + "specs/current/governance-policy.spec.md", + "specs/current/journey-evidence.spec.md", + "specs/current/opencode-integration.spec.md", + "specs/current/portable-renderer.spec.md" + ], + "currentSpecsUpdated": false, + "approval": { + "by": "aaron.zeng", + "at": "2026-08-18T15:22:56Z" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-18", + "archivedAt": null +} diff --git a/specs/changes/local-artifact-core-release/delta.md b/specs/changes/local-artifact-core-release/delta.md new file mode 100644 index 0000000..97f3b78 --- /dev/null +++ b/specs/changes/local-artifact-core-release/delta.md @@ -0,0 +1,103 @@ +# Specification delta: Certify the local artifact core release + +## MODIFIED + +### Requirement: OUT-04 + +Local artifact core is a certified level only when one immutable candidate passes accumulated +Phases 0-3 and every applicable recurring gate; public-preview evidence grants no waiver. + +#### Scenario: Normal behavior +- **Given:** a candidate with every applicable requirement passing on identical bytes +- **When:** accountable owners sign the release decision +- **Then:** the candidate may be labeled local artifact core + +#### Scenario: Failure or refusal +- **Given:** any required phase, evidence row, or external prerequisite fails or is missing +- **When:** the decision is evaluated +- **Then:** certification is refused without mutating a registry or provider + +#### Scenario: Relevant boundary +- **Given:** an earlier public preview has valid attestations +- **When:** a new certification candidate is assessed +- **Then:** only candidate-applicable evidence is reused and byte-bound gates rerun + +### Requirement: COMPAT-01 + +The certified candidate passes the exact target Node, OpenCode, OS, desktop-browser, and mobile- +browser support matrix; diagnostic observations and unverified cells do not become support. + +#### Scenario: Normal behavior +- **Given:** exact current target versions resolved on the test date +- **When:** clean candidate runs and first-use evidence pass every required cell +- **Then:** the release record names those supported cells and triggers for revalidation + +#### Scenario: Failure or refusal +- **Given:** a target cell is missing, stale, failed, or tied to different bytes +- **When:** certification is computed +- **Then:** the support gate and local-core decision fail + +#### Scenario: Relevant boundary +- **Given:** current/previous or latest-two labels advance before release +- **When:** the candidate is finalized +- **Then:** labels are resolved again to exact versions and affected cells rerun + +### Requirement: DIST-03 + +The candidate has aligned SemVer/tag intent, Conventional Commit history, checked release notes, +migrations, limits/failures, capability/certification label, and exact linked evidence before any release mutation. + +#### Scenario: Normal behavior +- **Given:** a complete certified candidate record +- **When:** claim consistency is checked +- **Then:** package, README, support, privacy, migration, and release surfaces agree on local artifact core + +#### Scenario: Failure or refusal +- **Given:** claims conflict, notes/evidence are incomplete, or the candidate version is inconsistent +- **When:** the release gate runs +- **Then:** release and certification are refused with the mismatch identified + +#### Scenario: Relevant boundary +- **Given:** the candidate passes locally but no explicit tag/publish authority exists +- **When:** verification completes +- **Then:** the decision may be retained but no tag, npm publish, or provider mutation occurs + +### Requirement: DIST-04 + +SBOM, audit, license, integrity, signature/provenance inputs, and consumer verification records +bind to the exact packed candidate; post-publication claims require a separately authorized registry readback. + +#### Scenario: Normal behavior +- **Given:** one packed candidate +- **When:** supply-chain gates run +- **Then:** its digest binds package inventory, SBOM, audit, licenses, source commit, workflow, and expected provenance + +#### Scenario: Failure or refusal +- **Given:** a vulnerability/policy failure, incompatible license, digest mismatch, or missing artifact +- **When:** the gate evaluates +- **Then:** release is blocked and the failure is retained + +#### Scenario: Relevant boundary +- **Given:** no publication is authorized +- **When:** candidate certification evidence is assembled +- **Then:** provider-derived signature/provenance readback remains pending and cannot be fabricated from local inputs + +### Requirement: QUAL-08 + +The release record resolves every applicable requirement and exposes failures, exclusions, +flaky dispositions, unsupported platforms, uncollected humans, owners, rollback, and residual risks. + +#### Scenario: Normal behavior +- **Given:** the accumulated requirements matrix and candidate evidence +- **When:** deterministic release validation runs +- **Then:** every row has pass/fail/not-applicable with reason, exact evidence, date, owner, and byte applicability + +#### Scenario: Failure or refusal +- **Given:** a silent row, unjustified not-applicable, stale/mismatched evidence, or unresolved blocker +- **When:** the decision is computed +- **Then:** certification fails closed and the missing obligation is visible + +#### Scenario: Relevant boundary +- **Given:** implementation passes but external human, platform, reviewer, or provider evidence is unavailable +- **When:** Goal 5 reaches its gate +- **Then:** the packet remains blocked/unverified and the release claim stays at its prior level diff --git a/specs/changes/local-artifact-core-release/design.md b/specs/changes/local-artifact-core-release/design.md new file mode 100644 index 0000000..0c910ed --- /dev/null +++ b/specs/changes/local-artifact-core-release/design.md @@ -0,0 +1,64 @@ +# Design: Certify the local artifact core release + +## Context and constraints + +Certification is a security- and compatibility-sensitive decision over one immutable package, +not a summary of repository progress. The candidate accumulates Phases 0-3 and every applicable +OUT/UX/OC/LIFE/RENDER/LOCAL-01/SEC/PRIV/OPS/PERF/COMPAT/DIST/QUAL obligation. Evidence can be +local, human, platform, or provider-derived, but must name scope, date, owner, exact candidate +applicability, and outcome. Existing preview evidence is reusable only when it truly applies to +the same inputs; it never waives a certified gate. + +## Chosen design + +Add a versioned machine-readable certification record plus a human release report. Freeze the +source commit, version, tarball bytes, SHA-256, SRI, and corpus versions before collecting +candidate evidence. Model each requirement row as `pending`, `pass`, `fail`, or `not-applicable`; +the latter requires a reason allowed for the local-core level. Evidence records carry type, +path, date, owner, candidate digest or explicit non-byte-bound scope, environment, and result. +The only transition to `certified` requires all applicable rows pass, external prerequisites +resolve, accountable sign-offs exist, claims agree, and no unresolved blocker remains. + +Candidate verification and release mutation are separate. This packet permits building and +testing candidate bytes. It does not permit creating a release/tag, publishing to npm, changing +trusted-publisher settings, or deploying; each external mutation needs exact contemporaneous +authority and any required post-provider evidence. + +## Alternatives + +- A prose-only checklist was rejected because omitted and mismatched evidence is hard to detect. +- Reusing the public-preview decision was rejected because certification has broader accumulated + gates and provider attestation is byte-specific. +- Certifying only the currently available Linux/Chromium cell was rejected because it would + contradict the normative COMPAT-01 target rather than transparently re-scope it. +- Treating absent humans/reference runs as not-applicable was rejected because OUT-02, OUT-03, + RENDER-12, QUAL-07, and QUAL-08 explicitly make them mandatory for this claim. + +## Trust, privacy, and failure boundaries + +The release record contains no secrets, participant identities, private artifact contents, or +provider credentials. Human raw data remains access-controlled under the approved retention and +withdrawal protocol; the repository receives aggregates and digests only. Claude captures obey +account authorization and redistribution constraints. Any failed, missing, stale, ambiguous, or +cross-candidate evidence yields a refusal. No release action occurs as a side effect of evidence +validation. Logs and reports are bounded and redact secret-like values. + +## Migration, rollout, and rollback + +Migration fixtures cover every released local schema and must pass upgrade, backup, idempotence, +fault, and rollback checks on the frozen candidate. Certification produces a decision artifact, +not a deployment. If later publication is authorized, rollout uses the existing trusted- +publishing workflow and requires exact registry integrity/signature/provenance readback; failure +halts or deprecates the candidate according to the security policy. Before publication, rollback +is simply refusal and rebuilding a new candidate. After publication, the documented compromised- +release and SemVer policies apply; evidence from changed bytes cannot be patched in place. + +## Formal-method decision + +- Decision: bounded state-machine model plus property-based transition tests. +- Property and rationale: `certified` is unreachable unless one immutable candidate has every + applicable row passing, every required sign-off present, no blockers, and consistent claims; + validation and refusal never perform provider mutation. This targets the dangerous class of + partial, stale, or cross-byte evidence accidentally producing a release claim. +- Model/evidence path: planned `test/model/local-core-release-model.ts` and transition tests, + with a dated exact-candidate report under `docs/evidence/releases/`. diff --git a/specs/changes/local-artifact-core-release/evidence.md b/specs/changes/local-artifact-core-release/evidence.md new file mode 100644 index 0000000..21d4987 --- /dev/null +++ b/specs/changes/local-artifact-core-release/evidence.md @@ -0,0 +1,42 @@ +# Evidence: Certify the local artifact core release + +## Requirement: OUT-04 + +- Validation: one immutable candidate must accumulate every phase and accountable sign-off. +- Verification: the model independently blocks every missing row, blocker, sign-off, claim, and + digest mismatch; the exact candidate evaluates to `refused` with zero provider mutations. +- Result: refused because human, comparative, support, registry, and sign-off gates remain pending. +- Evidence: [@model](test/model/local-core-release-model.ts) [@manual](docs/evidence/releases/2026-08-19-local-core-candidate.json) + +## Requirement: COMPAT-01 + +- Validation: diagnostic OS runs cannot become support without the complete browser/device and first-use cell. +- Verification: one shared tarball passes command-line observations on Linux, macOS, and Windows; + the evidence names all missing Windows 11/WSL, previous macOS, browser, mobile, and human scope. +- Result: incomplete; zero supported platform IDs are claimed. +- Evidence: [@manual](docs/evidence/releases/2026-08-19-local-core-ci.md) [@manual](docs/support-policy.md) + +## Requirement: DIST-03 + +- Validation: release surfaces must agree and absent publication authority must cause no mutation. +- Verification: the packaged claim audit and complete test suite keep public-preview, + unsupported, uncertified, no-comparison language aligned. +- Result: pass for pre-release refusal; no tag, publish, deployment, or provider mutation occurred. +- Evidence: [@test](test/release-integrity.test.ts) [@manual](docs/evidence/releases/2026-08-19-local-core-claims.md) + +## Requirement: DIST-04 + +- Validation: audit, license, SBOM, hashes, CI subject, and consumer verification must bind the exact tarball; + registry integrity/signature/provenance require separately authorized publication. +- Verification: exact CI evidence, two isolated local packs, and three OS installs agree on SHA-256 + `6d5d4df63bb2300f438a572fc0af4741b793489bbd630b55070c04987c67badd`. +- Result: local supply-chain inputs pass; registry-derived inputs remain pending. +- Evidence: [@manual](docs/evidence/releases/2026-08-19-local-core-ci.md) [@manual](docs/evidence/releases/2026-08-19-local-core-consumer-verification.md) + +## Requirement: QUAL-08 + +- Validation: every row, owner, exact-byte boundary, missing input, claim, and rollback must remain visible. +- Verification: the machine-readable record validates, resolves every row to pass/pending, names + all blockers/sign-offs, disables certification/comparison/support, and records refusal. +- Result: pass for fail-closed decision quality; certification is refused. +- Evidence: [@test](test/model/local-core-release-model.ts) [@manual](docs/evidence/releases/2026-08-19-local-core-status.md) diff --git a/specs/changes/local-artifact-core-release/proposal.md b/specs/changes/local-artifact-core-release/proposal.md new file mode 100644 index 0000000..929917e --- /dev/null +++ b/specs/changes/local-artifact-core-release/proposal.md @@ -0,0 +1,43 @@ +# Proposal: Certify the local artifact core release + +## Outcome + +One immutable candidate is either certified as local artifact core with a complete dated +requirements decision and exact-package evidence, or refused with every missing/failed gate +visible. Approval of this packet does not authorize a tag, npm publication, or provider change. + +## Context + +Goals 1-4 establish substantial local behavior, but a certified release accumulates Phases +0-3 and the recurring certification gate. It requires human outcomes, comparative quality, +supported platforms, migration, security/privacy, performance, clean-host/package, SBOM, +vulnerability/license, and claim-consistency evidence for the same candidate bytes. Prior +public-preview attestations cannot certify a new package. + +## Scope + +- In scope: freeze and hash the candidate; resolve every applicable requirement; run exact + clean-host/browser/migration/security/performance/supply-chain gates; bind SBOM and + provenance inputs; audit README/support/privacy/release claims; record owners, failures, + rollback, support window, and a deterministic certify/refuse decision. +- Out of scope: publishing, tagging, changing trusted-publisher settings, silently accepting + risk, converting missing evidence to not-applicable, certifying future bytes, or including + Phase 4+ service/hosting/connector behavior in the claim. + +## Risks and rollback + +- Risk: evidence from different bytes is combined, a platform or human gate is hand-waved, + publication outruns the decision, or a certified claim survives a failing prerequisite. +- Rollback: fail closed before mutation; keep the candidate unreleased/uncertified, preserve + the refusal report, fix or re-scope through a newly approved packet, then rebuild and rerun + all byte-bound gates. If a later provider mutation is proposed, request separate exact + authority immediately before it. + +## Validation plan + +A machine-readable release record models candidate, evidence, and decision states and rejects +missing, stale, mismatched, failed, or inapplicable-without-reason rows. Verification runs the +full suite/build/check, exact tarball inspection, packed OpenCode/support matrix, browser and +accessibility QA, migrations/rollback, threat cases, benchmark and user-study gates, audit, +license, SBOM and provenance-input binding. Independent accountable-role sign-offs and the +absence of unresolved blockers are required before the decision can be `certified`. diff --git a/specs/changes/local-artifact-core-release/tasks.md b/specs/changes/local-artifact-core-release/tasks.md new file mode 100644 index 0000000..68cf794 --- /dev/null +++ b/specs/changes/local-artifact-core-release/tasks.md @@ -0,0 +1,11 @@ +# Tasks: Certify the local artifact core release + +- [x] Confirm proposal validation and human approval. +- [x] Implement and model-test the versioned fail-closed certification record. +- [x] Freeze one candidate commit/version/tarball/digest and resolve every applicable requirement row. +- [ ] Run full code, package, host, browser, migration, security/privacy, performance, and support gates. +- [ ] Bind audit, license, SBOM, integrity, provenance inputs, and consumer-verification evidence. +- [ ] Attach passing page-quality, first-use, comprehension, and accountable-role evidence. +- [x] Audit every public claim, failure, exclusion, rollback, owner, and support obligation. +- [x] Record a certify/refuse decision without tag, publish, or provider mutation. +- [ ] Update all affected current specs and archive only after the complete gate passes. diff --git a/specs/changes/page-quality-corpus/change.json b/specs/changes/page-quality-corpus/change.json new file mode 100644 index 0000000..3f78b7d --- /dev/null +++ b/specs/changes/page-quality-corpus/change.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "id": "page-quality-corpus", + "title": "Establish the canonical page-quality corpus", + "lane": "standard", + "status": "approved", + "affectedRequirements": [ + "RENDER-11", + "QUAL-07", + "DIST-07" + ], + "currentSpecs": [ + "specs/current/portable-renderer.spec.md" + ], + "currentSpecsUpdated": true, + "approval": { + "by": "aaron.zeng", + "at": "2026-08-18T15:22:56Z" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-18", + "archivedAt": null +} diff --git a/specs/changes/page-quality-corpus/delta.md b/specs/changes/page-quality-corpus/delta.md new file mode 100644 index 0000000..41cea27 --- /dev/null +++ b/specs/changes/page-quality-corpus/delta.md @@ -0,0 +1,64 @@ +# Specification delta: Establish the canonical page-quality corpus + +## MODIFIED + +### Requirement: RENDER-11 + +The normal Markdown/component path covers a versioned corpus of exactly the eight normative +task families. Each bundle includes an exact neutral prompt, permission-safe sources, required +and forbidden facts, reader/decision, interactions, stress cases, provenance, and stable hashes. + +#### Scenario: Normal behavior +- **Given:** the complete frozen corpus +- **When:** its normalized fixtures pass preflight and render +- **Then:** all eight task families and their required facts/interactions are represented without raw HTML + +#### Scenario: Failure or refusal +- **Given:** a bundle has an invented/ambiguous fact, unsafe path, missing license, or incomplete rubric +- **When:** corpus validation runs +- **Then:** the bundle and any benchmark using it are refused with a stable diagnostic + +#### Scenario: Relevant boundary +- **Given:** a source, prompt, or expected fact changes after a corpus version is frozen +- **When:** hashes are checked +- **Then:** the mismatch fails and requires a new version rather than rewriting prior evidence + +### Requirement: QUAL-07 + +Page-quality evidence binds every generation and interaction trace to an exact validated corpus +version and source commit; exploratory or incomplete runs cannot enter a release denominator. + +#### Scenario: Normal behavior +- **Given:** a capture manifest references a valid bundle +- **When:** evidence is prepared +- **Then:** exact prompt, input hashes, task, viewport, interaction, and expected hard gates are available + +#### Scenario: Failure or refusal +- **Given:** a capture omits or changes corpus material +- **When:** evidence validation runs +- **Then:** it is excluded visibly and cannot support a quality claim + +#### Scenario: Relevant boundary +- **Given:** all local fixtures pass but no authorized reference exists +- **When:** corpus status is reported +- **Then:** pattern coverage may pass while equal-or-better remains unverified + +### Requirement: DIST-07 + +Every corpus input and retained output has explicit source provenance, redistribution status, +and a content hash; private or ambiguous third-party material remains outside distributable files. + +#### Scenario: Normal behavior +- **Given:** synthetic project-owned corpus content +- **When:** distribution inventory runs +- **Then:** its license, attribution, source, and exact hash are recorded + +#### Scenario: Failure or refusal +- **Given:** content lacks redistribution authority or contains private data +- **When:** it is proposed for the corpus or package +- **Then:** validation refuses inclusion and reports the offending inventory entry + +#### Scenario: Relevant boundary +- **Given:** an authorized private reference may be reviewed but not redistributed +- **When:** benchmark evidence is retained +- **Then:** only permitted metadata/digests and aggregate results enter git or the package diff --git a/specs/changes/page-quality-corpus/evidence.md b/specs/changes/page-quality-corpus/evidence.md new file mode 100644 index 0000000..728b35f --- /dev/null +++ b/specs/changes/page-quality-corpus/evidence.md @@ -0,0 +1,28 @@ +# Evidence: Establish the canonical page-quality corpus + +## Requirement: RENDER-11 + +- Validation: the benchmark requires eight exact, system-neutral tasks with decidable facts, + decisions, interactions, stress cases, provenance, and hashes. +- Verification: schema, mutation, hash, fixture, path, license, fact, and renderer-preflight + checks pass for all eight checked-in bundles. +- Result: automated corpus construction passes; independent ambiguity and neutrality review is + still missing, so the packet remains unverified. +- Evidence: [@test](test/page-quality-corpus.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-benchmark-status.md) + +## Requirement: QUAL-07 + +- Validation: incomplete or modified generations must be excluded from any comparison. +- Verification: the manifest and validator bind task IDs, exact inputs, interactions, viewports, + and expected hard gates; the status record retains a zero comparative denominator. +- Result: pass for the capture contract and incomplete-status disclosure; no comparative claim. +- Evidence: [@test](test/page-quality-benchmark.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-benchmark-status.md) + +## Requirement: DIST-07 + +- Validation: only permission-safe, attributable corpus material may be distributed. +- Verification: corpus validation requires provenance, license, stable hashes, and contained + relative paths, while the repository redistribution inventory covers shipped sources. +- Result: automated inventory passes; an independent privacy and redistribution review remains + required before this packet can be verified. +- Evidence: [@test](test/page-quality-corpus.test.ts) [@manual](docs/evidence/governance/redistribution-2026-08-16.md) diff --git a/specs/changes/page-quality-corpus/proposal.md b/specs/changes/page-quality-corpus/proposal.md new file mode 100644 index 0000000..cd15db0 --- /dev/null +++ b/specs/changes/page-quality-corpus/proposal.md @@ -0,0 +1,38 @@ +# Proposal: Establish the canonical page-quality corpus + +## Outcome + +The repository contains eight deterministic, permission-safe page-quality bundles that can be +run unchanged through OpenCode and an authorized Claude Code Artifact reference. Each bundle +binds an exact prompt and source material to required and forbidden facts, a reader decision, +interactions, stress cases, licensing, and stable hashes. + +## Context + +The checked-in pattern examples are renderer regressions, not a controlled same-input corpus. +The normative benchmark requires dashboard, incident, PR walkthrough, system explainer, +comparison, plan/checklist, findings-table, and interactive-decision tasks without selecting +only favorable generations. A frozen corpus is needed before comparative evidence can be +credible or repeatable. + +## Scope + +- In scope: versioned bundle schema and validator; eight synthetic bundles; exact prompts, + inputs, facts, decisions, interactions, desktop/mobile stress cases, provenance, licenses, + content hashes, and fact-normalized renderer fixtures; deterministic mutation/refusal tests. +- Out of scope: Claude account access, model runs, reviewer scores, redistributing private or + ambiguously licensed output, changing the benchmark threshold, or claiming parity. + +## Risks and rollback + +- Risk: a fixture may leak private material, encode a preferred system's output, or be too + ambiguous for factual/hard-gate scoring. +- Rollback: reject the bundle before capture, retain prior corpus versions by hash, and amend + and reapprove this packet if a post-freeze correction changes facts or decisions. + +## Validation plan + +Schema and snapshot tests validate all eight task IDs, complete facts/interactions/stress +cases, safe relative paths, byte bounds, hashes, licenses, and renderer preflight. Independent +manual review confirms each prompt is system-neutral and each rubric answer is decidable from +the bundle alone. No reference generation is run under this packet. diff --git a/specs/changes/page-quality-corpus/tasks.md b/specs/changes/page-quality-corpus/tasks.md new file mode 100644 index 0000000..5a38093 --- /dev/null +++ b/specs/changes/page-quality-corpus/tasks.md @@ -0,0 +1,9 @@ +# Tasks: Establish the canonical page-quality corpus + +- [x] Confirm proposal validation and human approval. +- [x] Define the versioned bundle schema, limits, hashes, provenance, and license inventory. +- [x] Add all eight permission-safe prompts, source bundles, fact rules, decisions, interactions, and stress cases. +- [x] Add fact-normalized Markdown/component fixtures and deterministic schema/snapshot tests. +- [x] Record automated corpus evidence and update the portable-renderer current spec. +- [ ] Run independent ambiguity, neutrality, privacy, and redistribution review. +- [ ] Archive the packet after the independent review passes. diff --git a/specs/changes/responsive-visual-composition/change.json b/specs/changes/responsive-visual-composition/change.json new file mode 100644 index 0000000..a4808b8 --- /dev/null +++ b/specs/changes/responsive-visual-composition/change.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "id": "responsive-visual-composition", + "title": "Add responsive visual composition", + "lane": "standard", + "status": "approved", + "affectedRequirements": [ + "RENDER-06", + "RENDER-09", + "RENDER-10", + "QUAL-04" + ], + "currentSpecs": [ + "specs/current/portable-renderer.spec.md" + ], + "currentSpecsUpdated": true, + "approval": { + "by": "aaron.zeng", + "at": "2026-08-18T15:22:56Z" + }, + "withdrawal": { + "by": "", + "at": "", + "reason": "" + }, + "createdAt": "2026-08-18", + "archivedAt": null +} diff --git a/specs/changes/responsive-visual-composition/delta.md b/specs/changes/responsive-visual-composition/delta.md new file mode 100644 index 0000000..3e24969 --- /dev/null +++ b/specs/changes/responsive-visual-composition/delta.md @@ -0,0 +1,83 @@ +# Specification delta: Add responsive visual composition + +## MODIFIED + +### Requirement: RENDER-06 + +Composition primitives preserve DOM reading/focus order across visual layouts, reflow without +loss at 200% equivalent zoom, respect reduced motion and color modes, and retain semantic labels. + +#### Scenario: Normal behavior +- **Given:** a split, full-bleed, annotated, or dense composition +- **When:** it is used by keyboard or assistive technology +- **Then:** landmarks, reading order, focus, names, states, and contrast remain coherent + +#### Scenario: Failure or refusal +- **Given:** a declaration would hide content or make visual order contradict semantic order +- **When:** preflight validates it +- **Then:** publication is refused before permission or writes + +#### Scenario: Relevant boundary +- **Given:** 200% equivalent zoom, RTL text, long labels, or reduced motion +- **When:** the page renders +- **Then:** content reflows without overlap, clipping, lost function, or motion dependency + +### Requirement: RENDER-09 + +Bounded task-aware hierarchy, density, scale, card emphasis, and narrative rhythm variants use +space intentionally and avoid accidental dead zones, undersized primary visuals, and uniform templates. + +#### Scenario: Normal behavior +- **Given:** any normalized core-corpus task +- **When:** its recommended composition is rendered at desktop width +- **Then:** the primary finding and decision lead an intentional task-specific hierarchy + +#### Scenario: Failure or refusal +- **Given:** a primary visual occupies less than half a full-width card with unexplained empty space +- **When:** layout hard gates run +- **Then:** the sample fails even if its remaining visual score is high + +#### Scenario: Relevant boundary +- **Given:** dense and quiet variants use identical underlying facts +- **When:** they render +- **Then:** both remain readable and materially distinct without changing or hiding facts + +### Requirement: RENDER-10 + +Charts, diagrams, media/mockups, comparisons, diffs, and annotations size to their container and +switch to defined narrow-screen compositions instead of shrinking a fixed desktop canvas. + +#### Scenario: Normal behavior +- **Given:** a supported visual inside a full, split, or framed composition +- **When:** its container or viewport changes +- **Then:** it uses available width, preserves labels, and reflows according to the declared primitive + +#### Scenario: Failure or refusal +- **Given:** a fixed dimension or invalid combination would overflow the portable page +- **When:** preflight or runtime layout validation runs +- **Then:** it is refused or uses the documented safe linear fallback without horizontal scroll + +#### Scenario: Relevant boundary +- **Given:** a 390x844 viewport with long labels and an interactive control +- **When:** the page recomposes +- **Then:** the narrative becomes a logical single-column order with readable visuals and operable controls + +### Requirement: QUAL-04 + +Every new composition is exercised through the real portable page at desktop/mobile widths, +supported modes, keyboard-only, accessibility tooling, manual screen reader, and zero-error console. + +#### Scenario: Normal behavior +- **Given:** the eight checked-in normalized fixtures +- **When:** the browser evidence suite and manual checklist run +- **Then:** screenshots, accessibility state, interactions, overflow, requests, and console results are retained + +#### Scenario: Failure or refusal +- **Given:** any viewport clips, scrolls unexpectedly, misorders focus, or emits a runtime error +- **When:** evidence is evaluated +- **Then:** the affected composition remains unverified and blocks the packet + +#### Scenario: Relevant boundary +- **Given:** automated accessibility checks pass +- **When:** no named manual screen-reader run exists +- **Then:** QUAL-04 remains incomplete rather than inferred from automation diff --git a/specs/changes/responsive-visual-composition/evidence.md b/specs/changes/responsive-visual-composition/evidence.md new file mode 100644 index 0000000..85e46ab --- /dev/null +++ b/specs/changes/responsive-visual-composition/evidence.md @@ -0,0 +1,34 @@ +# Evidence: Add responsive visual composition + +## Requirement: RENDER-06 + +- Validation: visual recomposition must preserve semantic and focus order, modes, labels, and reflow. +- Verification: component/accessibility tests plus 16 exact-candidate Chromium cells cover + desktop/mobile modes, keyboard paths, overflow, clipping, requests, and console failures. +- Result: automated and browser tracks pass; the packet remains incomplete pending its named + manual screen-reader run. +- Evidence: [@test](test/accessibility.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-local-composition.md) + +## Requirement: RENDER-09 + +- Validation: every task needs intentional hierarchy and enough primary-visual utilization. +- Verification: all eight normalized tasks pass the desktop/mobile utilization, layout-shift, + clipping, and useful-content gates on the exact candidate. +- Result: pass in 16 of 16 retained browser cells. +- Evidence: [@test](test/page-quality-local-report.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-local-composition.md) + +## Requirement: RENDER-10 + +- Validation: charts, diagrams, frames, and controls must recompose rather than shrink a fixed canvas. +- Verification: schema/security tests and settled browser measurements cover the chart, Mermaid, + framed, comparison, and interactive fixtures at 1440×900 and 390×844. +- Result: pass with no overflow, clipping, render error, or external request. +- Evidence: [@test](test/components.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-local-composition.md) + +## Requirement: QUAL-04 + +- Validation: automation cannot substitute for a named screen-reader observation of new layouts. +- Verification: retained machine reports cover the real portable surface, keyboard paths, + responsive modes, console, requests, timing, and layout; manual review is not yet present. +- Result: incomplete solely at the manual screen-reader boundary. +- Evidence: [@test](test/page-quality-local-report.test.ts) [@manual](docs/evidence/page-quality/2026-08-19-local-composition.md) diff --git a/specs/changes/responsive-visual-composition/proposal.md b/specs/changes/responsive-visual-composition/proposal.md new file mode 100644 index 0000000..d9d0958 --- /dev/null +++ b/specs/changes/responsive-visual-composition/proposal.md @@ -0,0 +1,37 @@ +# Proposal: Add responsive visual composition + +## Outcome + +The normal Markdown/component path can produce intentionally varied, task-oriented pages whose +primary visuals use available space at desktop widths and recompose into a readable logical +order at narrow widths, without relaxing CSP, accessibility, or size budgets. + +## Context + +Goal 3 proved renderer correctness but retained a visible quality gap: primary charts can sit +at a small fixed size inside a wide card, and existing primitives do not express full-bleed, +split, annotated, mockup, dense, or quiet compositions. The benchmark hard gates reject dead +space, clipping, fixed-canvas shrinking, and inaccessible reordering. + +## Scope + +- In scope: bounded declarative composition primitives, proportional chart/diagram sizing, + media/mockup and annotation frames, task-aware hierarchy variants, narrow-screen + recomposition, container-aware output, skill/reference guidance, and real browser evidence. +- Out of scope: raw CSS/HTML passthrough, arbitrary script, remote assets, CSP changes, + imitating Anthropic's visual identity, or treating screenshots as accessibility proof. + +## Risks and rollback + +- Risk: new layout vocabulary can create template repetition, reorder content semantically, + clip long/RTL text, or exceed portable-page budgets. +- Rollback: keep syntax additive, fall back to the existing linear card flow for unsupported + combinations, and refuse invalid declarations during preflight before publication. + +## Validation plan + +Component/schema tests cover normal and hostile declarations. All eight normalized corpus +fixtures render offline at 1440x900, 390x844, 200% equivalent zoom, light/dark, reduced motion, +LTR/RTL stress, keyboard interaction, and print. Retained browser evidence records layout, +accessibility tree, overflow, console, network, and byte/performance results; user-visible +changes receive manual screen-reader review before verification. diff --git a/specs/changes/responsive-visual-composition/tasks.md b/specs/changes/responsive-visual-composition/tasks.md new file mode 100644 index 0000000..28f6360 --- /dev/null +++ b/specs/changes/responsive-visual-composition/tasks.md @@ -0,0 +1,11 @@ +# Tasks: Add responsive visual composition + +- [x] Confirm proposal validation and human approval. +- [x] Freeze declarative primitive schemas, safe fallbacks, and hard-gate layout assertions. +- [x] Implement bounded full/split/framed/annotated/dense/quiet compositions and responsive visuals. +- [x] Apply task-aware hierarchy to all eight normalized fixtures without raw HTML or CSP changes. +- [x] Add schema, security, accessibility, RTL, reflow, overflow, byte, and performance tests. +- [x] Capture desktop/mobile/mode/keyboard browser evidence. +- [ ] Obtain manual screen-reader review of the new Goal 5 compositions. +- [x] Update component/skill guidance and the portable-renderer current spec. +- [ ] Archive the packet after manual review passes. diff --git a/specs/current/governance-policy.spec.md b/specs/current/governance-policy.spec.md index e37fea0..c9c024a 100644 --- a/specs/current/governance-policy.spec.md +++ b/specs/current/governance-policy.spec.md @@ -14,9 +14,10 @@ - `COMPAT-01`: Node 24 is the support floor. Target coverage is Ubuntu 24.04 LTS, current and previous macOS, Windows 11 native/WSL, current and oldest-supported stable OpenCode 1.x, latest-two desktop Chromium/Firefox/Safari and mobile Android Chrome/iOS Safari where - available. An exact Ubuntu/Node/OpenCode/single-Chromium packed-candidate observation is + available. An exact Ubuntu/Node/OpenCode/single-Chromium packed-candidate observation and + exact-shared-tarball command-line observations on one Linux, macOS, and Windows CI runner are tested but incomplete. There are zero supported complete cells; all target cells are - unverified. Public preview may expose that state but cannot promote the observation or claim + unverified. Public preview may expose that state but cannot promote those observations or claim support; certification still requires the target matrix. A goal-specific implementation gate may treat an unavailable cell as optional only after an explicit human decision and only when the cell remains unverified, platform-dependent behavior remains opt-in or diff --git a/specs/current/portable-renderer.spec.md b/specs/current/portable-renderer.spec.md index 2697a94..0f9b165 100644 --- a/specs/current/portable-renderer.spec.md +++ b/specs/current/portable-renderer.spec.md @@ -51,6 +51,16 @@ 200%-equivalent RTL evidence are green. A named 2026-08-18 user attestation records the manual screen-reader checklist passing on Fedora 44, Orca 50.2, and Chrome 151.0.7922.137; this closes the packet gate without declaring a broad supported-platform matrix. +- `QUAL-06`: frontmatter can select one of six bounded nonstandard compositions (`narrative`, + `dashboard`, `split`, `dense`, `quiet`, or `full`). The renderer classifies visual, data, and + insight sections without changing source order; cards and primary visuals recompose at narrow + widths. Vega-Lite defaults to container sizing, Mermaid output is bounded after layout, and the + semantic `frame` component provides a fixed mockup/media/code presentation surface. +- `QUAL-07`: `benchmarks/page-quality/v1/corpus.json` binds eight permission-safe normalized + tasks to exact fixture hashes, prompts, required/forbidden facts, reader decisions, + interactions, provenance, licenses, and all four required stress cases. Renderer-only local + evidence covers 16 desktop/mobile/mode cells, but same-input Claude runs and blinded reviewer + evidence remain unverified; no equal-or-better claim is active. - `PERF-01`: `benchmarks/renderer/v1/` owns hashed no-runtime, one-chart, and multi-runtime fixtures plus the two-core/4 GiB Node 24/Chromium 151 reference profile. Reports retain raw cold/warm samples, nearest-rank p50/p95, a five-sample minimum, 250ms scheduler floor, @@ -110,3 +120,8 @@ malformed or unknown markup. - This evidence combines a retained Linux/Chromium automated observation with one named Fedora/Orca/Chrome manual assistive-technology attestation. It does not certify a supported browser/OS matrix, a physical mobile device, or other assistive-technology combinations. +- `test/page-quality-corpus.test.ts`, `test/page-quality-local-report.test.ts`, and + `docs/evidence/page-quality/2026-08-19-local-composition.md` retain the refrozen exact-candidate + corpus binding and 16-cell Chromium renderer-track result. The earlier Fedora/Orca attestation + predates these composition changes and does not close their manual screen-reader gate. None of + this supplies comparative, representative-user, platform-support, or release-certification evidence. diff --git a/src/cli.ts b/src/cli.ts index 429a57b..e181f2d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,7 +1,8 @@ #!/usr/bin/env node +import { realpathSync } from "node:fs"; import { lstat, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { renderArtifact, renderRawHtml } from "./render.ts"; import type { PortableAssets } from "./assets.ts"; import type { ResolvedDesignTokens } from "./design-tokens.ts"; @@ -506,7 +507,15 @@ export async function main(argv: string[]): Promise { } } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +export function cliInvocationMatchesModule(argumentPath: string, moduleUrl = import.meta.url): boolean { + try { + return realpathSync(resolve(argumentPath)) === realpathSync(fileURLToPath(moduleUrl)); + } catch { + return moduleUrl === pathToFileURL(resolve(argumentPath)).href; + } +} + +if (process.argv[1] && cliInvocationMatchesModule(process.argv[1])) { main(process.argv.slice(2)).catch((err: unknown) => { console.error(err instanceof Error ? err.message : String(err)); process.exit(1); diff --git a/src/components.ts b/src/components.ts index cc2c2fd..982557f 100644 --- a/src/components.ts +++ b/src/components.ts @@ -12,7 +12,8 @@ export type ComponentKind = | "copy" | "mermaid" | "decisions" - | "table"; + | "table" + | "frame"; export const COMPONENT_KINDS: ReadonlySet = new Set([ "stats", @@ -26,6 +27,7 @@ export const COMPONENT_KINDS: ReadonlySet = new Set([ "mermaid", "decisions", "table", + "frame", ]); function asRecord(value: unknown): Record | undefined { @@ -146,6 +148,24 @@ export function validateComponent(kind: ComponentKind, source: string): Componen } const item = asRecord(value); if (!item) return issue(`${kind}-shape`, "expected a JSON object", "provide the documented object schema"); + if (kind === "frame") { + const frameKind = str(item, "kind"); + const issues: ComponentIssue[] = []; + if (frameKind !== "mockup" && frameKind !== "media" && frameKind !== "code") { + issues.push(...issue("frame-kind", "kind is not allowlisted", "use mockup, media, or code")); + } + if (!str(item, "title")) issues.push(...issue("frame-title", "title is required", "add a concise frame title")); + if (!str(item, "caption")) issues.push(...issue("frame-caption", "caption is required", "add a meaningful text equivalent")); + if (typeof item["content"] !== "string") issues.push(...issue("frame-content", "content must be text", "add bounded text content")); + const annotations = item["annotations"]; + if (annotations !== undefined && (!Array.isArray(annotations) || annotations.some((note) => typeof note !== "string"))) { + issues.push(...issue("frame-annotations", "annotations must be strings", "replace non-string annotations")); + } + if (typeof item["content"] === "string" && Buffer.byteLength(item["content"], "utf8") > 8 * 1024) { + issues.push(...issue("frame-content-size", "content exceeds 8 KiB", "shorten the framed excerpt")); + } + return issues; + } if (kind === "callout") { const tone = str(item, "tone"); return tone !== undefined && !ITEM_TONES.includes(tone as typeof ITEM_TONES[number]) ? issue("callout-tone", "tone is not allowlisted", "use good, bad, warn, info, or neutral") : []; @@ -319,6 +339,25 @@ function renderCallout(spec: unknown): string { ].join(""); } +function renderFrame(spec: unknown): string { + const item = asRecord(spec); + if (!item) return errorBox("frame", "expected a JSON object"); + const kind = str(item, "kind") ?? "media"; + const title = str(item, "title") ?? "Framed content"; + const caption = str(item, "caption") ?? "Framed content"; + const content = str(item, "content") ?? ""; + const annotations = Array.isArray(item["annotations"]) ? item["annotations"] : []; + const notes = annotations.length === 0 ? "" : `
    ${annotations.map((note) => `
  1. ${escapeHtmlText(String(note))}
    2. `).join("")}
`; + return [ + `
`, + `
${escapeHtmlText(title)}
`, + `
${escapeHtmlText(content)}
`, + notes, + `
${escapeHtmlText(caption)}
`, + "
", + ].join(""); +} + function renderProgress(spec: unknown): string { const item = asRecord(spec); if (!item) return errorBox("progress", "expected a JSON object"); @@ -514,5 +553,7 @@ export function renderComponent(kind: ComponentKind, json: string, id?: string, return renderDecisions(spec); case "table": return renderTable(spec, context, id); + case "frame": + return renderFrame(spec); } } diff --git a/src/markdown.ts b/src/markdown.ts index 25f0f72..0329f81 100644 --- a/src/markdown.ts +++ b/src/markdown.ts @@ -13,8 +13,12 @@ export interface Frontmatter { dir?: string; locale?: string; timezone?: string; + composition?: CompositionKind; } +export type CompositionKind = "standard" | "narrative" | "dashboard" | "split" | "dense" | "quiet" | "full"; +export const COMPOSITION_KINDS: ReadonlySet = new Set(["standard", "narrative", "dashboard", "split", "dense", "quiet", "full"]); + export type ChartKind = "vega-lite" | "vega" | "echarts"; const CHART_KINDS: ReadonlySet = new Set(["vega-lite", "vega", "echarts"]); @@ -72,6 +76,7 @@ function parseFrontmatter(source: string, warnings: string[]): { meta: Frontmatt else if (key === "dir") meta.dir = value; else if (key === "locale") meta.locale = value; else if (key === "timezone") meta.timezone = value; + else if (key === "composition" && COMPOSITION_KINDS.has(value)) meta.composition = value as CompositionKind; else warnings.push(`frontmatter key ignored: ${key}`); } return { meta, body: source.slice(match[0].length), lineOffset: (match[0].match(/\n/g) ?? []).length }; diff --git a/src/preflight.ts b/src/preflight.ts index 6defb5d..458ef65 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -4,7 +4,7 @@ import { resolvePortableAssets, AssetPreflightError, type PortableAssets } from import { validateChartSpec } from "./render.ts"; import { headingSlugify } from "./text.ts"; import { canonicalLocale, validTimeZone } from "./locale.ts"; -import { parseDocument } from "./markdown.ts"; +import { COMPOSITION_KINDS, parseDocument } from "./markdown.ts"; import { loadProjectDesignTokens, resolveDesignTokens, type ResolvedDesignTokens } from "./design-tokens.ts"; export type DiagnosticSeverity = "error" | "warning"; @@ -34,7 +34,7 @@ export interface PreflightResult { } const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/; -const KNOWN_FRONTMATTER = new Set(["title", "icon", "description", "theme", "source", "font", "lang", "dir", "locale", "timezone"]); +const KNOWN_FRONTMATTER = new Set(["title", "icon", "description", "theme", "source", "font", "lang", "dir", "locale", "timezone", "composition"]); const THEMES = new Set(["default", "report", "ops", "editorial"]); const CHART_KINDS = new Set(["vega-lite", "vega", "echarts"]); const ALERT_KINDS = new Set(["NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"]); @@ -72,6 +72,7 @@ function frontmatterDiagnostics(markdown: string): AuthoringDiagnostic[] { if (key === "dir" && parsed[2].trim() !== "ltr" && parsed[2].trim() !== "rtl") output.push(diagnostic("direction-invalid", "error", "frontmatter", index + 2, "dir must be ltr or rtl", "choose the document's logical direction")); if (key === "locale" && canonicalLocale(parsed[2].trim()) === undefined) output.push(diagnostic("locale-invalid", "error", "frontmatter", index + 2, "locale is not a valid BCP 47 locale", "use a locale such as en-US, de-DE, or ar-EG")); if (key === "timezone" && !validTimeZone(parsed[2].trim())) output.push(diagnostic("timezone-invalid", "error", "frontmatter", index + 2, "timezone is not a valid IANA time zone", "use a zone such as UTC, Europe/Berlin, or Asia/Tokyo")); + if (key === "composition" && !COMPOSITION_KINDS.has(parsed[2].trim())) output.push(diagnostic("composition-invalid", "error", "frontmatter", index + 2, "composition is not allowlisted", "use standard, narrative, dashboard, split, dense, quiet, or full")); } return output; } diff --git a/src/render.ts b/src/render.ts index f303b89..1377025 100644 --- a/src/render.ts +++ b/src/render.ts @@ -1,5 +1,5 @@ import { compile as compileVegaLite } from "vega-lite"; -import { parseDocument, type ChartSpec, type Frontmatter } from "./markdown.ts"; +import { parseDocument, type ChartSpec, type CompositionKind, type Frontmatter } from "./markdown.ts"; import { renderComponent } from "./components.ts"; import { runtimeBundle, type RuntimeName } from "./runtime.ts"; import { escapeHtmlText, headingSlugify } from "./text.ts"; @@ -82,12 +82,26 @@ html{overflow-wrap:anywhere}body{margin:0;background:var(--page-bg);color:var(-- .skip-link{position:fixed;z-index:100;inset-block-start:.5rem;inset-inline-start:.5rem;padding:.55rem .8rem;background:var(--card-bg);color:var(--ink);border:2px solid var(--accent);border-radius:8px;transform:translateY(-160%)} .skip-link:focus{transform:none} .artifact-header{display:flex;align-items:center;gap:.6rem;padding:.9rem 1.5rem;background:var(--card-bg);border-bottom:1px solid var(--line)} -.theme-toggle{margin-inline-start:auto;min-height:2rem;background:none;border:1px solid var(--line);border-radius:999px;padding:.25rem .8rem;font-size:.75rem;font-weight:600;color:var(--ink-2);cursor:pointer} +.theme-toggle{flex:none;white-space:nowrap;margin-inline-start:auto;min-height:2rem;background:none;border:1px solid var(--line);border-radius:999px;padding:.25rem .8rem;font-size:.75rem;font-weight:600;color:var(--ink-2);cursor:pointer} .theme-toggle:hover{border-color:var(--accent);color:var(--accent)} -.artifact-header h1{font-size:1.1rem;margin:0;letter-spacing:-.01em} +.artifact-header h1{min-width:0;font-size:1.1rem;margin:0;letter-spacing:-.01em} .artifact-icon{font-size:1.25rem} .artifact-body{max-width:1080px;margin:0 auto;padding:var(--body-pad) var(--body-pad) var(--body-pad-bottom)} .artifact-body>*:first-child{margin-top:0} +.artifact-intro{margin:0 0 var(--section-gap)} +.artifact-intro>p:first-child{max-width:64ch;font-size:clamp(1rem,1.7vw,1.2rem);line-height:1.55;color:var(--ink-2)} +.composition-narrative{max-width:920px}.composition-narrative .section-card:first-of-type{padding-block:clamp(1.5rem,4vw,3rem)} +.composition-narrative .section-card:first-of-type h2{font-size:clamp(1.6rem,3vw,2.35rem);max-width:22ch} +.composition-dashboard,.composition-full{max-width:1240px}.composition-dashboard{--section-gap:1rem} +.composition-split{max-width:1240px;display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--section-gap)} +.composition-split .artifact-intro{grid-column:1/-1;margin:0}.composition-split .section-card{margin:0;min-width:0} +.composition-split .section-visual,.composition-split .section-data{grid-column:1/-1} +.composition-split .section-card:last-child:nth-of-type(odd){grid-column:1/-1} +.composition-dense{max-width:1240px;--section-gap:.75rem;--section-pad-y:1rem;--section-pad-x:1.15rem;--table-font-size:.8rem} +.composition-quiet{max-width:820px;--section-gap:2rem}.composition-quiet .section-card{box-shadow:none;border-block-start:1px solid var(--line);border-radius:0;padding-inline:0} +.composition-full{max-width:1440px}.composition-full .section-visual{padding-inline:clamp(1rem,3vw,3rem)} +.section-visual .chart-frame,.section-visual .diagram-frame{width:100%}.section-visual .chart{height:clamp(320px,42vw,560px)} +.section-insight{background:var(--card-info-bg)} .artifact-footer{max-width:1080px;margin:0 auto;padding:1rem 1.5rem 2rem;font-size:.8rem;color:var(--ink-3)} .artifact-footer a{color:inherit} .section-card{background:var(--card-bg);border-radius:var(--radius);box-shadow:var(--shadow);padding:var(--section-pad-y) var(--section-pad-x);margin:var(--section-gap) 0} @@ -100,8 +114,10 @@ p{margin:.6rem 0} pre{background:var(--code-bg);padding:.75rem 1rem;overflow:auto;border-radius:10px} code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.9em} p code,li code,td code{background:var(--code-bg);padding:.1em .35em;border-radius:5px} -.chart-frame,.diagram-frame{margin:1rem 0}.chart{margin:0;min-height:320px;max-width:100%;overflow:hidden} +.chart-frame,.diagram-frame{margin:1rem 0}.chart{display:block;width:100%;margin:0;min-height:320px;max-width:100%;overflow:hidden} .chart-summary,.diagram-summary{margin:.5rem 0 0;color:var(--ink-2);font-size:.88rem} +.diagram-frame svg{display:block;width:auto!important;max-width:100%!important;max-height:560px;height:auto!important;margin-inline:auto} +.visual-frame{display:grid;grid-template-columns:minmax(0,1fr) minmax(12rem,.32fr);gap:1rem;margin:1rem 0;align-items:start}.frame-surface{min-width:0;border:1px solid var(--line);border-radius:14px;overflow:hidden;background:var(--code-bg);box-shadow:var(--shadow)}.frame-bar{display:flex;gap:.75rem;align-items:center;padding:.55rem .8rem;border-bottom:1px solid var(--line);background:var(--card-bg);font-size:.8rem}.frame-bar span{color:var(--ink-3);letter-spacing:.15em}.frame-content{min-height:12rem;margin:0;border-radius:0;white-space:pre-wrap}.frame-annotations{margin:0;padding-inline-start:1.6rem;border-inline-start:2px solid var(--accent)}.frame-annotations li{margin:0 0 .75rem}.visual-frame figcaption{grid-column:1/-1;color:var(--ink-2);font-size:.88rem}.frame-media .frame-content{font-family:var(--artifact-font);font-size:1rem;display:flex;align-items:center;justify-content:center;text-align:center}.frame-code .frame-content{font-family:ui-monospace,SFMono-Regular,Menlo,monospace} .chart-error{padding:.75rem 1rem;border:1px solid var(--bad);border-radius:10px;color:var(--bad);background:var(--bad-bg);margin:1rem 0} table{border-collapse:collapse;width:100%;margin:1rem 0;font-size:.92rem} th{text-align:start;background:var(--code-bg);font-weight:600} @@ -171,7 +187,7 @@ li.task input{margin-inline-end:.45rem;accent-color:var(--accent)} .copy-btn{background:var(--accent);color:var(--accent-ink);border:none;border-radius:8px;padding:.45rem 1rem;font-size:.85rem;font-weight:600;cursor:pointer} .copy-btn:hover{filter:brightness(1.08)} .copy-note{font-size:.8rem;color:var(--good)} -pre.mermaid{background:var(--card-bg);border:1px solid var(--line);border-radius:10px;padding:1rem;text-align:center} +pre.mermaid{display:grid;place-items:center;min-height:clamp(240px,42vw,560px);background:var(--card-bg);border:1px solid var(--line);border-radius:10px;padding:1rem;text-align:center} .section-card pre.mermaid{background:var(--page-bg)} .decisions{margin:1.25rem 0} .decisions-title{font-weight:700;font-size:1.05rem;margin-bottom:.75rem} @@ -234,7 +250,7 @@ th[data-dir="desc"] .th-sort::after{content:"↓";opacity:1;color:var(--accent)} .card .icon{font-size:1.6rem} .gallery-empty{color:var(--ink-3);text-align:center;padding:3rem 0} :focus-visible{outline:3px solid var(--accent);outline-offset:3px} -@media (max-width:600px){.artifact-header{padding:.75rem 1rem}.artifact-body{--body-pad:1rem;--body-pad-bottom:2rem}.section-card{--section-pad-y:1rem;--section-pad-x:1rem}.tl-item{gap:.55rem}.tl-time{width:3.6rem}.comments-dock,.comment-form{inset-inline:1rem;width:auto}.stat-grid,.compare-grid{grid-template-columns:1fr}} +@media (max-width:600px){.artifact-header{padding:.75rem 1rem}.artifact-body{--body-pad:1rem;--body-pad-bottom:2rem}.section-card{--section-pad-y:1rem;--section-pad-x:1rem}.tl-item{gap:.55rem}.tl-time{width:3.6rem}.comments-dock,.comment-form{inset-inline:1rem;width:auto}.stat-grid,.compare-grid{grid-template-columns:1fr}.composition-split{display:block}.composition-split .section-card{margin:var(--section-gap) 0}.section-visual .chart{height:clamp(280px,100vw,420px)}.visual-frame{grid-template-columns:1fr}.frame-annotations{border-inline-start:0!important;border-block-start:1px solid var(--line);padding-inline-start:0!important;padding-block-start:.75rem}} @media (max-width:700px){.comments-dock{position:static;margin:1rem;max-width:none;width:auto}} @media (prefers-reduced-motion:reduce){*,*::before,*::after{scroll-behavior:auto!important;animation-duration:.01ms!important;animation-iteration-count:1!important;transition-duration:.01ms!important}} @page{margin:15mm}@media print{body{background:#fff;color:#000}.skip-link,.theme-toggle,.copy-btn,.copy-note,.table-filter,.comments-dock,.comment-pop,.comment-launcher,.comment-form{display:none!important}.section-card,.stat,.variant,.card,.finding,.callout{box-shadow:none!important;break-inside:avoid;border:1px solid #bbb}.chart-frame,.diagram-frame,.table-wrap{break-inside:avoid}.table-scroll{overflow:visible}.data-table{font-size:9pt}a{color:inherit;text-decoration:underline}}`; @@ -321,7 +337,21 @@ const BOOT = `(function () { if (window.mermaid && mermaidEls.length > 0) { var dark = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches; window.mermaid.initialize({ startOnLoad: false, theme: dark ? "dark" : "neutral" }); - Promise.resolve(window.mermaid.run({ nodes: mermaidEls })).catch(function (err) { + Promise.resolve(window.mermaid.run({ nodes: mermaidEls })).then(function () { + mermaidEls.forEach(function (el) { + var svg = el.querySelector("svg"); + if (!svg || typeof svg.getBBox !== "function") return; + try { + var bounds = svg.getBBox(); + var pad = 16; + if (bounds.width > 0 && bounds.height > 0) { + svg.setAttribute("viewBox", [bounds.x - pad, bounds.y - pad, bounds.width + pad * 2, bounds.height + pad * 2].join(" ")); + svg.setAttribute("width", "100%"); + svg.removeAttribute("height"); + } + } catch (_) {} + }); + }).catch(function (err) { mermaidEls.forEach(function (el) { if (el.querySelector("svg")) return; el.textContent = ""; @@ -741,12 +771,21 @@ function enhanceBodyHtml(html: string): string { return out; } -function wrapSections(html: string): string { +function wrapSections(html: string, composition?: CompositionKind): string { const chunks = html.split(/(?= `
${chunk}
`); - return [intro, ...wrapped].join("\n"); + const wrapped = sections.map((chunk) => { + const classes = ["section-card"]; + if (/class="(?:chart-frame|diagram-frame|visual-frame)/.test(chunk)) classes.push("section-visual"); + if (chunk.includes('class="table-wrap"')) classes.push("section-data"); + if (chunk.includes('class="callout')) classes.push("section-insight"); + return `
${chunk}
`; + }); + const leading = composition !== undefined && composition !== "standard" && intro.trim() !== "" + ? `
${intro}
` + : intro; + return [leading, ...wrapped].join("\n"); } export function emojiFaviconDataUri(icon: string): string { @@ -761,6 +800,7 @@ export function validateChartSpec(chart: ChartSpec): ResolvedChart & { code?: st const record = typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : undefined; + if (record === undefined) return { kind, code: `${chart.kind}-invalid`, error: "chart spec must be an object" }; const description = record?.["description"]; if (typeof description !== "string" || description.trim().length < 8) { return { kind, code: "chart-summary-missing", error: "chart needs a meaningful text description" }; @@ -768,7 +808,13 @@ export function validateChartSpec(chart: ChartSpec): ResolvedChart & { code?: st const summary = description.trim(); if (kind === "echarts") return { kind, spec: parsed, summary }; if (chart.kind === "vega-lite") { - const compiled = compileVegaLite(parsed as Parameters[0]); + const responsive = { + ...record, + width: record["width"] ?? "container", + height: record["height"] ?? "container", + autosize: record["autosize"] ?? { type: "fit", contains: "padding" }, + }; + const compiled = compileVegaLite(responsive as Parameters[0]); return { kind, spec: compiled.spec, summary }; } return { kind, spec: parsed, summary }; @@ -796,6 +842,7 @@ interface AssembleInput { assetCss?: string; designTokens?: ResolvedDesignTokens; locale: LocaleContext; + composition?: CompositionKind; } function assemblePage(input: AssembleInput): string { @@ -839,7 +886,7 @@ function assemblePage(input: AssembleInput): string { "", '', `

${escapeHtmlText(input.title)}

`, - `
${input.bodyHtml}
`, + `
${input.bodyHtml}
`, FOOTER_PLACEHOLDER, ]; @@ -895,7 +942,7 @@ export function renderArtifact(markdown: string, options: RenderOptions = {}): R const accessible = `
${escapeHtmlText(chart.summary)}
`; bodyHtml = bodyHtml.replace(placeholder, accessible); }); - bodyHtml = wrapSections(enhanceBodyHtml(bodyHtml)); + bodyHtml = wrapSections(enhanceBodyHtml(bodyHtml), doc.meta.composition); const html = assemblePage({ title: doc.meta.title ?? "Artifact", @@ -912,6 +959,7 @@ export function renderArtifact(markdown: string, options: RenderOptions = {}): R : `@font-face{font-family:"Artifact Project";src:url(${options.assets.font.dataUri}) format("${fontFormat(options.assets.font.mime)}");font-display:swap}:root{--artifact-font:"Artifact Project",system-ui,-apple-system,"Segoe UI",sans-serif;--artifact-heading-font:var(--artifact-font)}`, designTokens, locale, + composition: doc.meta.composition, }); return { html, meta: doc.meta, chartCount: doc.charts.length }; } diff --git a/test/accessibility.test.ts b/test/accessibility.test.ts index e6e3286..fff7e9c 100644 --- a/test/accessibility.test.ts +++ b/test/accessibility.test.ts @@ -89,6 +89,8 @@ test("accessibility preflight refuses missing equivalents and invalid internatio assert.equal(validateComponent("mermaid", "graph TD\nA-->B")[0]?.code, "mermaid-summary"); assert.equal(validateComponent("table", '{"columns":[],"rows":[]}')[0]?.code, "table-caption"); assert.equal(validateComponent("table", '{"caption":"Dates","columns":[{"key":"at","label":"At","type":"datetime"}],"rows":[{"at":"2026-08-17 15:00"}]}')[0]?.code, "table-date"); + const composition = await preflightDocument("---\ncomposition: floating\n---\n# x"); + assert.equal(composition.diagnostics[0]?.code, "composition-invalid"); }); test("built-in semantic colors meet WCAG AA text contrast floors", () => { diff --git a/test/candidate-platform-smoke.test.ts b/test/candidate-platform-smoke.test.ts new file mode 100644 index 0000000..11cd552 --- /dev/null +++ b/test/candidate-platform-smoke.test.ts @@ -0,0 +1,38 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { assertPortableHtml, parsePackEntry } from "../scripts/candidate-platform-smoke.ts"; + +test("candidate platform pack parser requires one complete exact coordinate", () => { + const entry = { + filename: "opencode-artifacts-0.15.0.tgz", + integrity: "sha512-example", + shasum: "abc123", + size: 10, + unpackedSize: 20, + entryCount: 3, + }; + const expected = { + filename: "opencode-artifacts-0.15.0.tgz", + integrity: "sha512-example", + shasum: "abc123", + size: 10, + unpackedSize: 20, + entryCount: 3, + }; + assert.deepEqual(parsePackEntry(JSON.stringify([entry])), expected); + assert.deepEqual(parsePackEntry(JSON.stringify({ package: entry })), expected); + assert.throws(() => parsePackEntry("[]"), /exactly one entry/); + assert.throws(() => parsePackEntry(JSON.stringify([{ filename: "x" }])), /missing integrity/); +}); + +test("candidate platform output requires content and the strict offline CSP", () => { + const html = [ + "", + "", + "Renderer no-runtime benchmark", + "

Build results

", + "

Benchmark interaction

", + ].join("\n"); + assert.doesNotThrow(() => assertPortableHtml(html)); + assert.throws(() => assertPortableHtml(html.replace("connect-src 'none'", "connect-src https:")), /connect-src/); +}); diff --git a/test/cli.test.ts b/test/cli.test.ts index 80b025b..bfd3c3f 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,16 +1,39 @@ import { execFile } from "node:child_process"; import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { ArtifactLifecycleStore } from "../src/artifact-lifecycle.ts"; +import { cliInvocationMatchesModule } from "../src/cli.ts"; const run = promisify(execFile); const CLI = join(process.cwd(), "src", "cli.ts"); const TOKEN = "ghp_0123456789abcdefABCDEF0123456789"; +test("CLI entrypoint detection resolves an installed-bin symlink", async () => { + const dir = await mkdtemp(join(tmpdir(), "cli-entrypoint-")); + try { + const packageDir = join(dir, "package"); + const binDir = join(dir, "node_modules", ".bin"); + const modulePath = join(packageDir, "cli.js"); + const binPath = process.platform === "win32" + ? join(binDir, "package", "cli.js") + : join(binDir, "opencode-artifacts"); + await mkdir(packageDir); + await mkdir(binDir, { recursive: true }); + await writeFile(modulePath, "// entrypoint\n"); + if (process.platform === "win32") await symlink(packageDir, join(binDir, "package"), "junction"); + else await symlink(modulePath, binPath, "file"); + assert.equal(cliInvocationMatchesModule(binPath, pathToFileURL(modulePath).href), true); + assert.equal(cliInvocationMatchesModule(join(binDir, "missing"), pathToFileURL(modulePath).href), false); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("render scans a title override for sensitive content", async () => { const dir = await mkdtemp(join(tmpdir(), "cli-title-")); try { diff --git a/test/components.test.ts b/test/components.test.ts index 6b63525..981a180 100644 --- a/test/components.test.ts +++ b/test/components.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { renderComponent } from "../src/components.ts"; +import { renderComponent, validateComponent } from "../src/components.ts"; import { renderArtifact } from "../src/render.ts"; test("stats renders value, label, and toned delta pill; text is escaped", () => { @@ -63,6 +63,23 @@ test("callout renders toned insight card", () => { assert.match(html, /callout-title">T { + const source = JSON.stringify({ + kind: "mockup", + title: "Settings preview", + caption: "The preview keeps navigation beside the selected settings group.", + content: "Navigation\n General\n Security\n\nSelected: Security", + annotations: ["Navigation remains visible", "Selection is named in text"], + }); + const html = renderComponent("frame", source); + assert.match(html, /
/); + assert.match(html, /
The preview keeps navigation/); + assert.match(html, /
    /); + assert.deepEqual(validateComponent("frame", source), []); + assert.equal(validateComponent("frame", '{"kind":"remote","title":"x","caption":"x","content":"x"}')[0]?.code, "frame-kind"); + assert.equal(validateComponent("frame", JSON.stringify({ kind: "code", title: "x", caption: "x", content: "x".repeat(8193) }))[0]?.code, "frame-content-size"); +}); + test("progress renders a fill proportional to done/total", () => { const html = renderComponent("progress", JSON.stringify({ label: "Ready", done: 3, total: 4 })); assert.match(html, /Ready — 3\/4/); @@ -145,6 +162,7 @@ test("mermaid fence becomes an escaped pre and inlines the mermaid runtime", () const { html } = renderArtifact("```mermaid\n%% summary: A leads to B.\ngraph TD\n A-->B\n```\n"); assert.match(html, /