diff --git a/.github/workflows/contract.yml b/.github/workflows/contract.yml new file mode 100644 index 0000000..bfbeba5 --- /dev/null +++ b/.github/workflows/contract.yml @@ -0,0 +1,81 @@ +name: Contract + +# Watch the contract this plugin maps against. +# +# BitRouter Cloud publishes its `GET /v1/models` response schema, generated +# from the Rust types that serialize the response. This job re-vendors that +# schema and, when it has moved, opens a pull request carrying the diff. +# +# The point is not the schema file. It is `test/schema.test.ts`, which fails +# when a field appears that nobody has decided about, or disappears while this +# package still reads it. A dropped field is the dangerous direction: nothing +# errors, the mapping silently falls back to a default, and the catalog goes +# quietly wrong. That is not hypothetical — it is what happened when this +# package read `context_window`. +# +# Deliberately not part of the pull-request build. That build has to be +# offline and deterministic to be worth gating merges on; this one talks to a +# live service, so it runs on a clock where a flake costs nothing. +on: + schedule: + - cron: "0 5 * * *" + workflow_dispatch: + +permissions: {} + +jobs: + refresh: + name: re-vendor the published schema + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v6 + with: { persist-credentials: false } + - uses: actions/setup-node@v4 + with: { node-version: "20", cache: npm } + - run: npm ci + + - name: re-vendor schema/models.schema.json + run: npm run schema:refresh + + # A pull request opened with the default GITHUB_TOKEN does not trigger + # workflows — GitHub suppresses that to stop a job from re-triggering + # itself. Which would leave this pull request with no checks at all, and + # a required check that never reports blocks a merge forever. So it + # wants a real token. Set CONTRACT_PR_TOKEN to a fine-grained PAT (or an + # app installation token) with contents:write and pull-requests:write on + # this repository. + - uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + token: ${{ secrets.CONTRACT_PR_TOKEN }} + branch: chore/contract-refresh + base: main + add-paths: schema/models.schema.json + commit-message: "chore: re-vendor BitRouter Cloud's /v1/models schema" + title: "chore: BitRouter Cloud's /v1/models contract has moved" + labels: contract + body: | + `schema/models.schema.json` no longer matches the schema BitRouter + Cloud publishes at . + + The diff is the contract change. What matters is whether the suite + still passes on it: + + - **Green** — the change is one this package already tolerates. + Merge it. + - **Red on the field-set assertion** — a field appeared or + vanished. Decide what to do with it and record the decision in + `ACKNOWLEDGED` in `test/schema.test.ts`. + - **Red on fixture conformance** — `test/fixtures/cloud-models.json` + predates the change. Recapture it: + `curl -s https://api.bitrouter.ai/v1/models`. + - **Red on tiered pricing** — a fixture row with `context_tiers` + is gone from the catalog. Pick another from the same call. + + # After the pull request, not before: if the new contract breaks the + # assertions, this job should go red *and* the pull request should still + # exist to show why. + - name: check the vendored contract against this package's assumptions + run: npm test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..b2816de --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,76 @@ +name: Publish + +# The human gate. +# +# Everything else in this repository is allowed to merge itself: a Dependabot +# bump or an automated compatibility fix that goes green on `main` needs no +# review, because `main` reaches nobody. This workflow is the step that does +# reach somebody, so it is the one a person has to perform — by drafting a +# GitHub release and pressing publish. +# +# Deliberately NOT `on: push: branches: [main]`. That trigger would collapse +# the merge gate and the release gate into one, and the whole safety of +# auto-merge rests on them being separate. +on: + release: + types: [published] + +permissions: {} + +jobs: + publish: + name: publish to npm + runs-on: ubuntu-latest + environment: npm + permissions: + contents: read + # npm provenance: the registry attests that this tarball was built by + # this workflow from this commit, which is checkable by anyone. + id-token: write + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.release.tag_name }} + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + registry-url: https://registry.npmjs.org + + # A release tag that disagrees with package.json publishes a version + # nobody asked for, under a git ref that does not contain it. Cheaper to + # refuse here than to deprecate afterwards. + - name: check the tag against package.json + env: + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + PKG=$(node -p "require('./package.json').version") + if [ "${TAG#v}" != "$PKG" ]; then + echo "::error::release tag '$TAG' does not match package.json version '$PKG'" + exit 1 + fi + echo "publishing $PKG from $TAG" + + # The same checks `main` had to pass, run once more against the tagged + # tree. A release can be cut from any ref, so being green on `main` is + # not by itself evidence about what is in the tarball. + - run: npm ci + - run: npm run build + - run: npx tsc --noEmit -p tsconfig.test.json + - run: npm test + + # A prerelease goes out under the `next` tag, so `npm i @bitrouter/opencode` + # keeps resolving to the last stable one. + - name: publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + PRERELEASE: ${{ github.event.release.prerelease }} + run: | + set -euo pipefail + if [ "$PRERELEASE" = "true" ]; then + npm publish --provenance --access public --tag next + else + npm publish --provenance --access public + fi diff --git a/package-lock.json b/package-lock.json index dbedb43..44e3a43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@opencode-ai/plugin": "^1.18.18", "@opencode-ai/sdk": "^1.18.18", "@types/node": "^20.0.0", + "ajv": "^8.20.0", "typescript": "^5.4.0", "vitest": "^1.6.0" }, @@ -1064,6 +1065,23 @@ "node": ">=0.4.0" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", @@ -1318,6 +1336,30 @@ "node": ">=12.17.0" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.6.tgz", + "integrity": "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/find-my-way-ts": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/find-my-way-ts/-/find-my-way-ts-0.1.6.tgz", @@ -1417,6 +1459,13 @@ "dev": true, "license": "(AFL-2.1 OR BSD-3-Clause)" }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/kubernetes-types": { "version": "1.30.0", "resolved": "https://registry.npmjs.org/kubernetes-types/-/kubernetes-types-1.30.0.tgz", @@ -1765,6 +1814,16 @@ "dev": true, "license": "MIT" }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/rollup": { "version": "4.62.4", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.4.tgz", diff --git a/package.json b/package.json index 4609c2c..8bb1636 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,9 @@ "build": "tsc", "typecheck": "tsc --noEmit", "test": "vitest run", - "prepublishOnly": "npm run build" + "prepublishOnly": "npm run build", + "schema:refresh": "node scripts/refresh-schema.mjs", + "schema:check": "node scripts/refresh-schema.mjs --check" }, "peerDependencies": { "@opencode-ai/plugin": ">=1.18.0" @@ -52,6 +54,7 @@ "@opencode-ai/plugin": "^1.18.18", "@opencode-ai/sdk": "^1.18.18", "@types/node": "^20.0.0", + "ajv": "^8.20.0", "typescript": "^5.4.0", "vitest": "^1.6.0" } diff --git a/schema/models.schema.json b/schema/models.schema.json new file mode 100644 index 0000000..a2f3d64 --- /dev/null +++ b/schema/models.schema.json @@ -0,0 +1,326 @@ +{ + "$comment": "The 200 response schema for GET /v1/models, extracted verbatim from BitRouter Cloud's published OpenAPI document at https://api.bitrouter.ai/openapi.json. Generated by scripts/refresh-schema.mjs — do not hand-edit; run `npm run schema:refresh`.", + "$defs": { + "ContextTier": { + "description": "A higher context-pricing bracket. Some upstreams bill a model at a steeper\nrate once the prompt crosses a context-length threshold (e.g. one rate up\nto 128k input tokens, a higher one between 128k and 256k). The selected\nbracket's rates apply to the **whole** request — a step function, not\ngraduated marginal brackets — and the bracket is chosen by the request's\ntotal input (prompt) token count.\n\nUpstreams that publish such tiers:\n- Alibaba Qwen Model Studio:\n \n- Google Gemini API long-context pricing:\n ", + "properties": { + "above_input_tokens": { + "description": "**Exclusive** lower bound on total input tokens. A request whose input\nsize is strictly greater than this is eligible for the bracket; a\nrequest exactly at the bound stays in the lower bracket (so a base\nbracket documented as \"≤ 128k\" is expressed as a tier with\n`above_input_tokens: 128000`).", + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "input_tokens": { + "$ref": "#/$defs/InputTokenPricing" + }, + "output_tokens": { + "$ref": "#/$defs/OutputTokenPricing" + } + }, + "required": [ + "above_input_tokens" + ], + "type": "object" + }, + "InputTokenPricing": { + "description": "Input token pricing per million tokens.", + "properties": { + "cache_read": { + "description": "Cost per million cache-read input tokens.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "cache_write": { + "description": "Cost per million cache-write input tokens.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "no_cache": { + "description": "Cost per million non-cached input tokens.", + "format": "double", + "type": [ + "number", + "null" + ] + } + }, + "type": "object" + }, + "ModelFirstTokenLatencyResponse": { + "properties": { + "best_p50_ms": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "best_p95_ms": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "sample_count": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "window_seconds": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "sample_count", + "window_seconds" + ], + "type": "object" + }, + "ModelLatencyResponse": { + "properties": { + "first_token": { + "$ref": "#/$defs/ModelFirstTokenLatencyResponse" + } + }, + "required": [ + "first_token" + ], + "type": "object" + }, + "ModelOutputTokenThroughputResponse": { + "properties": { + "best_p50_tokens_per_second": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "best_p95_tokens_per_second": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "sample_count": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "window_seconds": { + "format": "uint64", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "sample_count", + "window_seconds" + ], + "type": "object" + }, + "ModelPricing": { + "description": "Token pricing per million tokens for a model.\n\n`input_tokens`/`output_tokens` are the **base bracket**: the rates for the\nlowest context range and the fallback when no [`context_tiers`] entry\napplies. When `context_tiers` is empty the pricing is a single flat rate\nfor every request size (the historical behaviour).\n\n[`context_tiers`]: ModelPricing::context_tiers", + "properties": { + "context_tiers": { + "description": "Optional higher context brackets. Empty ⇒ flat pricing. The registry\nvalidator enforces ascending, unique `above_input_tokens` and a\ncomplete base bracket, so the per-request resolver does not depend on\nlist order.", + "items": { + "$ref": "#/$defs/ContextTier" + }, + "type": "array" + }, + "input_tokens": { + "$ref": "#/$defs/InputTokenPricing" + }, + "output_tokens": { + "$ref": "#/$defs/OutputTokenPricing" + } + }, + "type": "object" + }, + "ModelResponse": { + "properties": { + "capabilities": { + "description": "Union of inference capabilities advertised across this model's providers\n(e.g. `structured_outputs`). Omitted when empty.", + "items": { + "type": "string" + }, + "type": "array" + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "hosted": { + "description": "`true` for a hosted (platform-served) model; `false` for a BYOK/\nBYO-subscription model that only appears when the caller has a key for\none of its providers and requires that key to route.", + "type": "boolean" + }, + "id": { + "type": "string" + }, + "input_modalities": { + "items": { + "type": "string" + }, + "type": "array" + }, + "latency": { + "$ref": "#/$defs/ModelLatencyResponse" + }, + "max_input_tokens": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "max_output_tokens": { + "format": "uint64", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "open_weights": { + "description": "`true` when the model's weights are openly licensed, `false` when they\nare proprietary. Omitted entirely for models the registry does not\ndeclare (external BYOK offerings) — an absent field means \"unknown\",\nnever \"proprietary\".", + "type": [ + "boolean", + "null" + ] + }, + "output_modalities": { + "items": { + "type": "string" + }, + "type": "array" + }, + "pricing": { + "anyOf": [ + { + "$ref": "#/$defs/ModelPricing" + }, + { + "type": "null" + } + ] + }, + "providers": { + "$ref": "#/$defs/ProvidersSummary" + }, + "throughput": { + "$ref": "#/$defs/ModelThroughputResponse" + } + }, + "required": [ + "id", + "input_modalities", + "output_modalities", + "capabilities", + "hosted", + "providers", + "latency", + "throughput" + ], + "type": "object" + }, + "ModelThroughputResponse": { + "properties": { + "output_tokens": { + "$ref": "#/$defs/ModelOutputTokenThroughputResponse" + } + }, + "required": [ + "output_tokens" + ], + "type": "object" + }, + "OutputTokenPricing": { + "description": "Output token pricing per million tokens.", + "properties": { + "audio": { + "description": "Cost per million audio output tokens. Reserved — see `image`.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "image": { + "description": "Cost per million image output tokens. Reserved — not yet wired\ninto `calculate_charge_micro_usd`.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "reasoning": { + "description": "Cost per million reasoning output tokens.", + "format": "double", + "type": [ + "number", + "null" + ] + }, + "text": { + "description": "Cost per million text output tokens.", + "format": "double", + "type": [ + "number", + "null" + ] + } + }, + "type": "object" + }, + "ProvidersSummary": { + "properties": { + "total_online": { + "format": "uint", + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "total_online" + ], + "type": "object" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Envelope shape for `GET /v1/models`. Wraps the catalogue in `{ \"data\": [...] }`\nso future top-level fields (pagination cursors, etc.) can be added without a\nbreaking change.", + "properties": { + "data": { + "items": { + "$ref": "#/$defs/ModelResponse" + }, + "type": "array" + } + }, + "required": [ + "data" + ], + "title": "ModelListResponse", + "type": "object" +} diff --git a/scripts/refresh-schema.mjs b/scripts/refresh-schema.mjs new file mode 100644 index 0000000..58cc3cf --- /dev/null +++ b/scripts/refresh-schema.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +/** + * Vendor BitRouter Cloud's `GET /v1/models` response schema. + * + * node scripts/refresh-schema.mjs # write schema/models.schema.json + * node scripts/refresh-schema.mjs --check # exit 1 if the vendored copy is stale + * + * BitRouter Cloud generates an OpenAPI document from the Rust types that + * serialize the response (`ModelListResponse` / `ModelResponse`, via `aide`), + * serves it unauthenticated at `/openapi.json`, and syncs it into the public + * bitrouter-docs repository on every release. So the wire contract this plugin + * maps against is *published*, and there is no need to guess at it, diff live + * responses, or hold an API key in CI to find out when it moves. + * + * The copy is vendored rather than fetched at test time on purpose: the suite + * that gates merges has to be deterministic and offline. Refreshing is the + * moving part, and it happens in a scheduled workflow that opens a pull + * request — where a change to the contract shows up as a reviewable diff + * against the assertions in `test/schema.test.ts`. + * + * Nothing time-varying is written into the output, or `--check` would fail on + * every run regardless of whether the contract had moved. + */ + +import { readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const OUT = join(ROOT, "schema", "models.schema.json"); + +/** Where the contract is published. Named in the vendored file regardless of + * where this run actually read it from, so the provenance recorded in the + * repository is the canonical source and not somebody's local override. */ +const CANONICAL_URL = "https://api.bitrouter.ai/openapi.json"; +const SPEC_URL = process.env.BITROUTER_OPENAPI_URL ?? CANONICAL_URL; +const check = process.argv.includes("--check"); + +/** Recursively key-sort, so a cosmetic reordering upstream is not a diff here. */ +function sorted(value) { + if (Array.isArray(value)) return value.map(sorted); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((k) => [k, sorted(value[k])]), + ); + } + return value; +} + +function extract(spec) { + const schema = spec?.paths?.["/v1/models"]?.get?.responses?.["200"]?.content?.[ + "application/json" + ]?.schema; + if (!schema || typeof schema !== "object") { + throw new Error( + `no 200 application/json schema for GET /v1/models in ${SPEC_URL} — the spec moved, and this script needs updating before the vendored copy can be trusted`, + ); + } + return schema; +} + +/** + * The spec source. `BITROUTER_OPENAPI_URL` may also name a local file, which + * is how this runs behind a network that will not let the process out — fetch + * the document by whatever means work, then point this at it. + */ +async function loadSpec() { + if (!/^https?:/i.test(SPEC_URL)) { + return JSON.parse(readFileSync(SPEC_URL.replace(/^file:\/\//, ""), "utf8")); + } + const res = await fetch(SPEC_URL, { headers: { accept: "application/json" } }); + if (!res.ok) throw new Error(`GET ${SPEC_URL} — HTTP ${res.status}`); + return res.json(); +} + +const rendered = + JSON.stringify( + { + $comment: + `The 200 response schema for GET /v1/models, extracted verbatim from BitRouter Cloud's published OpenAPI document at ${CANONICAL_URL}. Generated by scripts/refresh-schema.mjs — do not hand-edit; run \`npm run schema:refresh\`.`, + ...sorted(extract(await loadSpec())), + }, + null, + 2, + ) + "\n"; + +let current = null; +try { + current = readFileSync(OUT, "utf8"); +} catch { + /* first run */ +} + +if (current === rendered) { + console.log("schema/models.schema.json is up to date"); + process.exit(0); +} + +if (check) { + console.error( + "schema/models.schema.json is stale — BitRouter Cloud's published contract has moved.\n" + + "Run: npm run schema:refresh", + ); + process.exit(1); +} + +mkdirSync(dirname(OUT), { recursive: true }); +writeFileSync(OUT, rendered); +console.log(`wrote schema/models.schema.json (from ${SPEC_URL})`); diff --git a/test/fixtures/cloud-models.json b/test/fixtures/cloud-models.json index 559b4b6..cf2e71d 100644 --- a/test/fixtures/cloud-models.json +++ b/test/fixtures/cloud-models.json @@ -26,8 +26,26 @@ "capabilities": [ "reasoning" ], + "hosted": true, + "open_weights": false, "providers": { "total_online": 1 + }, + "latency": { + "first_token": { + "best_p50_ms": null, + "best_p95_ms": null, + "sample_count": 0, + "window_seconds": 3600 + } + }, + "throughput": { + "output_tokens": { + "best_p50_tokens_per_second": null, + "best_p95_tokens_per_second": null, + "sample_count": 0, + "window_seconds": 3600 + } } }, { @@ -55,8 +73,77 @@ "capabilities": [ "tools" ], + "hosted": true, + "open_weights": false, "providers": { "total_online": 1 + }, + "latency": { + "first_token": { + "best_p50_ms": null, + "best_p95_ms": null, + "sample_count": 0, + "window_seconds": 3600 + } + }, + "throughput": { + "output_tokens": { + "best_p50_tokens_per_second": null, + "best_p95_tokens_per_second": null, + "sample_count": 0, + "window_seconds": 3600 + } + } + }, + { + "id": "anthropic/claude-sonnet-5", + "name": "Anthropic: Claude Sonnet 5", + "description": "Anthropic's Claude Sonnet 5.", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "input_modalities": [ + "text", + "image" + ], + "output_modalities": [ + "text" + ], + "pricing": { + "input_tokens": { + "no_cache": 2.0, + "cache_read": 0.2, + "cache_write": 2.5 + }, + "output_tokens": { + "text": 10.0 + } + }, + "capabilities": [ + "image_input", + "reasoning", + "structured_outputs", + "tools" + ], + "hosted": true, + "open_weights": false, + "providers": { + "total_online": 1 + }, + "latency": { + "first_token": { + "best_p50_ms": null, + "best_p95_ms": null, + "sample_count": 0, + "window_seconds": 3600 + } + }, + "throughput": { + "output_tokens": { + "best_p50_tokens_per_second": null, + "best_p95_tokens_per_second": null, + "sample_count": 0, + "window_seconds": 3600 + } } }, { @@ -86,9 +173,86 @@ "structured_outputs", "tools" ], + "hosted": true, + "open_weights": false, + "providers": { + "total_online": 2 + }, + "latency": { + "first_token": { + "best_p50_ms": null, + "best_p95_ms": null, + "sample_count": 0, + "window_seconds": 3600 + } + }, + "throughput": { + "output_tokens": { + "best_p50_tokens_per_second": null, + "best_p95_tokens_per_second": null, + "sample_count": 0, + "window_seconds": 3600 + } + } + }, + { + "id": "x-ai/grok-4.3", + "name": "xAI: Grok 4.3", + "max_input_tokens": 1000000, + "max_output_tokens": 30000, + "input_modalities": [ + "text", + "image" + ], + "output_modalities": [ + "text" + ], + "pricing": { + "input_tokens": { + "no_cache": 1.25, + "cache_read": 0.2 + }, + "output_tokens": { + "text": 2.5 + }, + "context_tiers": [ + { + "above_input_tokens": 200000, + "input_tokens": { + "no_cache": 2.5, + "cache_read": 0.4 + }, + "output_tokens": { + "text": 5.0 + } + } + ] + }, + "capabilities": [ + "reasoning", + "tools" + ], + "hosted": true, + "open_weights": false, "providers": { "total_online": 2 + }, + "latency": { + "first_token": { + "best_p50_ms": null, + "best_p95_ms": null, + "sample_count": 0, + "window_seconds": 3600 + } + }, + "throughput": { + "output_tokens": { + "best_p50_tokens_per_second": null, + "best_p95_tokens_per_second": null, + "sample_count": 0, + "window_seconds": 3600 + } } } ] -} \ No newline at end of file +} diff --git a/test/schema.test.ts b/test/schema.test.ts new file mode 100644 index 0000000..78e8384 --- /dev/null +++ b/test/schema.test.ts @@ -0,0 +1,205 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import Ajv2020 from "ajv/dist/2020.js"; + +/** + * The wire contract, checked against the contract BitRouter publishes. + * + * `test/wire.test.ts` proves the mapper reads a captured body correctly. It + * cannot prove the capture still resembles what the server sends — a fixture + * is a snapshot, and a snapshot goes stale in silence. That is not a + * hypothetical failure: this package once read `context_window`, a flat + * `cost`, and a boolean `reasoning`, none of which BitRouter has ever sent, so + * every model sat at its default window priced at zero. No test failed. + * Somebody had to read the code. + * + * BitRouter Cloud generates an OpenAPI document from the Rust types that + * serialize the response and serves it unauthenticated, so the contract is + * *published* — `schema/models.schema.json` is a vendored copy, refreshed by + * `npm run schema:refresh` and by a scheduled workflow that opens a pull + * request when it moves. + * + * The load-bearing assertion is the first one. It is not "does the fixture + * parse"; it is "has the set of fields BitRouter serves changed since somebody + * last looked". A new field fails this suite once, and the fix is to decide — + * in `ACKNOWLEDGED`, in one line — whether to map it or to ignore it on + * purpose. That is the whole mechanism: drift cannot pass silently, and + * acknowledging it is cheap. + * + * The local daemon has no equivalent. `GET /v1/models` in + * `crates/bitrouter-sdk/src/server.rs` builds its body from an inline + * `serde_json::json!` literal rather than a schema-derived type, so there is + * nothing published to check `local-models.json` against. + */ + +// `uint64`, `uint`, and `double` are Rust's numeric widths surviving into the +// document as format hints. They are not JSON Schema formats, ajv has no +// validator for them, and saying so keeps it from narrating that fact once per +// occurrence per compile. +const ajv = new Ajv2020({ strict: false, allErrors: true, validateFormats: false }); + +function load(path: string): any { + return JSON.parse(readFileSync(fileURLToPath(new URL(path, import.meta.url)), "utf8")); +} + +const schema = load("../schema/models.schema.json"); +const modelResponse = schema.$defs.ModelResponse; + +/** + * Every property of BitRouter Cloud's `ModelResponse`, and what this package + * does with it. Adding a key here is a decision, which is the point: the test + * below fails until one has been made. + * + * `use` is what the fixture-coverage assertion reads, so it has to describe + * this package honestly rather than aspirationally: + * + * - `model` — reaches the model entry this plugin hands opencode. + * - `read` — parsed and exported, but not carried into the model. + * - `none` — neither, and for a stated reason. + */ +const ACKNOWLEDGED: Record = { + id: { use: "model", why: "the model id — the only field both data planes share" }, + name: { use: "model", why: "display name, falling back to the id" }, + max_input_tokens: { + use: "model", + why: "`contextWindow`. There is no `context_window` field, and reading one is the bug this suite exists to prevent", + }, + max_output_tokens: { use: "model", why: "`maxTokens`" }, + input_modalities: { use: "model", why: "`input`, vision included" }, + capabilities: { + use: "model", + why: "open list of tokens: `reasoning` and `tools` become the booleans opencode wants, and `image_input`/`file_input` merge into `input` — which is also what sets `attachment`", + }, + pricing: { + use: "model", + why: "`cost`, reshaped by `toCost`. Both sides are already per-million, so it is a reshape and not a conversion — but see the note on `context_tiers` in the assertions below: `toCost` reads the base bracket only", + }, + providers: { + use: "read", + why: "`providerCount` reads `{ total_online }` here and `string[]` from the local daemon; exported, but an opencode model entry has nowhere to put it", + }, + + output_modalities: { use: "model", why: "`modalities.output`, defaulting to text" }, + + // Present on the wire, deliberately not carried. An opencode model entry + // has no field that would hold them, and inventing one would be this plugin + // asserting something the harness cannot act on. + description: { + use: "none", + why: "no corresponding field; only the synthesized auto entry carries one, written here", + }, + hosted: { use: "none", why: "provisioning fact, not a capability of the model" }, + open_weights: { use: "none", why: "licensing fact, not a capability of the model" }, + latency: { use: "none", why: "a rolling measurement, not part of a model's description" }, + throughput: { use: "none", why: "a rolling measurement, not part of a model's description" }, +}; + +describe("BitRouter Cloud's published /v1/models contract", () => { + it("serves exactly the fields this package has looked at", () => { + const served = Object.keys(modelResponse.properties).sort(); + const acknowledged = Object.keys(ACKNOWLEDGED).sort(); + + // Spelled as two directed differences rather than one equality, because + // the two failures mean opposite things and want different fixes. + const appeared = served.filter((f) => !(f in ACKNOWLEDGED)); + const vanished = acknowledged.filter((f) => !(f in modelResponse.properties)); + + expect( + appeared, + "BitRouter now serves fields this package has never considered. Decide whether to map each one, then record the decision in ACKNOWLEDGED.", + ).toEqual([]); + expect( + vanished, + "BitRouter has stopped serving fields this package expects. Anything mapped is now silently falling back to a default — the failure mode this suite exists to catch.", + ).toEqual([]); + }); + + it("still requires the id every other mapping hangs off", () => { + expect(modelResponse.required).toContain("id"); + expect(modelResponse.properties.id.type).toBe("string"); + }); + + it("still carries the context window as a nullable integer named max_input_tokens", () => { + // The specific shape matters: `type: [integer, "null"]` is why the mapper + // treats a null as "undisclosed" and falls back, rather than as zero. + expect(modelResponse.properties.max_input_tokens.type).toEqual(["integer", "null"]); + expect(modelResponse.properties.max_output_tokens.type).toEqual(["integer", "null"]); + }); + + it("still nests pricing per million tokens rather than flattening it", () => { + expect(Object.keys(schema.$defs.InputTokenPricing.properties).sort()).toEqual([ + "cache_read", + "cache_write", + "no_cache", + ]); + expect(Object.keys(schema.$defs.OutputTokenPricing.properties).sort()).toEqual([ + "audio", + "image", + "reasoning", + "text", + ]); + }); + + it("prices some models in context brackets, which toCost flattens away", () => { + // A known and deliberate gap, asserted so it stays known. `context_tiers` + // raises the rate for the whole request once input crosses a threshold — + // a step function, not marginal brackets. `toCost` reports the base + // bracket, so opencode understates the cost of a long prompt to a tiered + // model. An opencode model entry carries one flat rate per direction and + // has nowhere to put a ladder; closing this properly needs an + // opencode-side change, not a mapper one. + expect(Object.keys(schema.$defs.ModelPricing.properties).sort()).toEqual([ + "context_tiers", + "input_tokens", + "output_tokens", + ]); + expect(schema.$defs.ContextTier.required).toContain("above_input_tokens"); + }); + + it("still counts providers as an object, which is what tells the planes apart", () => { + // The local daemon sends `providers: string[]`. Cloud sends an object. + // `providerCount()` branches on exactly this. + expect(Object.keys(schema.$defs.ProvidersSummary.properties)).toEqual(["total_online"]); + }); + + it("declares no fixed vocabulary for capability tokens", () => { + // Worth asserting rather than assuming: `capabilities` is an open list of + // strings, so `hasCapability` matching an unknown token is a miss and not + // an error, and a new token upstream cannot break this package. If an + // `enum` ever appears here, that reasoning stops holding. + expect(modelResponse.properties.capabilities.items).toEqual({ type: "string" }); + expect(modelResponse.properties.capabilities.items.enum).toBeUndefined(); + }); +}); + +describe("the committed cloud fixture", () => { + it("is a body BitRouter Cloud could actually serve", () => { + // Catches the fixture being trimmed or hand-edited into a shape the + // server would never send — which would make every assertion in + // wire.test.ts a test of a fiction. It is also what fails when a field + // becomes `required` upstream and the capture predates it. + const validate = ajv.compile(schema); + const ok = validate(load("./fixtures/cloud-models.json")); + expect(ok, ajv.errorsText(validate.errors ?? [], { separator: "\n" })).toBe(true); + }); + + it("carries every field this package actually consumes", () => { + // Conformance alone would accept a capture in which every optional field + // is absent — valid, and useless as a regression test, since the mapping + // would never run. + const rows = load("./fixtures/cloud-models.json").data as Record[]; + const present = new Set(rows.flatMap((r) => Object.keys(r))); + const consumed = Object.entries(ACKNOWLEDGED) + .filter(([, entry]) => entry.use !== "none") + .map(([field]) => field); + expect(consumed.filter((f) => !present.has(f))).toEqual([]); + }); + + it("includes a tiered-pricing model, so the shape stays covered", () => { + // Without one, the `context_tiers` branch of the schema is never + // exercised by conformance and could change unnoticed. + const rows = load("./fixtures/cloud-models.json").data as any[]; + expect(rows.some((r) => r.pricing?.context_tiers?.length > 0)).toBe(true); + }); +}); diff --git a/test/wire.test.ts b/test/wire.test.ts index 0340b36..12f7ae1 100644 --- a/test/wire.test.ts +++ b/test/wire.test.ts @@ -9,8 +9,9 @@ import type { DiscoveredModel } from "../src/discovery.js"; * planes, so a future change to the field mapping is caught by the wire and * not by a hand-written guess at it. * - * The fixtures are trimmed to the fields this package reads; every value in - * them is exactly what the endpoint served. + * The fixtures are verbatim rows, not trimmed ones: `test/schema.test.ts` + * checks them against BitRouter Cloud's published schema, and a capture with + * the uninteresting fields cut out would conform to nothing. */ function catalog(name: string): DiscoveredModel[] { const path = fileURLToPath(new URL(`./fixtures/${name}.json`, import.meta.url)); @@ -62,7 +63,7 @@ describe("BitRouter Cloud wire shape", () => { it("leads with the auto route", () => { const { models } = mapped("cloud-models"); expect(models[0].id).toBe("bitrouter/auto"); - expect(models).toHaveLength(4); // three served + auto + expect(models).toHaveLength(6); // five served + auto }); });