From 84539295628ce1c85c14352bcd8396a201afaded Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 12 Sep 2026 12:32:40 +0800 Subject: [PATCH] fix(playground): pull the speculative tier - quantized framing degrades output Measurement behind the tier (interscript-ml RESULTS.md 2026-09-12): dynamic-int8 decoder graphs quantize activations per fed tensor, so single-step and batched decode framings produce materially different numerics - the batched-verifier output lost a word on a canonical test row, and runtime acceptance measured 0.46 vs the probe's 0.99 (uniform-framing artifact). int4 CPU kernels are also slower than int8 here, so the tier had no speed story either. SpeculativeModel stays in the npm API as measurement infrastructure with the constraint documented; the playground serves only the verified tiers. --- e2e/playground-spec-live.spec.ts | 25 ------------------------- src/pages/playground.astro | 28 ++-------------------------- 2 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 e2e/playground-spec-live.spec.ts diff --git a/e2e/playground-spec-live.spec.ts b/e2e/playground-spec-live.spec.ts deleted file mode 100644 index 369d46f..0000000 --- a/e2e/playground-spec-live.spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -// LIVE playground speculative-tier verification — downloads the real -// drafter+verifier pair through the assets proxy and decodes in the -// browser with the verifier deciding every token. Opt-in (SPEC_E2E=1): -// SPEC_E2E=1 npx playwright test e2e/playground-spec-live.spec.ts -import { expect, test } from "@playwright/test" - -test("speculative tier downloads both models and vocalizes in the browser", async ({ page }) => { - test.skip(!process.env.SPEC_E2E, "live download test — set SPEC_E2E=1") - test.setTimeout(900_000) - await page.goto(`${process.env.SPEC_E2E_BASE ?? "http://localhost:4321"}/playground`) - await page.fill("#input", "قوله فحكمها في الوفاة") - await page.selectOption("#vocalize", "spec:ara-diac-small-2.1-int8") - await page.click("#run") - // both downloads stream progress, then the pair decodes - await expect(page.locator("#status")).toBeVisible({ timeout: 30_000 }) - const vocalized = page.locator("#vocalized") - // fresh profile = 359 MB total; budget for a slow network - await expect(vocalized).not.toBeEmpty({ timeout: 840_000 }) - const text = (await vocalized.textContent()) ?? "" - expect(text).toMatch(/[ً-ْٰٓ-ٕ]/) - expect(text).not.toMatch(/Error|cause:/) - // the acceptance readout surfaces once the worker call resolves - // after the vocalized text — poll rather than read once - await expect(page.locator("#status")).toContainText("accepted", { timeout: 60_000 }) -}) diff --git a/src/pages/playground.astro b/src/pages/playground.astro index 8da25df..7ad9bb5 100644 --- a/src/pages/playground.astro +++ b/src/pages/playground.astro @@ -27,9 +27,6 @@ import catalogue from "../data/maps-catalogue.json" - @@ -168,10 +165,6 @@ import catalogue from "../data/maps-catalogue.json" if (!voc) { return `import { transliterateAsync } from "interscript"\n\nconst out = await transliterateAsync(\n "${code}",\n "…",\n)` } - if (voc.startsWith("spec:")) { - const verifier = voc.slice(5) - return `import { transliterateAsync } from "interscript"\nimport { imf } from "interscript/ml"\n\nconst drafter = await imf.IMFModel.fromZipBytes(\n (await imf.resolve("ara-diac-layerdrop-1.0-int4")).bytes,\n)\nconst verifierModel = await imf.IMFModel.fromZipBytes(\n (await imf.resolve("${verifier}")).bytes,\n)\nconst spec = new imf.SpeculativeModel(drafter, verifierModel)\nconst vocalized = await spec.translate("…")\nconst out = await transliterateAsync(\n "${code}",\n vocalized,\n)` - } return `import { transliterateAsync } from "interscript"\nimport { imf } from "interscript/ml"\n\nconst resolved = await imf.resolve("${voc}")\nconst model = await imf.IMFModel.fromZipBytes(resolved.bytes)\nconst vocalized = await model.translate("…")\nconst out = await transliterateAsync(\n "${code}",\n vocalized,\n)` } $("snippet").textContent = snippetText() @@ -189,10 +182,7 @@ import catalogue from "../data/maps-catalogue.json" async function ensureModel(id: string) { if (mlModel && mlModelId === id) return mlModel const status = $("status") - const spec = id.startsWith("spec:") - const verifierId = spec ? id.slice(5) : id - const drafterId = spec ? "ara-diac-layerdrop-1.0-int4" : "" - const wanted = spec ? [drafterId, verifierId] : [verifierId] + const wanted = [id] const models: Record>> = {} for (const wantedId of wanted) { status.hidden = false @@ -204,12 +194,7 @@ import catalogue from "../data/maps-catalogue.json" }) models[wantedId] = await imf.IMFModel.fromZipBytes(resolved.bytes) } - mlModel = spec - ? (new imf.SpeculativeModel(models[drafterId]!, models[verifierId]!) as unknown as { - translate(t: string, m?: number): Promise - stats(): { accepted: number; drafted: number; blocks: number } | undefined - }) - : (models[verifierId] as unknown as { translate(t: string, m?: number): Promise }) + mlModel = models[id] as unknown as { translate(t: string, m?: number): Promise } mlModelId = id status.hidden = true return mlModel @@ -234,15 +219,6 @@ import catalogue from "../data/maps-catalogue.json" } const result = await worker.transliterate(systemSelect.value, text) out.textContent = result - if (modelId?.startsWith("spec:") && mlModel) { - const stats = ( - mlModel as { stats?: () => { accepted: number; drafted: number; blocks: number } } - ).stats?.() - if (stats) { - status.hidden = false - status.textContent = `Speculative: ${stats.accepted}/${stats.drafted} draft tokens accepted across ${stats.blocks} verifier passes` - } - } const share = new URLSearchParams({ q: ($("input") as HTMLTextAreaElement).value, sys: systemSelect.value,