WO03: Thai hybrid bench + results

benchmarks/thai_hybrid_bench.py

@@ -0,0 +1,70 @@
+"""WO03: Thai hybrid bench — dict-only vs model-only vs hybrid.
+
+Protocol: kaikki test (1,219 sentences), corpus-level character PER
+(true Levenshtein over the concatenated corpus), greedy, int8 runtime,
+CPU latency wall-clock per utterance.
+
+Usage: PYTHONPATH=<interscript-py>/src python3 thai_hybrid_bench.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+TEST = Path.home() / "src/interscript/ml-qwen-feat/benchmarks/thai-kaikki-g2p/test.jsonl"
+ART = Path("/tmp/tha-g2p-small-1.0-int8.zip")
+LEX = Path("/tmp/tha-lexicon.json")
+
+
+def levenshtein(a: str, b: str) -> int:
+    if abs(len(a) - len(b)) > 4000:
+        return max(len(a), len(b))
+    prev = list(range(len(b) + 1))
+    for i, ca in enumerate(a, 1):
+        cur = [i]
+        for j, cb in enumerate(b, 1):
+            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
+        prev = cur
+    return prev[-1]
+
+
+def corpus_per(pairs: list[tuple[str, str]]) -> float:
+    dist = sum(levenshtein(p, t) for p, t in pairs)
+    total = sum(len(t) for _, t in pairs)
+    return 100.0 * dist / total
+
+
+def main() -> None:
+    from interscript.ml.model import Model
+    from interscript.ml.thai_hybrid import hybrid_translate, segment
+
+    lex = json.loads(LEX.read_text())
+    model = Model.load(str(ART))
+
+    rows = [json.loads(l) for l in TEST.read_text().splitlines() if l.strip()]
+    print(f"sentences={len(rows)} lexicon={len(lex)}", flush=True)
+
+    def dict_only(text: str) -> str:
+        return "".join(ipa if ipa is not None else chunk
+                       for chunk, ipa in segment(text, lex))
+
+    def hybrid(text: str) -> str:
+        return hybrid_translate(text, lex, model)
+
+    for name, fn in (("dict-only", dict_only),
+                     ("hybrid", hybrid),
+                     ("model-only", lambda t: model.translate(t))):
+        t0 = time.time()
+        pairs = [(fn(r["src"]), r["tgt"]) for r in rows]
+        wall = time.time() - t0
+        per = corpus_per(pairs)
+        print(json.dumps({"mode": name, "per": round(per, 4),
+                          "ms_per_utt": round(1000 * wall / len(rows), 3)},
+                         ensure_ascii=False), flush=True)
+
+
+if __name__ == "__main__":
+    main()

docs/RESULTS.md

@@ -1144,3 +1144,29 @@ framing: the gap is register/reading-selection, deeper than OOV, and
 closes only with in-register gold supervision — which is why the
 teacher-labeled news mix in r7 moved in-domain DER but not this number.
 Never quote QCRI's 2.70 against our numbers without this paragraph.
+
+## WO03 Thai hybrid tier — dictionary fast-path beats the neural model on BOTH axes (2026-10-07)
+
+FastThaiG2P (arXiv 2608.12814) response built as specced in TODO.sota/03:
+Kaikki Thai headword IPA (17,434 munch keys, >=2-char) as a maximal-munch
+fast path, neural model as whole-span OOV fallback
+(interscript-py thai_hybrid; bench at benchmarks/thai_hybrid_bench.py).
+
+| mode (kaikki test, 1,219 sent, greedy, int8, CPU) | corpus PER | ms/utt |
+|---|---|---|
+| dict-only | 0.2217 | 0.66 |
+| **hybrid (dict + neural OOV)** | **0.1478** | **0.73** |
+| model-only (tha-g2p-small-1.0 int8) | 2.9166 | 82.11 |
+
+Gate: hybrid within 1.2x dict latency (1.10x) and within +0.3pp PER of
+model-only — CLEARED with margin (hybrid BEATS model-only by 2.77pp).
+
+Honest caveats: (1) the benchmark is kaikki-derived, so dictionary
+coverage/quality are inflated versus wild text — wild OOV runs go to the
+neural model and hybrid converges toward model-only; (2) FastThaiG2P's
+0.15 ms/utt is ~5x faster than our pure-python munch (optimizable, and
+not the bottleneck: TTS decode dwarfs G2P); (3) their PER (26.9) is on
+their own synthetic set — never quote across sets. The lexicon artifact
+(tha-lexicon-kaikki.json) ships as a model-release asset with CC BY-SA
+3.0 attribution to Wiktionary/Kaikki — it is deliberately NOT committed
+to this BSD-3 repo.