From cc960d040b89a6c298937f1e0db389e3a4d4d3d0 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Wed, 7 Oct 2026 15:42:20 +0800 Subject: [PATCH] =?UTF-8?q?docs+bench:=20WO03=20Thai=20hybrid=20=E2=80=94?= =?UTF-8?q?=20dict=20fast-path=200.1478=20PER=20at=200.73=20ms/utt,=20110x?= =?UTF-8?q?=20faster=20than=20model-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/thai_hybrid_bench.py | 70 +++++++++++++++++++++++++++++++++ docs/RESULTS.md | 26 ++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 benchmarks/thai_hybrid_bench.py diff --git a/benchmarks/thai_hybrid_bench.py b/benchmarks/thai_hybrid_bench.py new file mode 100644 index 0000000..0886e4c --- /dev/null +++ b/benchmarks/thai_hybrid_bench.py @@ -0,0 +1,70 @@ +"""WO03: Thai hybrid bench — dict-only vs model-only vs hybrid. + +Protocol: kaikki test (1,219 sentences), corpus-level character PER +(true Levenshtein over the concatenated corpus), greedy, int8 runtime, +CPU latency wall-clock per utterance. + +Usage: PYTHONPATH=/src python3 thai_hybrid_bench.py +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +TEST = Path.home() / "src/interscript/ml-qwen-feat/benchmarks/thai-kaikki-g2p/test.jsonl" +ART = Path("/tmp/tha-g2p-small-1.0-int8.zip") +LEX = Path("/tmp/tha-lexicon.json") + + +def levenshtein(a: str, b: str) -> int: + if abs(len(a) - len(b)) > 4000: + return max(len(a), len(b)) + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + cur = [i] + for j, cb in enumerate(b, 1): + cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + + +def corpus_per(pairs: list[tuple[str, str]]) -> float: + dist = sum(levenshtein(p, t) for p, t in pairs) + total = sum(len(t) for _, t in pairs) + return 100.0 * dist / total + + +def main() -> None: + from interscript.ml.model import Model + from interscript.ml.thai_hybrid import hybrid_translate, segment + + lex = json.loads(LEX.read_text()) + model = Model.load(str(ART)) + + rows = [json.loads(l) for l in TEST.read_text().splitlines() if l.strip()] + print(f"sentences={len(rows)} lexicon={len(lex)}", flush=True) + + def dict_only(text: str) -> str: + return "".join(ipa if ipa is not None else chunk + for chunk, ipa in segment(text, lex)) + + def hybrid(text: str) -> str: + return hybrid_translate(text, lex, model) + + for name, fn in (("dict-only", dict_only), + ("hybrid", hybrid), + ("model-only", lambda t: model.translate(t))): + t0 = time.time() + pairs = [(fn(r["src"]), r["tgt"]) for r in rows] + wall = time.time() - t0 + per = corpus_per(pairs) + print(json.dumps({"mode": name, "per": round(per, 4), + "ms_per_utt": round(1000 * wall / len(rows), 3)}, + ensure_ascii=False), flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/RESULTS.md b/docs/RESULTS.md index d9d4c72..b10c4d1 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -1144,3 +1144,29 @@ framing: the gap is register/reading-selection, deeper than OOV, and closes only with in-register gold supervision — which is why the teacher-labeled news mix in r7 moved in-domain DER but not this number. Never quote QCRI's 2.70 against our numbers without this paragraph. + +## WO03 Thai hybrid tier — dictionary fast-path beats the neural model on BOTH axes (2026-10-07) + +FastThaiG2P (arXiv 2608.12814) response built as specced in TODO.sota/03: +Kaikki Thai headword IPA (17,434 munch keys, >=2-char) as a maximal-munch +fast path, neural model as whole-span OOV fallback +(interscript-py thai_hybrid; bench at benchmarks/thai_hybrid_bench.py). + +| mode (kaikki test, 1,219 sent, greedy, int8, CPU) | corpus PER | ms/utt | +|---|---|---| +| dict-only | 0.2217 | 0.66 | +| **hybrid (dict + neural OOV)** | **0.1478** | **0.73** | +| model-only (tha-g2p-small-1.0 int8) | 2.9166 | 82.11 | + +Gate: hybrid within 1.2x dict latency (1.10x) and within +0.3pp PER of +model-only — CLEARED with margin (hybrid BEATS model-only by 2.77pp). + +Honest caveats: (1) the benchmark is kaikki-derived, so dictionary +coverage/quality are inflated versus wild text — wild OOV runs go to the +neural model and hybrid converges toward model-only; (2) FastThaiG2P's +0.15 ms/utt is ~5x faster than our pure-python munch (optimizable, and +not the bottleneck: TTS decode dwarfs G2P); (3) their PER (26.9) is on +their own synthetic set — never quote across sets. The lexicon artifact +(tha-lexicon-kaikki.json) ships as a model-release asset with CC BY-SA +3.0 attribution to Wiktionary/Kaikki — it is deliberately NOT committed +to this BSD-3 repo.