diff --git a/.github/workflows/hf-publish.yml b/.github/workflows/hf-publish.yml new file mode 100644 index 0000000..8639e06 --- /dev/null +++ b/.github/workflows/hf-publish.yml @@ -0,0 +1,44 @@ +name: hf-publish + +# Publish a models.yaml entry to the Hugging Face org mirror +# (interscript/): assembles the release artifact from index parts +# (sha256-verified), generates the model card, uploads both. +on: + workflow_dispatch: + inputs: + id: + description: model id from models.yaml (e.g. ara-diac-2.0) + required: true + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - run: pip install huggingface_hub pyyaml + - name: assemble artifact from index parts + env: + MODEL_ID: ${{ inputs.id }} + run: | + python - "$MODEL_ID" << 'PY' + import sys, yaml + from pathlib import Path + sys.path.insert(0, "scripts") + from assemble_release import assemble + mid = sys.argv[1] + index = yaml.safe_load(Path("models.yaml").read_text())["models"] + entry = index[mid] + out = assemble(entry, Path("artifact.zip")) + print("assembled", out) + PY + - name: publish to HF + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + MODEL_ID: ${{ inputs.id }} + run: python scripts/publish_hf.py --id "$MODEL_ID" --zip artifact.zip diff --git a/cards/ara-diac-2.0.md b/cards/ara-diac-2.0.md new file mode 100644 index 0000000..292cfc0 --- /dev/null +++ b/cards/ara-diac-2.0.md @@ -0,0 +1,27 @@ +## Benchmark + +SadeedDiac-25 — full 1,200-paragraph set, Misraj's evaluator, zero +skipped examples, greedy decode (protocol-matched published numbers +only): + +| system | Total DER | kind | +|---|---|---| +| Claude-3.7-Sonnet | 1.39% | frontier LLM | +| **ara-diac-2.0 (this model)** | **2.29%** | dedicated, 580M | +| GLM-5.2 | 2.69% | frontier LLM | +| Gemini-Flash-2.0 | 3.19% | frontier LLM | +| GPT-4 | 3.86% | frontier LLM | +| Sadeed-1.5B | 7.29% | leaderboard baseline | + +Word-final iʿrāb endings (the hardest component): 1.33% DER. +Out-of-domain generalization (WikiNews-2024 multi-reference): +17.38 WER / 11.83 DER. + +## Architecture + +ByT5-large-class seq2seq transformer (580M parameters), byte-level +input/output (no tokenizer — id = byte + 3). Trained on classical and +modern Arabic running text with a news-domain adaptation stage over a +morphology-auxiliary curriculum. The canonical distribution is the +Secryst IMF artifact (sha256-verified, parity-gated against the +PyTorch reference); this repository mirrors those exact bytes. diff --git a/docs/RESULTS.md b/docs/RESULTS.md index d5eb7e9..b729811 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -1058,3 +1058,17 @@ effect measured −1.17pp at the small tier); the teacher's three- generation curriculum lineage (r5→r6→r7) also remains un-matched by any single run. Two arms follow: epoch-2 extension and r7-label distillation into the plane architecture. + +## run-020 plane-distill — label-limited distillation loses to raw-text scale (2026-10-05) + +Arm B of the plane-large verdict follow-up: ByT5-small plane encoder +trained on r7 teacher labels (29,322 unique pairs ×6, 3 epochs) scores +full-set **Total DER 4.4462 / Morph 2.8620**. Two clean readings: +(a) on MATCHED teacher-label supervision the plane architecture beats +the seq2seq student (4.4462 vs 4.5701 — architecture advantage +confirmed independent of supervision); (b) label-limited distillation +loses to raw-text diversity at scale (run-018's 551k unique units → +3.5905) — for the plane family the lever is unique-data volume, not +supervision source. A full-corpus teacher labeling pass (~15-20h GPU) +is the conditional distill follow-up, only if the epoch-2 arm falls +short of the 2.2864 gate. diff --git a/scripts/assemble_release.py b/scripts/assemble_release.py new file mode 100644 index 0000000..29050ad --- /dev/null +++ b/scripts/assemble_release.py @@ -0,0 +1,41 @@ +"""Assemble a model artifact from its index parts, sha-verified. + +Downloads every part URL in the entry (in order), verifies each part's +sha256 against the entry, concatenates, verifies the whole-file sha256, +and writes the artifact. Used by the hf-publish workflow and anywhere a +release asset must be reassembled without trusting the transport. +""" + +from __future__ import annotations + +import hashlib +import urllib.request +from pathlib import Path + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _download(url: str, expected_sha: str, label: str) -> bytes: + with urllib.request.urlopen(url, timeout=600) as r: + data = r.read() + got = _sha(data) + if got != expected_sha: + raise ValueError(f"{label}: sha256 mismatch (got {got[:16]}…, want {expected_sha[:16]}…)") + return data + + +def assemble(entry: dict, out: Path, index_dir: Path | None = None) -> Path: + parts = entry.get("parts") or [] + chunks: list[bytes] = [] + for p in parts: + name = p["url"].rsplit("/", 1)[-1] + chunks.append(_download(p["url"], p["sha256"], name)) + whole = b"".join(chunks) + want = entry.get("sha256") + if want and _sha(whole) != want: + raise ValueError(f"assembled artifact sha256 mismatch (want {want[:16]}…)") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(whole) + return out diff --git a/scripts/model_card.py b/scripts/model_card.py new file mode 100644 index 0000000..609f926 --- /dev/null +++ b/scripts/model_card.py @@ -0,0 +1,103 @@ +"""HF model-card generator: a models.yaml entry becomes the README.md +model card for riboseinc/ (the org mirror of the index id). + +Curated benchmark/prose blocks live in cards/.md and are wrapped +by render_card; entries without a curated block get the metrics table +alone. The index remains the single source of truth for numbers. +""" + +from __future__ import annotations + +HF_ORG = "interscript" + +_LANG_BY_PREFIX = { + "ara": "ar", "heb": "he", "fas": "fa", "tha": "th", "urd": "ur", + "khm": "km", +} + +_GIB = 1024 ** 3 + + +def hf_repo_for(model_id: str) -> str: + return f"{HF_ORG}/{model_id}" + + +def _fmt_size(size: int | None) -> str: + if not size: + return "" + return f"{size / _GIB:.2f} GiB" + + +def render_card(model_id: str, entry: dict, variants: list[str], + body: str = "", org: str = HF_ORG) -> str: + lang = _LANG_BY_PREFIX.get(model_id.split("-")[0], "multilingual") + lic = str(entry.get("license", "BSD-3-Clause")).lower() + size = _fmt_size(entry.get("size")) + lines = [ + "---", + "license: " + lic, + "language: " + lang, + "inference: false", + "tags:", + " - secryst", + " - transliteration", + ] + task = entry.get("task", "") + if task == "diacritization": + lines.append(" - diacritization") + elif task == "g2p": + lines.append(" - grapheme-to-phoneme") + lines += [ + " - onnx", + " - byt5", + "---", + "", + f"# {model_id}", + "", + f"Part of the [Secryst](https://secryst.github.io) phonological-layer " + f"family — a dedicated model for **{task}** ({lang.upper()}), distributed " + f"as a Secryst IMF artifact and loadable through any Secryst crystal:", + "", + "```python", + "from secryst import Model", + f"Model.load(\"{model_id}\")", + "```", + "", + ] + if size: + lines += [f"- Artifact: `{entry.get('filename', '')}` ({size})"] + if variants: + lines += ["- Precision variants: " + ", ".join(f"`{v}`" for v in variants)] + metrics = entry.get("metrics") or [] + if metrics: + lines += [ + "", + "## Measured quality", + "", + "Full evaluation sets, zero skipped examples, the benchmark's own evaluator. " + "Provenance for every number is the results record of the training project.", + "", + "| metric | value | provenance |", + "|---|---|---|", + ] + for m in metrics: + prov = m.get("source", "") + lines.append(f"| {m['name']} | {m['value']} | {prov} |") + if body: + lines += ["", body] + lines += [ + "", + "## License", + "", + lic.upper(), + "", + "## Provenance", + "", + "Trained in the [Secryst training monorepo](https://github.com/secryst/secryst-train) " + "and published through the [interscript-ml](https://github.com/interscript/interscript-ml) " + "distribution contract (models.yaml index, sha256-verified release assets, " + "parity-gated artifacts). The canonical download channel is GitHub Releases; " + "this repository is the Hugging Face mirror of the same bytes, pinned to the " + "index revision.", + ] + return "\n".join(lines) + "\n" diff --git a/scripts/publish_hf.py b/scripts/publish_hf.py new file mode 100644 index 0000000..3979906 --- /dev/null +++ b/scripts/publish_hf.py @@ -0,0 +1,100 @@ +"""Publish a models.yaml entry to the Hugging Face org mirror. + +One command, both channels: the GitHub Releases asset stays the +runtime channel; this adds the HF mirror (riboseinc/) with a +generated model card. + +Usage: + python3 scripts/publish_hf.py --id ara-diac-2.0 --zip /path/to/ara-diac-2.0-fp32.zip --dry-run + python3 scripts/publish_hf.py --id ara-diac-2.0 --zip /path/... # uploads; needs HF_TOKEN + +--zip accepts the assembled zip (parts reassembled: cat *.part-* > zip). +Variant ids (e.g. ara-diac-2.0-int8) are separate repos: run once per id. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from model_card import hf_repo_for, render_card # noqa: E402 + +INDEX = "models.yaml" +CARDS_DIR = Path(__file__).resolve().parent.parent / "cards" + + +def load_entry(model_id: str) -> dict: + index = yaml.safe_load(Path(INDEX).read_text(encoding="utf-8"))["models"] + entry = index.get(model_id) + if not entry: + raise SystemExit(f"{model_id} not in {INDEX}") + return entry + + +def variants_of(model_id: str, index: dict) -> list[str]: + return [k for k in index if k != model_id and k.startswith(model_id + "-")] + + +def curated_body(model_id: str) -> str: + card = CARDS_DIR / f"{model_id}.md" + return card.read_text(encoding="utf-8") if card.exists() else "" + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--id", required=True) + ap.add_argument("--zip", required=True, type=Path) + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + index = yaml.safe_load(Path(INDEX).read_text(encoding="utf-8"))["models"] + entry = load_entry(args.id) + repo = hf_repo_for(args.id) + card = render_card(args.id, entry, variants_of(args.id, index), + body=curated_body(args.id)) + zipped = args.zip.read_bytes() + want = entry.get("sha256") + if want: + import hashlib + + got = hashlib.sha256(zipped).hexdigest() + if got != want: + raise SystemExit( + f"artifact sha256 mismatch: got {got[:16]}…, index wants {want[:16]}…" + " — refusing to publish") + + print(f"repo: {repo}") + print(f"artifact: {args.zip} ({len(zipped)} bytes; index sha256 " + f"{entry.get('sha256', '?')[:16]}…)") + print(f"card: {len(card)} chars, {len(card.splitlines())} lines") + if args.dry_run: + print(card) + print("[dry-run] no upload performed") + return + + token = os.environ.get("HF_TOKEN") + if not token: + raise SystemExit("HF_TOKEN not set (fine-grained, write-scoped to the repo)") + from huggingface_hub import HfApi + + api = HfApi(token=token) + api.create_repo(repo, repo_type="model", private=False, exist_ok=True) + api.upload_file(path_or_fileobj=card.encode("utf-8"), + path_in_repo="README.md", repo_id=repo, repo_type="model") + api.upload_file(path_or_fileobj=str(args.zip), + path_in_repo=entry.get("filename", args.zip.name), + repo_id=repo, repo_type="model") + classes = args.zip.with_name("classes.json") + if classes.exists(): # plane artifacts carry the class inventory + api.upload_file(path_or_fileobj=str(classes), + path_in_repo="classes.json", repo_id=repo, repo_type="model") + print(f"uploaded https://huggingface.co/{repo}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_assemble_release.py b/tests/test_assemble_release.py new file mode 100644 index 0000000..0d965b1 --- /dev/null +++ b/tests/test_assemble_release.py @@ -0,0 +1,106 @@ +"""Tests for release-asset assembly from index parts (HF publish path).""" + +import hashlib +import sys +import unittest +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from threading import Thread + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +from assemble_release import assemble # noqa: E402 + +PART_A = b"A" * 1024 +PART_B = b"B" * 700 + + +class TestAssemble(unittest.TestCase): + def setUp(self): + import hashlib + + self.whole = PART_A + PART_B + self.sha = hashlib.sha256(self.whole).hexdigest() + self.bad_sha = hashlib.sha256(b"other").hexdigest() + + def _serve(self, handler): + server = HTTPServer(("127.0.0.1", 0), handler) + Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.shutdown) + return f"http://127.0.0.1:{server.server_address[1]}" + + def _entry(self, base: str, sha: str) -> dict: + return { + "parts": [ + {"url": f"{base}/x.part-00", "sha256": hashlib.sha256(PART_A).hexdigest(), + "size": len(PART_A)}, + {"url": f"{base}/x.part-01", "sha256": hashlib.sha256(PART_B).hexdigest(), + "size": len(PART_B)}, + ], + "sha256": sha, + "filename": "x-1.0.zip", + } + + def test_assembles_parts_in_order_and_verifies_sha(self): + parts = {"x.part-00": PART_A, "x.part-01": PART_B} + base = self._serve(_files_handler(parts)) + d = tempfile.TemporaryDirectory() + self.addCleanup(d.cleanup) + out = Path(d.name) / "x.zip" + assemble(self._entry(base, self.sha), out) + self.assertEqual(out.read_bytes(), self.whole) + + def test_sha_mismatch_rejected(self): + parts = {"x.part-00": PART_A, "x.part-01": PART_B} + base = self._serve(_files_handler(parts)) + d = tempfile.TemporaryDirectory() + self.addCleanup(d.cleanup) + out = Path(d.name) / "x.zip" + with self.assertRaisesRegex(ValueError, "sha256"): + assemble(self._entry(base, self.bad_sha), out) + + def test_part_sha_mismatch_rejected(self): + parts = {"x.part-00": b"X" * 1024, "x.part-01": PART_B} + base = self._serve(_files_handler(parts)) + d = tempfile.TemporaryDirectory() + self.addCleanup(d.cleanup) + out = Path(d.name) / "x.zip" + with self.assertRaisesRegex(ValueError, "part-00"): + assemble(self._entry(base, self.sha), out) + + +def _files_handler(parts: dict[str, bytes]): + class H(BaseHTTPRequestHandler): + def do_GET(self): + name = self.path.rsplit("/", 1)[-1] + data = parts.get(name) + if data is None: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *a): + pass + + return H + + +import contextlib # noqa: E402 +import tempfile # noqa: E402 + + +@contextlib.contextmanager +def _tmpdir(): + d = tempfile.TemporaryDirectory() + try: + yield Path(d.name) + finally: + d.cleanup() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_model_card.py b/tests/test_model_card.py new file mode 100644 index 0000000..e17a423 --- /dev/null +++ b/tests/test_model_card.py @@ -0,0 +1,69 @@ +"""Tests for the HF model-card generator (models.yaml -> README.md).""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +from model_card import hf_repo_for, render_card + +ENTRY = { + "task": "diacritization", + "precision": "fp32", + "filename": "ara-diac-2.0-fp32.zip", + "size": 2780299051, + "metrics": [ + {"name": "der_total_greedy", "value": 2.2864, + "source": "rababa/docs/RESULTS.md#r7-verdict"}, + {"name": "der_morph_greedy", "value": 1.3343, + "source": "rababa/docs/RESULTS.md#r7-verdict"}, + ], + "license": "BSD-3-Clause", +} +VARIANTS = ["ara-diac-2.0-int8"] + + +class TestHfRepo(unittest.TestCase): + def test_repo_is_org_plus_index_id(self): + self.assertEqual(hf_repo_for("ara-diac-2.0"), "interscript/ara-diac-2.0") + + def test_variant_ids_keep_their_own_repo(self): + self.assertEqual(hf_repo_for("ara-diac-small-2.1-int8"), + "interscript/ara-diac-small-2.1-int8") + + +class TestRenderCard(unittest.TestCase): + def test_frontmatter_fields(self): + card = render_card("ara-diac-2.0", ENTRY, VARIANTS, body="## Benchmark\n\n2.29% DER.") + self.assertIn("license: bsd-3-clause", card) + self.assertIn("language: ar", card) + self.assertIn("inference: false", card) + self.assertIn("secryst", card) + + def test_metrics_table_with_provenance(self): + card = render_card("ara-diac-2.0", ENTRY, VARIANTS, body="") + self.assertIn("| der_total_greedy | 2.2864 |", card) + self.assertIn("rababa/docs/RESULTS.md#r7-verdict", card) + + def test_task_and_size(self): + card = render_card("ara-diac-2.0", ENTRY, VARIANTS, body="") + self.assertIn("diacritization", card) + self.assertIn("2.59 GiB", card) + + def test_variants_listed(self): + card = render_card("ara-diac-2.0", ENTRY, VARIANTS, body="") + self.assertIn("ara-diac-2.0-int8", card) + + def test_body_included(self): + card = render_card("ara-diac-2.0", ENTRY, VARIANTS, body="## Benchmark\n\n2.29% DER.") + self.assertIn("## Benchmark", card) + self.assertIn("2.29% DER.", card) + + def test_language_mapping_per_task_language(self): + card = render_card("tha-g2p-base-1.0", {**ENTRY, "task": "g2p"}, [], body="") + self.assertIn("language: th", card) + + +if __name__ == "__main__": + unittest.main()