feat(hf): Hugging Face org mirror pipeline (riboseinc/<id>)

.github/workflows/hf-publish.yml

@@ -0,0 +1,44 @@
+name: hf-publish
+
+# Publish a models.yaml entry to the Hugging Face org mirror
+# (interscript/<id>): assembles the release artifact from index parts
+# (sha256-verified), generates the model card, uploads both.
+on:
+  workflow_dispatch:
+    inputs:
+      id:
+        description: model id from models.yaml (e.g. ara-diac-2.0)
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+      - run: pip install huggingface_hub pyyaml
+      - name: assemble artifact from index parts
+        env:
+          MODEL_ID: ${{ inputs.id }}
+        run: |
+          python - "$MODEL_ID" << 'PY'
+          import sys, yaml
+          from pathlib import Path
+          sys.path.insert(0, "scripts")
+          from assemble_release import assemble
+          mid = sys.argv[1]
+          index = yaml.safe_load(Path("models.yaml").read_text())["models"]
+          entry = index[mid]
+          out = assemble(entry, Path("artifact.zip"))
+          print("assembled", out)
+          PY
+      - name: publish to HF
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          MODEL_ID: ${{ inputs.id }}
+        run: python scripts/publish_hf.py --id "$MODEL_ID" --zip artifact.zip

cards/ara-diac-2.0.md

@@ -0,0 +1,27 @@
+## Benchmark
+
+SadeedDiac-25 — full 1,200-paragraph set, Misraj's evaluator, zero
+skipped examples, greedy decode (protocol-matched published numbers
+only):
+
+| system | Total DER | kind |
+|---|---|---|
+| Claude-3.7-Sonnet | 1.39% | frontier LLM |
+| **ara-diac-2.0 (this model)** | **2.29%** | dedicated, 580M |
+| GLM-5.2 | 2.69% | frontier LLM |
+| Gemini-Flash-2.0 | 3.19% | frontier LLM |
+| GPT-4 | 3.86% | frontier LLM |
+| Sadeed-1.5B | 7.29% | leaderboard baseline |
+
+Word-final iʿrāb endings (the hardest component): 1.33% DER.
+Out-of-domain generalization (WikiNews-2024 multi-reference):
+17.38 WER / 11.83 DER.
+
+## Architecture
+
+ByT5-large-class seq2seq transformer (580M parameters), byte-level
+input/output (no tokenizer — id = byte + 3). Trained on classical and
+modern Arabic running text with a news-domain adaptation stage over a
+morphology-auxiliary curriculum. The canonical distribution is the
+Secryst IMF artifact (sha256-verified, parity-gated against the
+PyTorch reference); this repository mirrors those exact bytes.

docs/RESULTS.md

@@ -1058,3 +1058,17 @@ effect measured −1.17pp at the small tier); the teacher's three-
 generation curriculum lineage (r5→r6→r7) also remains un-matched by
 any single run. Two arms follow: epoch-2 extension and r7-label
 distillation into the plane architecture.
+
+## run-020 plane-distill — label-limited distillation loses to raw-text scale (2026-10-05)
+
+Arm B of the plane-large verdict follow-up: ByT5-small plane encoder
+trained on r7 teacher labels (29,322 unique pairs ×6, 3 epochs) scores
+full-set **Total DER 4.4462 / Morph 2.8620**. Two clean readings:
+(a) on MATCHED teacher-label supervision the plane architecture beats
+the seq2seq student (4.4462 vs 4.5701 — architecture advantage
+confirmed independent of supervision); (b) label-limited distillation
+loses to raw-text diversity at scale (run-018's 551k unique units →
+3.5905) — for the plane family the lever is unique-data volume, not
+supervision source. A full-corpus teacher labeling pass (~15-20h GPU)
+is the conditional distill follow-up, only if the epoch-2 arm falls
+short of the 2.2864 gate.

scripts/assemble_release.py

@@ -0,0 +1,41 @@
+"""Assemble a model artifact from its index parts, sha-verified.
+
+Downloads every part URL in the entry (in order), verifies each part's
+sha256 against the entry, concatenates, verifies the whole-file sha256,
+and writes the artifact. Used by the hf-publish workflow and anywhere a
+release asset must be reassembled without trusting the transport.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import urllib.request
+from pathlib import Path
+
+
+def _sha(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _download(url: str, expected_sha: str, label: str) -> bytes:
+    with urllib.request.urlopen(url, timeout=600) as r:
+        data = r.read()
+    got = _sha(data)
+    if got != expected_sha:
+        raise ValueError(f"{label}: sha256 mismatch (got {got[:16]}…, want {expected_sha[:16]}…)")
+    return data
+
+
+def assemble(entry: dict, out: Path, index_dir: Path | None = None) -> Path:
+    parts = entry.get("parts") or []
+    chunks: list[bytes] = []
+    for p in parts:
+        name = p["url"].rsplit("/", 1)[-1]
+        chunks.append(_download(p["url"], p["sha256"], name))
+    whole = b"".join(chunks)
+    want = entry.get("sha256")
+    if want and _sha(whole) != want:
+        raise ValueError(f"assembled artifact sha256 mismatch (want {want[:16]}…)")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_bytes(whole)
+    return out

scripts/model_card.py

@@ -0,0 +1,103 @@
+"""HF model-card generator: a models.yaml entry becomes the README.md
+model card for riboseinc/<id> (the org mirror of the index id).
+
+Curated benchmark/prose blocks live in cards/<id>.md and are wrapped
+by render_card; entries without a curated block get the metrics table
+alone. The index remains the single source of truth for numbers.
+"""
+
+from __future__ import annotations
+
+HF_ORG = "interscript"
+
+_LANG_BY_PREFIX = {
+    "ara": "ar", "heb": "he", "fas": "fa", "tha": "th", "urd": "ur",
+    "khm": "km",
+}
+
+_GIB = 1024 ** 3
+
+
+def hf_repo_for(model_id: str) -> str:
+    return f"{HF_ORG}/{model_id}"
+
+
+def _fmt_size(size: int | None) -> str:
+    if not size:
+        return ""
+    return f"{size / _GIB:.2f} GiB"
+
+
+def render_card(model_id: str, entry: dict, variants: list[str],
+                body: str = "", org: str = HF_ORG) -> str:
+    lang = _LANG_BY_PREFIX.get(model_id.split("-")[0], "multilingual")
+    lic = str(entry.get("license", "BSD-3-Clause")).lower()
+    size = _fmt_size(entry.get("size"))
+    lines = [
+        "---",
+        "license: " + lic,
+        "language: " + lang,
+        "inference: false",
+        "tags:",
+        "  - secryst",
+        "  - transliteration",
+    ]
+    task = entry.get("task", "")
+    if task == "diacritization":
+        lines.append("  - diacritization")
+    elif task == "g2p":
+        lines.append("  - grapheme-to-phoneme")
+    lines += [
+        "  - onnx",
+        "  - byt5",
+        "---",
+        "",
+        f"# {model_id}",
+        "",
+        f"Part of the [Secryst](https://secryst.github.io) phonological-layer "
+        f"family — a dedicated model for **{task}** ({lang.upper()}), distributed "
+        f"as a Secryst IMF artifact and loadable through any Secryst crystal:",
+        "",
+        "```python",
+        "from secryst import Model",
+        f"Model.load(\"{model_id}\")",
+        "```",
+        "",
+    ]
+    if size:
+        lines += [f"- Artifact: `{entry.get('filename', '')}` ({size})"]
+    if variants:
+        lines += ["- Precision variants: " + ", ".join(f"`{v}`" for v in variants)]
+    metrics = entry.get("metrics") or []
+    if metrics:
+        lines += [
+            "",
+            "## Measured quality",
+            "",
+            "Full evaluation sets, zero skipped examples, the benchmark's own evaluator. "
+            "Provenance for every number is the results record of the training project.",
+            "",
+            "| metric | value | provenance |",
+            "|---|---|---|",
+        ]
+        for m in metrics:
+            prov = m.get("source", "")
+            lines.append(f"| {m['name']} | {m['value']} | {prov} |")
+    if body:
+        lines += ["", body]
+    lines += [
+        "",
+        "## License",
+        "",
+        lic.upper(),
+        "",
+        "## Provenance",
+        "",
+        "Trained in the [Secryst training monorepo](https://github.com/secryst/secryst-train) "
+        "and published through the [interscript-ml](https://github.com/interscript/interscript-ml) "
+        "distribution contract (models.yaml index, sha256-verified release assets, "
+        "parity-gated artifacts). The canonical download channel is GitHub Releases; "
+        "this repository is the Hugging Face mirror of the same bytes, pinned to the "
+        "index revision.",
+    ]
+    return "\n".join(lines) + "\n"

scripts/publish_hf.py

@@ -0,0 +1,100 @@
+"""Publish a models.yaml entry to the Hugging Face org mirror.
+
+One command, both channels: the GitHub Releases asset stays the
+runtime channel; this adds the HF mirror (riboseinc/<id>) with a
+generated model card.
+
+Usage:
+    python3 scripts/publish_hf.py --id ara-diac-2.0 --zip /path/to/ara-diac-2.0-fp32.zip --dry-run
+    python3 scripts/publish_hf.py --id ara-diac-2.0 --zip /path/...   # uploads; needs HF_TOKEN
+
+--zip accepts the assembled zip (parts reassembled: cat *.part-* > zip).
+Variant ids (e.g. ara-diac-2.0-int8) are separate repos: run once per id.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from model_card import hf_repo_for, render_card  # noqa: E402
+
+INDEX = "models.yaml"
+CARDS_DIR = Path(__file__).resolve().parent.parent / "cards"
+
+
+def load_entry(model_id: str) -> dict:
+    index = yaml.safe_load(Path(INDEX).read_text(encoding="utf-8"))["models"]
+    entry = index.get(model_id)
+    if not entry:
+        raise SystemExit(f"{model_id} not in {INDEX}")
+    return entry
+
+
+def variants_of(model_id: str, index: dict) -> list[str]:
+    return [k for k in index if k != model_id and k.startswith(model_id + "-")]
+
+
+def curated_body(model_id: str) -> str:
+    card = CARDS_DIR / f"{model_id}.md"
+    return card.read_text(encoding="utf-8") if card.exists() else ""
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--id", required=True)
+    ap.add_argument("--zip", required=True, type=Path)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    index = yaml.safe_load(Path(INDEX).read_text(encoding="utf-8"))["models"]
+    entry = load_entry(args.id)
+    repo = hf_repo_for(args.id)
+    card = render_card(args.id, entry, variants_of(args.id, index),
+                       body=curated_body(args.id))
+    zipped = args.zip.read_bytes()
+    want = entry.get("sha256")
+    if want:
+        import hashlib
+
+        got = hashlib.sha256(zipped).hexdigest()
+        if got != want:
+            raise SystemExit(
+                f"artifact sha256 mismatch: got {got[:16]}…, index wants {want[:16]}…"
+                " — refusing to publish")
+
+    print(f"repo:      {repo}")
+    print(f"artifact:  {args.zip} ({len(zipped)} bytes; index sha256 "
+          f"{entry.get('sha256', '?')[:16]}…)")
+    print(f"card:      {len(card)} chars, {len(card.splitlines())} lines")
+    if args.dry_run:
+        print(card)
+        print("[dry-run] no upload performed")
+        return
+
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        raise SystemExit("HF_TOKEN not set (fine-grained, write-scoped to the repo)")
+    from huggingface_hub import HfApi
+
+    api = HfApi(token=token)
+    api.create_repo(repo, repo_type="model", private=False, exist_ok=True)
+    api.upload_file(path_or_fileobj=card.encode("utf-8"),
+                    path_in_repo="README.md", repo_id=repo, repo_type="model")
+    api.upload_file(path_or_fileobj=str(args.zip),
+                    path_in_repo=entry.get("filename", args.zip.name),
+                    repo_id=repo, repo_type="model")
+    classes = args.zip.with_name("classes.json")
+    if classes.exists():  # plane artifacts carry the class inventory
+        api.upload_file(path_or_fileobj=str(classes),
+                        path_in_repo="classes.json", repo_id=repo, repo_type="model")
+    print(f"uploaded https://huggingface.co/{repo}")
+
+
+if __name__ == "__main__":
+    main()