From 1e716665260e02d203fed991275faecbe4ce9d21 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 12 Sep 2026 14:33:02 +0800 Subject: [PATCH 1/4] docs(paper-c): decode framing parity section; static-int8 experiment + soup probe scripts Paper C gains section 5.1: dynamically quantized graphs compute activation scales per fed tensor, so decode framing (batch shape) is a parity variable on the same machine/runtime/artifact - measured 0.03/element KV divergence, 33/66 self-acceptance, 0.99-vs-0.46 framing-relative acceptance. Quantized quality parity is framing-scoped; static scales are the proposed structural fix. scripts/static_int8_experiment.py: the pre-registered experiment (TODO.impl/11) - quantize_static on the 2.1 decoder calibrated over 365 real decode feeds in both framings; measures framing equality (gate), fp32 drift (proxy), and speed vs the shipped dynamic int8. modal_teacher_sadeed: --soup-both probes the r6+r7 50/50 weight average under the same windowed protocol (same-basin check; garbage output closes the axis on its own). --- docs/paper-c.adoc | 27 ++++ scripts/static_int8_experiment.py | 231 ++++++++++++++++++++++++++++++ src/gpu/modal_teacher_sadeed.py | 40 ++++-- 3 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 scripts/static_int8_experiment.py diff --git a/docs/paper-c.adoc b/docs/paper-c.adoc index 69a2742..d2ec734 100644 --- a/docs/paper-c.adoc +++ b/docs/paper-c.adoc @@ -76,6 +76,33 @@ tag-pinned index channel (index-v5). - controlled probe matrix isolates the head; fp32 head = 36x fewer flips at +0.4% size; now the export default + release gate +=== 5.1 Decode framing is a parity axis (measured 2026-09-12) + +Dynamically quantized graphs compute activation scales per fed +tensor at runtime (ONNX DynamicQuantizeLinear). Decode framing — how +many tokens share one decoder invocation — is therefore a parity +variable ON THE SAME machine, runtime, and artifact, which neither +cross-runtime nor cross-hardware studies cover: + +- batched (K-token) vs single-step feeds: KV values diverge ~0.03 per + element where the fp32 cache agrees to ~1e-6; inner-position argmax + flips are routine on near-uniform output distributions +- the batched-framing greedy is a materially different decode: a + drafter==verifier pair self-agreed on only 33/66 positions; the + int4->int8 tier's batched-verifier output dropped a word on a + canonical row +- speculative-decode acceptance measured 0.99 under uniform + plain-path framing but 0.46 under runtime framing (single-step + drafting, batched verification) — acceptance figures are + framing-relative and must disclose the framing + +Contract consequence: our quantized quality-parity guarantee is +framing-scoped (the framing is part of the protocol that a cer_delta +gate pins). The proposed structural fix is static activation scales +(quantize_static with calibration over both framings), which removes +the per-run scale computation entirely — pre-registered as +TODO.impl/11 with framing-equality as the gate. + == 6. Reproducibility discipline - subset-overstatement: five instances, up to 3.2x inflation - full-set-only rule; paired sentence-level bootstrap CIs on every diff --git a/scripts/static_int8_experiment.py b/scripts/static_int8_experiment.py new file mode 100644 index 0000000..d81d8e9 --- /dev/null +++ b/scripts/static_int8_experiment.py @@ -0,0 +1,231 @@ +"""Static-int8 decoder experiment (TODO.impl/11, from the framing finding). + +The shipped int8 graphs use quantize_dynamic: DynamicQuantizeLinear +computes ACTIVATION scales per fed tensor at runtime, so decode framing +(single-step vs batched feeds) changes the numerics materially +(RESULTS.md 2026-09-12). This experiment re-quantizes the decoder with +STATIC activation scales (quantize_static, calibrated on real decode +feeds in both framings) and measures: + + A. framing equality - the same greedy decode driven token-by-token + vs driven in 8-token batched calls: identical trajectories? + B. drift vs the fp32 decoder's greedy (quality proxy; full-set DER is + the gate if framing clears) + C. speed - KV-greedy tokens/sec vs the shipped dynamic int8 + +Usage: python3 scripts/static_int8_experiment.py +""" + +from __future__ import annotations + +import json +import sys +import time +import zipfile +from pathlib import Path + +import numpy as np +import onnxruntime as ort + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "runtime" / "src")) +from interscript_ml.tokens import EOS_ID, PAD_ID, encode # noqa: E402 + +GOLDEN = Path.home() / "ml-logs" / "golden" / "ara-diac-small-2.1-int8.jsonl" +DYNAMIC_INT8_ZIP = ( + Path.home() / ".cache" / "secryst" / "models" + / "ara-diac-small-2.1-int8" / "ara-diac-small-2.1-int8.zip" +) +STEPS = 96 + + +def session(path: Path | str | bytes) -> ort.InferenceSession: + opts = ort.SessionOptions() + opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + return ort.InferenceSession( + path if isinstance(path, (bytes, bytearray)) else str(path), opts + ) + + +class Decoder: + """The decoder graph driven through one door; `pasts` are carried + externally so any framing can be expressed on top.""" + + def __init__(self, sess: ort.InferenceSession): + self.sess = sess + self.past_names = [i.name for i in sess.get_inputs() if i.name.startswith("past_")] + self.present_names = [o.name for o in sess.get_outputs() if o.name.startswith("present_")] + + def run(self, tokens: list[int], hidden, pasts: dict[str, np.ndarray] | None): + meta = {i.name: i for i in self.sess.get_inputs()} + feed = { + "input_ids": np.array([tokens], dtype=np.int64), + "encoder_hidden_states": hidden, + } + for name in self.past_names: + feed[name] = ( + pasts[name] + if pasts and name in pasts + else np.zeros( + (1, meta[name].shape[1], 0, meta[name].shape[3]), dtype=np.float32 + ) + ) + out = self.sess.run(None, feed) + names = [o.name for o in self.sess.get_outputs()] + return dict(zip(names, out, strict=True)) + + def split(self, out) -> tuple[int, dict[str, np.ndarray]]: + argmax = int(np.argmax(out["logits"][0, -1])) + pasts = { + n.replace("present_", "past_"): out[n] + for n in self.present_names + } + return argmax, pasts + + @staticmethod + def trim(pasts: dict[str, np.ndarray], seq_len: int) -> dict[str, np.ndarray]: + """Keep the first `seq_len` positions of each KV tensor.""" + return {k: v[:, :, :seq_len, :].copy() for k, v in pasts.items()} + + +def greedy(dec: Decoder, hidden, batch: int) -> list[int]: + """Greedy decode driven with `batch`-token feeds: position i's + prediction comes from a call fed the `batch` tokens ending at i. + batch=1 is the classic incremental loop; batch=8 is the speculative + verification framing.""" + argmax, pasts = dec.split(dec.run([PAD_ID], hidden, None)) + traj = [argmax] + while len(traj) < STEPS and traj[-1] != EOS_ID: + window = traj[-batch:] + out = dec.run(window, hidden, Decoder.trim(pasts, len(pasts[next(iter(pasts))]) if pasts else 0)) + argmax, _ = dec.split(out) + # rebuild pasts through honest incremental steps (the cache must + # reflect every consumed token, mirroring a real runtime) + for tok in window: + _, pasts = dec.split(dec.run([tok], hidden, pasts)) + traj.append(argmax) + return traj + + +def encode_hidden(enc: ort.InferenceSession, text: str): + ids = encode(text) + return enc.run(None, {"input_ids": np.array([ids], dtype=np.int64)})[0] + + +def calibration_samples(fp32: Decoder, enc, rows) -> list[dict]: + """Decoder feeds across framings: prefills, incremental steps, and + batched windows (the shapes whose scales must hold).""" + samples = [] + + def record(tokens, hidden, pasts): + feed = {"input_ids": np.array([tokens], dtype=np.int64), + "encoder_hidden_states": hidden} + for name in fp32.past_names: + feed[name] = ( + pasts[name] + if pasts and name in pasts + else np.zeros((1, feed_shapes[name][0], 0, feed_shapes[name][1]), dtype=np.float32) + ) + samples.append(feed) + + feed_shapes = { + name: (i.shape[1], i.shape[3]) + for i in fp32.sess.get_inputs() + for name in [i.name] + if name.startswith("past_") + } + + for text in rows: + hidden = encode_hidden(enc, text) + argmax, pasts = fp32.split(fp32.run([PAD_ID], hidden, None)) + record([PAD_ID], hidden, None) + window = [] + for _ in range(64): + window.append(argmax) + record([argmax], hidden, pasts) + argmax, pasts = fp32.split(fp32.run([argmax], hidden, pasts)) + if argmax == EOS_ID: + break + if len(window) == 8: + record(window, hidden, Decoder.trim(pasts, max(pasts[next(iter(pasts))].shape[2] - len(window), 0))) + window = [] + return samples + + +def main() -> None: + fp32_zip, workdir = Path(sys.argv[1]), Path(sys.argv[2]) + workdir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(fp32_zip) as zf: + for name in zf.namelist(): + if name.endswith(".onnx"): + (workdir / name).write_bytes(zf.read(name)) + enc = session(workdir / "encoder.onnx") + fp32 = Decoder(session(workdir / "decoder-kv.onnx")) + rows = [json.loads(l)["input"] for l in GOLDEN.read_text().splitlines() if l.strip()] + + print("calibration feeds...", flush=True) + samples = calibration_samples(fp32, enc, rows[:5]) + print(f" {len(samples)} samples", flush=True) + + from onnxruntime.quantization import CalibrationDataReader, QuantFormat, QuantType, quantize_static + + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from imf.export import head_matmul_names + + class Reader(CalibrationDataReader): + def __init__(self, data): + self.data = data + + def get_next(self): + return self.data.pop(0) if self.data else None + + def rewind(self): + pass # one pass; the sample set is the calibration corpus + + static_path = workdir / "decoder-kv-static.onnx" + print("quantize_static...", flush=True) + quantize_static( + str(workdir / "decoder-kv.onnx"), + str(static_path), + calibration_data_reader=Reader(samples), + quant_format=QuantFormat.QOperator, + activation_type=QuantType.QUInt8, + weight_type=QuantType.QInt8, + op_types_to_quantize=["MatMul"], + nodes_to_exclude=head_matmul_names(workdir / "decoder-kv.onnx"), + ) + static = Decoder(session(static_path)) + + test_rows = rows[:5] + for name, dec in (("fp32", fp32), ("static-int8", static)): + same = total = 0 + drift = 0 + for text in test_rows: + hidden = encode_hidden(enc, text) + single = greedy(dec, hidden, 1) + batched = greedy(dec, hidden, 8) + for a, b in zip(single, batched): + total += 1 + same += a == b + ref = greedy(fp32, hidden, 1) if name != "fp32" else single + drift += sum(1 for a, b in zip(single, ref) if a != b) + print(f"{name}: framing single==batched {same}/{total}; drift-vs-fp32 {drift}/{total}") + + # speed vs the shipped dynamic int8 + with zipfile.ZipFile(DYNAMIC_INT8_ZIP) as zf: + dyn_bytes = zf.read("decoder-kv.onnx") + dynamic = Decoder(session(dyn_bytes)) + for name, dec in (("dynamic-int8", dynamic), ("static-int8", static)): + t0 = time.time() + toks = 0 + for text in test_rows: + hidden = encode_hidden(enc, text) + argmax, pasts = dec.split(dec.run([PAD_ID], hidden, None)) + while toks < 10_000 and argmax != EOS_ID: + toks += 1 + argmax, pasts = dec.split(dec.run([argmax], hidden, pasts)) + dt = time.time() - t0 + print(f"{name}: {toks} tokens in {dt:.1f}s = {toks / dt:.0f} tok/s") + + +if __name__ == "__main__": + main() diff --git a/src/gpu/modal_teacher_sadeed.py b/src/gpu/modal_teacher_sadeed.py index 32377aa..bcf70ee 100644 --- a/src/gpu/modal_teacher_sadeed.py +++ b/src/gpu/modal_teacher_sadeed.py @@ -1,11 +1,14 @@ -"""Modal probe: teacher-only SadeedDiac-25 predictions (TODO.qwen-next/10 §2). +"""Modal probe: teacher-only SadeedDiac-25 predictions (TODO.impl/06, +the seed-soup freebie; originally TODO.qwen-next/10 §2). -Runs the canonical teacher (r6 = run-006-morph) over the benchmark -under the published windowed protocol and writes per-paragraph -predictions to the checkpoints volume, so per-domain DER slicing can -be done offline (r7's slice comes from run-007's teacher column). +Runs a teacher over the benchmark under the published windowed +protocol and writes per-paragraph predictions to the checkpoints +volume. With --soup-both, two checkpoints are weight-averaged 50/50 +first (same-basin model soup; garbage output means different basins +and the probe closes negative on its own). modal run --detach src/gpu/modal_teacher_sadeed.py + modal run --detach src/gpu/modal_teacher_sadeed.py --soup-both """ from __future__ import annotations @@ -36,7 +39,9 @@ CHECKPOINTS = modal.Volume.from_name("rababa-checkpoints") TEACHER = "/checkpoints/rababa_arabic_byt5/run-006-morph/best" +SOUP_PARTNER = "/checkpoints/rababa_arabic_byt5/run-007-news/best" OUT = "/checkpoints/probes/r6_sadeed_preds.jsonl" +OUT_SOUP = "/checkpoints/probes/r67_soup_sadeed_preds.jsonl" app = modal.App("interscript-ml-teacher-sadeed", image=IMAGE) @@ -48,11 +53,12 @@ timeout=2 * 3600, volumes={"/checkpoints": CHECKPOINTS}, ) -def teacher_preds() -> dict: +def teacher_preds(soup_both: bool = False) -> dict: import json import sys import pyarrow.parquet as pq + import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # Modal copies the entry file to /root/.py while the repo @@ -63,7 +69,19 @@ def teacher_preds() -> dict: from harness.sadeed import strip_diacritics, windowed_paragraphs tok = AutoTokenizer.from_pretrained("google/byt5-small") - teacher = AutoModelForSeq2SeqLM.from_pretrained(TEACHER).to("cuda").eval() + teacher_path = SOUP_PARTNER if soup_both else TEACHER + out_path = OUT_SOUP if soup_both else OUT + teacher = AutoModelForSeq2SeqLM.from_pretrained(teacher_path) + if soup_both: + partner = AutoModelForSeq2SeqLM.from_pretrained(TEACHER) + soup = { + name: (teacher.state_dict()[name] + partner.state_dict()[name]) / 2 + for name in teacher.state_dict() + } + del partner + teacher.load_state_dict(soup) + del soup + teacher = teacher.to("cuda").eval() teacher.generation_config.max_length = 100_000 table = pq.read_table("/opt/rababa/data/sadeed-diac-25/train.parquet") @@ -71,7 +89,7 @@ def teacher_preds() -> dict: preds = windowed_paragraphs(teacher, tok, inputs, window=1400) - out = Path(OUT) + out = Path(out_path) out.parent.mkdir(parents=True, exist_ok=True) with open(out, "w", encoding="utf-8") as f: for i, (src, pred) in enumerate(zip(inputs, preds, strict=True)): @@ -80,9 +98,9 @@ def teacher_preds() -> dict: + "\n" ) CHECKPOINTS.commit() - return {"rows": len(preds), "out": OUT} + return {"rows": len(preds), "out": str(out_path), "souped": soup_both} @app.local_entrypoint() -def main() -> None: - print(teacher_preds.remote()) +def main(soup_both: bool = False) -> None: + print(teacher_preds.remote(soup_both=soup_both)) From 883bf2ac44ba70c4eeac24336c3378d48534f251 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 12 Sep 2026 14:34:35 +0800 Subject: [PATCH 2/4] docs(RESULTS): r6+r7 soup verdict - same-basin, 2.4188, no gain over r7 --- docs/RESULTS.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 258e3b0..525527e 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -713,3 +713,18 @@ acceptance. The technique's domain is fp-class artifacts or serving paths with consistent framing. The playground tier was pulled accordingly; the runtime keeps SpeculativeModel as measurement infrastructure with the constraint documented. + +## r6+r7 weight soup: same-basin, no free lunch — 2.4188 (2026-09-12) + +50/50 weight average of the two measured Arabic teachers (580M, +run-006-morph and run-007-news), scored under the windowed protocol +on all 1200 rows: **2.4188** vs r6's 2.5997 and r7's 2.289. Per +domain: classical 1.38 (r7 1.36), news 3.31 (r7 3.21), wiki 2.66 +(r7 2.08) — strictly between the parents everywhere; r7 remains the +best available teacher and the supervision choice is unchanged. + +Two conclusions: (a) the checkpoints are same-basin (the soup is a +functional model, confirming linear connectivity between the two +teacher lineages — model-soup mechanics apply), and (b) at this pair +and scale the soup buys nothing over the better parent. The axis +closes negative; recorded so it is not re-derived. From 10fb95eed2ade69a34b0e6d3969d135a7474853f Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 12 Sep 2026 15:15:00 +0800 Subject: [PATCH 3/4] style: unused torch import --- src/gpu/modal_teacher_sadeed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gpu/modal_teacher_sadeed.py b/src/gpu/modal_teacher_sadeed.py index bcf70ee..0aca8c2 100644 --- a/src/gpu/modal_teacher_sadeed.py +++ b/src/gpu/modal_teacher_sadeed.py @@ -58,7 +58,6 @@ def teacher_preds(soup_both: bool = False) -> dict: import sys import pyarrow.parquet as pq - import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer # Modal copies the entry file to /root/.py while the repo From 00272539f8c93c65faa2967d7ee6abd057590c19 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Sat, 12 Sep 2026 15:16:52 +0800 Subject: [PATCH 4/4] docs(RESULTS): lite2 verdict - trained-init depth cut is worse (7.14 vs 5.78) --- docs/RESULTS.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/RESULTS.md b/docs/RESULTS.md index 525527e..6e12d04 100644 --- a/docs/RESULTS.md +++ b/docs/RESULTS.md @@ -728,3 +728,27 @@ functional model, confirming linear connectivity between the two teacher lineages — model-soup mechanics apply), and (b) at this pair and scale the soup buys nothing over the better parent. The axis closes negative; recorded so it is not re-derived. + +## ara-diac-small-lite2 — trained-init depth cut is WORSE: 7.1402 (2026-09-12) + +The lite cell's one untested variable was the layer-drop INIT SOURCE +(TODO.impl/04): run-009 (5.78) drops from generic pretrained +byt5-small; lite2 drops the same layers from the TRAINED 2.1 student, +then runs the identical 6-epoch sequence-KD distill (canonical r7 +labels, sha e70ce991; teacher re-scores 2.2921 on the same run). + +Result: **7.1402** full-set (n=1200), paired-bootstrap gap to teacher +4.29pp [3.83, 4.78]. The trained init is 1.36pp WORSE than the +generic init, not better. + +Reading: generic pretraining keeps encoder layers redundant and +interchangeable, so every-other-layer deletion survives; task +adaptation prunes that redundancy — the layers become co-specialized, +and deleting half of a co-adapted stack breaks more learned +computation. Depth compression on this family survives on generic +init and degrades on adapted init, from either direction (the Hebrew +layerdrop collapsed from generic init under a weaker recipe; the +Arabic adapted-init collapses under the strong one). The lite tier +remains run-009 (5.78); init-source closes negative and the depth +axis now reads 2-of-3 negative. Remaining architecture lever: +TODO.impl/10 (lexical memory), gated as before.