fix(imf): fp16 via torch-native half export — ORT converter broken on real ByT5

The onnxruntime float16 converter produced all-zero encoder hiddens on
the real khm-latn checkpoint (CER 1939pp, every sample mismatched) while
looking fine on the tiny fixture. Exporting the model under .half() is
exact on gold pairs; graph IO becomes float16 (int64 ids unchanged) and
_zero_pasts follows session dtypes.

Measured on 300 khm test samples: fp16 delta 0.43pp, int8 0.84pp —
quantization noise (argmax flips cascading under greedy decode), not
breakage. Whether lossy precisions may exceed the 0.2pp export-fidelity
bar is a policy call pending; fp32 measures 0.0pp.

Author: Ronald Tse

docs/imf-v1.md

@@ -47,7 +47,7 @@ model.zip
 | `tokenizer` | enum | `bytes` (the only v1 value) |
 | `opset` | int | 7..14; must equal the graphs' opset |
 | `decoder` | enum | `plain` \| `kv` (`kv` requires decoder-kv.onnx) |
-| `precision` | enum | `fp32` \| `fp16` \| `int8` |
+| `precision` | enum | `fp32` \| `fp16` \| `int8` (fp16 = torch-native half export: float16 graph IO, int64 ids unchanged; runtimes read dtypes from the session — the ORT float16 converter produces all-zero hiddens on real ByT5 and must not be used) |
 | `license` | str | non-empty (strict gate) |
 | `trained_from` | str | repo + run/checkpoint id |
 | `metrics` | list | `{name, value, protocol, source}`; `source` must be a `RESULTS.md#anchor` (strict gate) |

src/imf/export.py

@@ -242,27 +242,18 @@ def export_graphs(model, out_dir: Path | str) -> dict[str, Path]:
     return paths
 
 
-def convert_fp16(src: Path | str, dst: Path | str) -> Path:
-    """fp32 -> mixed fp16, IO types preserved (encoder/decoder compose cleanly).
-
-    LayerNorm/softmax math stays fp32: ORT's session-time
-    SimplifiedLayerNormFusion crashes on half-converted LN subgraphs
-    (InsertPrecisionFreeCast name mismatch), so the whole decomposition
-    must stay one dtype. Weights (MatMuls) carry the size win.
+def convert_fp16(model):
+    """A fp16 copy of the model for torch-native half export.
+
+    The onnxruntime float16 CONVERTER is not usable here: on real ByT5
+    checkpoints it produces all-zero encoder hiddens (found 2026-08-16,
+    khm-latn — 1939pp CER); exporting the torch model under .half() is
+    exact on gold pairs. Graph IO becomes float16 (input_ids stay int64);
+    runtimes read dtypes from the session, and _zero_pasts follows them.
     """
-    import onnx
-    from onnxruntime.transformers import float16
-
-    block_list = list(float16.DEFAULT_OP_BLOCK_LIST) + [
-        "ReduceMean", "Pow", "Sqrt", "Div", "Sub", "Add", "Mul",
-        "Softmax", "Range", "Exp", "Where", "Less", "Cast",
-    ]
-    model = onnx.load(str(src))
-    converted = float16.convert_float_to_float16(
-        model, keep_io_types=True, op_block_list=block_list
-    )
-    onnx.save(converted, str(dst))
-    return Path(dst)
+    import copy
+
+    return copy.deepcopy(model).half()
 
 
 def quantize_int8(src: Path | str, dst: Path | str) -> Path:
@@ -320,7 +311,8 @@ def _zero_pasts(kv_sess) -> dict[str, object]:
         shape = meta.shape  # [batch, heads, past_seq, d_kv] with str dynamic dims
         heads = shape[1] if isinstance(shape[1], int) else 4
         d_kv = shape[3] if isinstance(shape[3], int) else 8
-        pasts[meta.name] = np.zeros((1, heads, 0, d_kv), dtype=np.float32)
+        dtype = np.float16 if meta.type == "tensor(float16)" else np.float32
+        pasts[meta.name] = np.zeros((1, heads, 0, d_kv), dtype=dtype)
     return pasts
 
 
@@ -372,18 +364,22 @@ def export_zips(
     with tempfile.TemporaryDirectory() as tmp:
         tmp = Path(tmp)
         graphs = export_graphs(model, tmp / "graphs")
+        graphs_16 = (
+            export_graphs(convert_fp16(model), tmp / "graphs-fp16")
+            if "fp16" in precisions
+            else {}
+        )
 
         for precision in precisions:
             variant_dir = tmp / precision
             variant_dir.mkdir()
-            for name, src in graphs.items():
+            sources = graphs if precision != "fp16" else graphs_16
+            for name, src in sources.items():
                 dst = variant_dir / name
-                if precision == "fp32":
+                if precision == "fp32" or precision == "fp16":
                     dst.write_bytes(src.read_bytes())
-                elif precision == "fp16":
-                    convert_fp16(src, dst)
                 elif precision == "int8":
-                    quantize_int8(src, dst)
+                    quantize_int8(graphs[name], dst)
                 else:
                     raise ValueError(f"unknown precision {precision!r}")
             meta = replace(metadata, precision=precision)