Skip to content

Commit 138c0b5

Browse files
authored
Merge pull request #207 from interscript/golden-matrix-test
test(runtime): golden-v1 matrix consumer
2 parents 11c94be + eb6535a commit 138c0b5

1 file changed

Lines changed: 22 additions & 0 deletions

File tree

‎runtime/tests/test_model.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,3 +156,25 @@ def test_golden_set_e2e() -> None:
156156
rows = [json.loads(line) for line in golden.read_text(encoding="utf-8").splitlines()]
157157
for row in rows:
158158
assert model.translate(row["input"], max_len=128) == row["output"], row["input"]
159+
160+
161+
def test_golden_matrix() -> None:
162+
"""golden-v1 corpus: every model's released zip must reproduce its
163+
golden rows byte-identically. Set GOLDEN_DIR (release checkout);
164+
each golden file maps to a model id resolved from the index."""
165+
import glob
166+
167+
golden_dir = os.environ.get("GOLDEN_DIR")
168+
if not golden_dir:
169+
pytest.skip("set GOLDEN_DIR to a golden-v1 checkout")
170+
files = sorted(glob.glob(str(Path(golden_dir) / "*.jsonl")))
171+
assert files, "no golden files"
172+
for path in files:
173+
model_id = Path(path).stem
174+
model = Model.load(model_id)
175+
rows = [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()]
176+
assert rows, model_id
177+
for row in rows:
178+
got = model.translate(row["input"], max_len=max(256, 4 * len(row["input"])))
179+
assert got == row["output"], (model_id, row["input"])
180+
del model

0 commit comments

Comments
 (0)