Skip to content

Commit 2e7e582

Browse files
author
Ronald Tse
committed
ci: neural cross-runtime parity workflow (golden-v2, quantized tolerance)
1 parent be1384f commit 2e7e582

1 file changed

Lines changed: 63 additions & 0 deletions

File tree

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: neural-parity
2+
3+
# Cross-runtime parity on the neural golden corpus (golden-v2+).
4+
# Quantized contract: per-row agreement >= 99.5%, corpus >= 99.9%.
5+
# Python is the reference (golden generator); ruby and ts run the same
6+
# sets. The ts plane leg lands with the ts plane runtime port.
7+
8+
on:
9+
workflow_dispatch:
10+
inputs:
11+
model:
12+
description: "model id (plane family)"
13+
required: true
14+
default: ara-diac-plane-1.0
15+
16+
permissions:
17+
contents: read
18+
19+
jobs:
20+
ruby:
21+
runs-on: ubuntu-latest
22+
steps:
23+
- uses: actions/checkout@v7
24+
- uses: ruby/setup-ruby@v1
25+
with:
26+
ruby-version: "3.4"
27+
bundler-cache: false
28+
- run: git clone --depth 1 https://github.com/interscript/interscript-ruby /tmp/rt
29+
- name: run the golden set
30+
working-directory: /tmp/rt
31+
run: |
32+
bundle install --jobs 4
33+
mkdir -p /tmp/golden && curl -sL "https://github.com/interscript/interscript-models/releases/download/golden-v2/${MODEL}.jsonl" -o /tmp/golden/set.jsonl
34+
curl -sL "https://github.com/interscript/interscript-models/releases/download/${MODEL}/${MODEL}.zip" -o /tmp/model.zip
35+
SECRYST_E2E_ZIP=/tmp/model.zip SECRYST_GOLDEN=/tmp/golden/set.jsonl SECRYST_E2E_PLANE=1 SKIP_JS=1 SKIP_PYTHON=1 bundle exec rspec spec/interscript/ml/golden_spec.rb
36+
env:
37+
MODEL: ${{ inputs.model }}
38+
39+
python:
40+
runs-on: ubuntu-latest
41+
steps:
42+
- uses: actions/checkout@v7
43+
- uses: actions/setup-python@v6
44+
with:
45+
python-version: "3.12"
46+
- run: git clone --depth 1 https://github.com/interscript/interscript-py /tmp/py
47+
- name: regenerate byte-exact from the reference
48+
working-directory: /tmp/py
49+
run: |
50+
pip install -e ".[ml]"
51+
curl -sL "https://github.com/interscript/interscript-models/releases/download/${MODEL}/${MODEL}.zip" -o /tmp/model.zip
52+
curl -sL "https://github.com/interscript/interscript-models/releases/download/golden-v2/${MODEL}.jsonl" -o /tmp/set.jsonl
53+
python - << 'PY'
54+
import json, os
55+
from pathlib import Path
56+
from interscript.ml.plane import PlaneModel
57+
m = PlaneModel.from_zip(Path("/tmp/model.zip").read_bytes())
58+
rows = [json.loads(l) for l in open("/tmp/set.jsonl") if l.strip()]
59+
for r in rows:
60+
assert m.translate(r["input"]) == r["output"], r["input"][:60]
61+
PY
62+
env:
63+
MODEL: ${{ inputs.model }}

0 commit comments

Comments
 (0)