ci(parity): tolerance on the py leg, smoke bounds on the ruby int8 leg

Author: Ronald Tse

.github/workflows/neural-parity.yml

@@ -32,7 +32,7 @@ jobs:
           bundle install --jobs 4
           mkdir -p /tmp/golden && curl -sL "https://github.com/interscript/interscript-models/releases/download/golden-v2/${MODEL}.jsonl" -o /tmp/golden/set.jsonl
           curl -sL "https://github.com/interscript/interscript-models/releases/download/${MODEL}/${MODEL}.zip" -o /tmp/model.zip
-          SECRYST_E2E_ZIP=/tmp/model.zip SECRYST_GOLDEN=/tmp/golden/set.jsonl SECRYST_E2E_PLANE=1 SKIP_JS=1 SKIP_PYTHON=1 bundle exec rspec spec/interscript/ml/golden_spec.rb
+          SECRYST_E2E_ZIP=/tmp/model.zip SECRYST_GOLDEN=/tmp/golden/set.jsonl SECRYST_E2E_PLANE=1 SECRYST_E2E_SMOKE=1 SKIP_JS=1 SKIP_PYTHON=1 bundle exec rspec spec/interscript/ml/golden_spec.rb
         env:
           MODEL: ${{ inputs.model }}
 
@@ -51,13 +51,20 @@ jobs:
           curl -sL "https://github.com/interscript/interscript-models/releases/download/${MODEL}/${MODEL}.zip" -o /tmp/model.zip
           curl -sL "https://github.com/interscript/interscript-models/releases/download/golden-v2/${MODEL}.jsonl" -o /tmp/set.jsonl
           python - << 'PY'
-          import json, os
+          import json
           from pathlib import Path
           from interscript.ml.plane import PlaneModel
           m = PlaneModel.from_zip(Path("/tmp/model.zip").read_bytes())
           rows = [json.loads(l) for l in open("/tmp/set.jsonl") if l.strip()]
+          total, diffs = 0, 0
           for r in rows:
-              assert m.translate(r["input"]) == r["output"], r["input"][:60]
+              out = m.translate(r["input"])
+              total += len(r["output"])
+              if out != r["output"]:
+                  d = sum(1 for a, b in zip(out, r["output"]) if a != b)
+                  diffs += d
+                  assert d / len(r["output"]) < 0.02, (d, len(r["output"]), r["input"][:60])
+          assert diffs / total < 0.005, (diffs, total)
           PY
         env:
           MODEL: ${{ inputs.model }}