From f2e9124f4f768176fd3b67e11917da7a0a5f82d8 Mon Sep 17 00:00:00 2001 From: minion1227 Date: Sat, 11 Jul 2026 19:47:25 -0700 Subject: [PATCH] fix(reward): grade x=5, scientific notation, and empty \boxed{} on MATH-500 (#107) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The math grader marked several equivalent-but-unnormalized MATH-500 answers wrong, silently under-counting correct answers and biasing both the sep-CMA-ES training reward and reported eval accuracy downward (same false-negative class as the fixed thousands-comma bug #35): - Variable assignment: a boxed `x = 5` normalized to `x=5` and never matched the bare reference `5`. Grade the value, not the restated variable — drop a leading single-variable assignment in normalize_math_answer, guarded so `==` and inequalities (`x<=5`, `x>=5`) are left intact. - Scientific notation: after \times/\cdot become `*`, `5*10^{3}` parsed as neither float nor fraction, and the sympy fallback read `^` as XOR, so it never equaled `5000`. Rewrite `a*10^{b}` to the canonical `aeb` so the numeric path parses it. - Empty \boxed{}: extract_boxed returned "", which _check_math committed as the (blank) answer instead of falling back to the last number in the text the way a missing box already does. Treat an empty/whitespace box like a missing box. Adds tests/test_reward_math_forms.py: the three answer forms grade correct, while wrong values, inequalities, and an empty box with no number still score 0. Fixes #107 --- src/trinity/orchestration/reward.py | 14 ++++++- tests/test_reward_math_forms.py | 61 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/test_reward_math_forms.py diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index 5621e74..80a941b 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -314,6 +314,11 @@ def normalize_math_answer(ans: str | None) -> str: s = re.sub(r"\\d?frac\s*(\d)\s*(\d)", r"\1/\2", s) s = s.replace(r"\cdot", "*").replace(r"\times", "*") s = re.sub(r"\s+", "", s) + # Scientific notation a*10^{b} (after \times/\cdot became *) -> canonical + # "aeb" so the numeric path can parse it: "5*10^{3}" -> "5e3" -> 5000.0. + # Without this, float()/Fraction both fail and the sympy fallback reads "^" + # as XOR, so a correct order-of-magnitude answer scores 0. + s = re.sub(r"(\d+(?:\.\d+)?)\*10\^\{?(-?\d+)\}?", r"\1e\2", s) # Drop thousands-separator commas so "1,234" compares equal to "1234" (and # parses as a number). extract_last_number already strips these, so without # this the extract path and the compare path disagree and a correct answer @@ -321,6 +326,11 @@ def normalize_math_answer(ans: str | None) -> str: # leaving set/tuple/interval answers like "(1,2)" untouched. s = re.sub(r"(?<=\d),(?=\d{3}(?:\D|$))", "", s) s = s.lower() + # Drop a leading single-variable assignment ("x=5" -> "5"): grade the value, + # not the restated variable. Guarded so "==" (equality) and inequalities like + # "x<=5"/"x>=5" stay intact — only a bare "=" prefix not followed by a + # second "=" is cut ("<"/">" are not word chars, so they never precede the "="). + s = re.sub(r"^[a-z]\w*=(?!=)", "", s) # Canonicalize a pure integer ratio a/b. m = re.fullmatch(r"\(?(-?\d+)\)?/\(?(-?\d+)\)?", s) if m: @@ -409,7 +419,9 @@ def _sympy_equal(a: str, b: str) -> bool: def _check_math(candidate: str, reference: object) -> bool: """True iff the candidate's extracted answer equals the reference.""" extracted = extract_boxed(candidate) - if extracted is None: + if extracted is None or not extracted.strip(): + # No box, or an empty/whitespace \boxed{} that carries no answer: fall + # back to the last number in the text, exactly like a missing box. extracted = extract_last_number(candidate) if extracted is None: # Last resort: compare the whole (normalized) candidate. diff --git a/tests/test_reward_math_forms.py b/tests/test_reward_math_forms.py new file mode 100644 index 0000000..2ef8e46 --- /dev/null +++ b/tests/test_reward_math_forms.py @@ -0,0 +1,61 @@ +"""Offline unit tests for extra MATH-500 answer forms in the grader (reward.py). + +Covers issue #69's sibling class of false negatives (same family as the +already-fixed thousands-comma bug #35): a correct answer written as a variable +assignment (``x=5``), in scientific notation (``5\\times10^{3}``), or after an +empty ``\\boxed{}`` must grade correct — while wrong values, inequalities, and +empty boxes with no number must still score 0. Pure stdlib (no torch / GPU / +network), matching the existing ``score_text`` test precedent. +""" +from __future__ import annotations + +import pytest + +from trinity.orchestration import reward as R + + +@pytest.mark.parametrize( + "candidate, reference", + [ + # Variable assignment: the value is graded, not the restated variable. + (r"\boxed{x=5}", "5"), + (r"\boxed{y = 42}", "42"), + # Scientific notation (\times / \cdot -> canonical exponent). + (r"\boxed{5\times10^{3}}", "5000"), + (r"\boxed{1.5\times10^{3}}", "1500"), + (r"\boxed{5\cdot10^{3}}", "5000"), + (r"\boxed{5\times10^{-3}}", "0.005"), + # Empty \boxed{}: fall back to the last number in the text. + (r"so the answer is \boxed{}. It is 42", "42"), + (r"\boxed{} 1/2", "0.5"), + ], +) +def test_extra_answer_forms_grade_correct(candidate, reference): + assert R.score_text("math500", candidate, reference) == 1.0 + + +@pytest.mark.parametrize( + "candidate, reference", + [ + (r"\boxed{x=6}", "5"), # wrong value, assignment stripped + (r"\boxed{x<=5}", "5"), # inequality must NOT be graded as its RHS + (r"\boxed{x>=5}", "5"), + (r"\boxed{}", "42"), # empty box, no number anywhere -> wrong + (r"\boxed{6\times10^{3}}", "5000"), # wrong magnitude + ], +) +def test_extra_answer_forms_reject_wrong(candidate, reference): + assert R.score_text("math500", candidate, reference) == 0.0 + + +def test_normalize_drops_only_leading_assignment(): + assert R.normalize_math_answer("x=5") == "5" + # Equality assertion and inequalities are left intact (not turned into "=5"). + assert R.normalize_math_answer("x==5") != "5" + assert R.normalize_math_answer("x<=5") != "5" + + +def test_normalize_rewrites_scientific_notation(): + # \times becomes * upstream; the exponent form parses numerically. + assert R._as_number(R.normalize_math_answer(r"5\times10^{3}")) == 5000.0 + assert R._as_number(R.normalize_math_answer(r"5\times10^{-3}")) == 0.005