diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index 5621e74..80a941b 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -314,6 +314,11 @@ def normalize_math_answer(ans: str | None) -> str: s = re.sub(r"\\d?frac\s*(\d)\s*(\d)", r"\1/\2", s) s = s.replace(r"\cdot", "*").replace(r"\times", "*") s = re.sub(r"\s+", "", s) + # Scientific notation a*10^{b} (after \times/\cdot became *) -> canonical + # "aeb" so the numeric path can parse it: "5*10^{3}" -> "5e3" -> 5000.0. + # Without this, float()/Fraction both fail and the sympy fallback reads "^" + # as XOR, so a correct order-of-magnitude answer scores 0. + s = re.sub(r"(\d+(?:\.\d+)?)\*10\^\{?(-?\d+)\}?", r"\1e\2", s) # Drop thousands-separator commas so "1,234" compares equal to "1234" (and # parses as a number). extract_last_number already strips these, so without # this the extract path and the compare path disagree and a correct answer @@ -321,6 +326,11 @@ def normalize_math_answer(ans: str | None) -> str: # leaving set/tuple/interval answers like "(1,2)" untouched. s = re.sub(r"(?<=\d),(?=\d{3}(?:\D|$))", "", s) s = s.lower() + # Drop a leading single-variable assignment ("x=5" -> "5"): grade the value, + # not the restated variable. Guarded so "==" (equality) and inequalities like + # "x<=5"/"x>=5" stay intact — only a bare "=" prefix not followed by a + # second "=" is cut ("<"/">" are not word chars, so they never precede the "="). + s = re.sub(r"^[a-z]\w*=(?!=)", "", s) # Canonicalize a pure integer ratio a/b. m = re.fullmatch(r"\(?(-?\d+)\)?/\(?(-?\d+)\)?", s) if m: @@ -409,7 +419,9 @@ def _sympy_equal(a: str, b: str) -> bool: def _check_math(candidate: str, reference: object) -> bool: """True iff the candidate's extracted answer equals the reference.""" extracted = extract_boxed(candidate) - if extracted is None: + if extracted is None or not extracted.strip(): + # No box, or an empty/whitespace \boxed{} that carries no answer: fall + # back to the last number in the text, exactly like a missing box. extracted = extract_last_number(candidate) if extracted is None: # Last resort: compare the whole (normalized) candidate. diff --git a/tests/test_reward_math_forms.py b/tests/test_reward_math_forms.py new file mode 100644 index 0000000..2ef8e46 --- /dev/null +++ b/tests/test_reward_math_forms.py @@ -0,0 +1,61 @@ +"""Offline unit tests for extra MATH-500 answer forms in the grader (reward.py). + +Covers issue #69's sibling class of false negatives (same family as the +already-fixed thousands-comma bug #35): a correct answer written as a variable +assignment (``x=5``), in scientific notation (``5\\times10^{3}``), or after an +empty ``\\boxed{}`` must grade correct — while wrong values, inequalities, and +empty boxes with no number must still score 0. Pure stdlib (no torch / GPU / +network), matching the existing ``score_text`` test precedent. +""" +from __future__ import annotations + +import pytest + +from trinity.orchestration import reward as R + + +@pytest.mark.parametrize( + "candidate, reference", + [ + # Variable assignment: the value is graded, not the restated variable. + (r"\boxed{x=5}", "5"), + (r"\boxed{y = 42}", "42"), + # Scientific notation (\times / \cdot -> canonical exponent). + (r"\boxed{5\times10^{3}}", "5000"), + (r"\boxed{1.5\times10^{3}}", "1500"), + (r"\boxed{5\cdot10^{3}}", "5000"), + (r"\boxed{5\times10^{-3}}", "0.005"), + # Empty \boxed{}: fall back to the last number in the text. + (r"so the answer is \boxed{}. It is 42", "42"), + (r"\boxed{} 1/2", "0.5"), + ], +) +def test_extra_answer_forms_grade_correct(candidate, reference): + assert R.score_text("math500", candidate, reference) == 1.0 + + +@pytest.mark.parametrize( + "candidate, reference", + [ + (r"\boxed{x=6}", "5"), # wrong value, assignment stripped + (r"\boxed{x<=5}", "5"), # inequality must NOT be graded as its RHS + (r"\boxed{x>=5}", "5"), + (r"\boxed{}", "42"), # empty box, no number anywhere -> wrong + (r"\boxed{6\times10^{3}}", "5000"), # wrong magnitude + ], +) +def test_extra_answer_forms_reject_wrong(candidate, reference): + assert R.score_text("math500", candidate, reference) == 0.0 + + +def test_normalize_drops_only_leading_assignment(): + assert R.normalize_math_answer("x=5") == "5" + # Equality assertion and inequalities are left intact (not turned into "=5"). + assert R.normalize_math_answer("x==5") != "5" + assert R.normalize_math_answer("x<=5") != "5" + + +def test_normalize_rewrites_scientific_notation(): + # \times becomes * upstream; the exponent form parses numerically. + assert R._as_number(R.normalize_math_answer(r"5\times10^{3}")) == 5000.0 + assert R._as_number(R.normalize_math_answer(r"5\times10^{-3}")) == 0.005