From 0a99b5612134c284cf55b997b4ee2f9acd59b7a5 Mon Sep 17 00:00:00 2001 From: promptsmith1990 <319963136+promptsmith1990@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:16:11 -0700 Subject: [PATCH] fix(cuad): stop collapsing an undefined AUPR to a score of 0 get_aupr() called np.trapz(processed_precisions, recalls) and, whenever that returned NaN because the integral was undefined, collapsed it to 0 - the worst attainable AUPR. An uncomputable score and a model that scored the worst possible result were indistinguishable in the returned metrics, and CUAD is a legal-contract benchmark where this aggregate is the number people quote. NaN happens whenever a sample's precision/recall is itself NaN, which compute_precision_recall produces for a 0/0 case - a question with no ground-truth answers and no predicted answer text (a legitimate "correctly abstained" sample). That NaN needs to survive two steps to reach the aggregate honestly, and only the second one was fixed by just touching get_aupr: 1. process_precisions() runs a cumulative max() from the high-recall end backward to build the precision envelope. Python's max() is order-dependent with NaN - max(0.5, nan) is 0.5, but max(nan, 0.5) is nan - so an undefined precision could get silently overwritten by a neighboring real value depending on which side of the pair it landed on. Fixed to propagate NaN regardless of argument order. 2. get_aupr() itself: removed the `if np.isnan(aupr): return 0` floor. Also fixed in the same function: np.trapz was removed outright in NumPy 2.0 (not just deprecated), and this package's own setup.py allows numpy>=1.17 with no upper bound in the base install (the numpy<2.0 pin only applies to the tensorflow extras), so get_aupr() raises AttributeError on any modern numpy - not something covered by the reported issue, but the same line, and worth fixing alongside it rather than leaving a second break in the same function. Picks np.trapezoid when available and falls back to np.trapz otherwise. Fixes #801. Tests (metrics/cuad/test_cuad.py, new): - process_precisions propagates NaN through the running max regardless of which side of the pair it starts on (both orderings) - get_aupr still returns a real 0.0 for a genuine single/zero-area curve, unaffected by the fix - get_aupr returns nan instead of flooring to 0 when given a NaN input - end-to-end compute_score() reports a nan aupr for a dataset with one normally-scored sample and one 0/0 (unanswerable, no prediction) sample, rather than silently scoring it as the worst possible AUPR All 5 pass locally (`pytest metrics/cuad/test_cuad.py`), and `black --check` / `isort --check-only` / `flake8` are clean on both touched files. Verified the existing docstring example in cuad.py still produces byte-identical output. --- metrics/cuad/compute_score.py | 21 +++++++-- metrics/cuad/test_cuad.py | 89 +++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 metrics/cuad/test_cuad.py diff --git a/metrics/cuad/compute_score.py b/metrics/cuad/compute_score.py index f35a086e8..3b9030422 100644 --- a/metrics/cuad/compute_score.py +++ b/metrics/cuad/compute_score.py @@ -11,6 +11,10 @@ IOU_THRESH = 0.5 +# np.trapz was removed in NumPy 2.0; np.trapezoid is its replacement and +# doesn't exist before 2.0, so pick whichever the installed numpy provides. +_trapezoid = getattr(np, "trapezoid", None) or np.trapz + def get_jaccard(prediction, ground_truth): remove_tokens = [".", ",", ";", ":"] @@ -107,17 +111,24 @@ def process_precisions(precisions): """ precision_best = precisions[::-1] for i in range(1, len(precision_best)): - precision_best[i] = max(precision_best[i - 1], precision_best[i]) + # Plain max() is order-dependent with NaN (max(0.5, nan) == 0.5, but + # max(nan, 0.5) == nan), so an undefined precision from a 0/0 sample + # could be silently overwritten by a neighboring real value depending + # on which side of the pair it landed on. Make NaN propagate either way. + prev, curr = precision_best[i - 1], precision_best[i] + precision_best[i] = float("nan") if (np.isnan(prev) or np.isnan(curr)) else max(prev, curr) precisions = precision_best[::-1] return precisions def get_aupr(precisions, recalls): + # The integral is NaN when it is undefined (e.g. a precision/recall pair + # that is itself NaN, from a sample with zero ground truths and zero + # predictions). That is "could not be computed", not "worst possible + # score" - collapsing it to 0 would make the two indistinguishable in any + # aggregate, so it is propagated as-is instead. processed_precisions = process_precisions(precisions) - aupr = np.trapz(processed_precisions, recalls) - if np.isnan(aupr): - return 0 - return aupr + return _trapezoid(processed_precisions, recalls) def get_prec_at_recall(precisions, recalls, recall_thresh): diff --git a/metrics/cuad/test_cuad.py b/metrics/cuad/test_cuad.py new file mode 100644 index 000000000..3116ca258 --- /dev/null +++ b/metrics/cuad/test_cuad.py @@ -0,0 +1,89 @@ +# Copyright 2026 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import numpy as np +from compute_score import compute_score, get_aupr, process_precisions + + +class TestProcessPrecisions(unittest.TestCase): + def test_normal_case_is_unchanged(self): + # Reversed input: [0.2, 0.5, 0.1] -> running max from the end -> [0.5, 0.5, 0.1] + self.assertEqual(process_precisions([0.1, 0.5, 0.2]), [0.5, 0.5, 0.2]) + + def test_nan_propagates_regardless_of_which_side_of_the_pair_it_is_on(self): + # Python's built-in max() is order-dependent with NaN: max(0.5, nan) + # is 0.5, but max(nan, 0.5) is nan. process_precisions's running-max + # pass must not let that order dependence silently drop an undefined + # precision - once nan enters the running max (in either argument + # order), it must propagate through every subsequent entry. The last + # element (highest recall) is never touched by the running max itself + # and is unaffected either way. + result = process_precisions([0.5, float("nan"), 0.2]) + self.assertTrue(np.isnan(result[0])) + self.assertTrue(np.isnan(result[1])) + self.assertEqual(result[2], 0.2) + + result = process_precisions([0.2, float("nan"), 0.5]) + self.assertTrue(np.isnan(result[0])) + self.assertTrue(np.isnan(result[1])) + self.assertEqual(result[2], 0.5) + + +class TestCUADAupr(unittest.TestCase): + def test_get_aupr_normal_case_is_unchanged(self): + # Single point: np.trapz has nothing to integrate over, so this is a + # genuine 0.0, not an undefined integral - must stay 0.0. + self.assertEqual(get_aupr([1.0], [1.0]), 0.0) + + # Two points spanning the full recall range at full precision: area is 1.0. + self.assertAlmostEqual(get_aupr([1.0, 1.0], [0.0, 1.0]), 1.0) + + def test_get_aupr_propagates_nan_instead_of_flooring_to_zero(self): + # A NaN anywhere in precisions/recalls (e.g. from a sample with zero + # ground truths and zero predictions, which compute_precision_recall + # scores as 0/0 -> nan) makes the integral undefined. That must be + # reported as nan, not silently floored to 0 (the worst attainable score). + aupr = get_aupr([1.0, float("nan"), 0.5], [0.0, 0.5, 1.0]) + self.assertTrue(np.isnan(aupr)) + + def test_compute_score_reports_nan_aupr_when_a_sample_has_undefined_precision_recall(self): + # End-to-end, with a second, normally-scored sample alongside the + # degenerate one (a single-sample dataset is a separate edge case: + # np.trapezoid returns 0.0 for any one-point curve regardless of its + # value, real or nan, since there's no interval to integrate over). + # + # q2 has no ground-truth answers and no predicted answer text - a + # legitimate "correctly abstained" case, but it leaves precision and + # recall undefined (0/0) for that sample. The aggregate aupr must + # surface as nan rather than as a perfect-worst-case 0.0. + dataset = [ + { + "paragraphs": [ + { + "qas": [ + {"id": "q1", "answers": [{"text": "foo"}]}, + {"id": "q2", "answers": []}, + ] + } + ] + } + ] + predictions = {"q1": ["foo"], "q2": []} + result = compute_score(dataset, predictions) + self.assertTrue(np.isnan(result["aupr"])) + + +if __name__ == "__main__": + unittest.main()