From 8303bb7a51930af0d088d0fe0c151ca5d1b03ec6 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:57:45 +0530 Subject: [PATCH] fix(truthbench): retry a generated-code run once after a native crash The generated_code_sandbox gate failed three times in a few hours with "generated code exited -11" on different copilot cases (finance, and healthcare, plus the copilot test on py3.13). Each time it was a segfault inside pandas native code, and each rerun of the same commit passed. The crash is not in the generated code, and failing the release gate on it hides real regressions behind flake reruns. When the sandbox child is killed by a signal (negative exit code), run it once more with the input file restored. Timeouts and ordinary non-zero exits are not retried, and a crash on the retry still fails the gate with the faulthandler excerpt. Every crash, including one a retry recovered from, is kept in GeneratedCodeResult.native_crashes, and the runner prints each one to stderr so a flaky pass stays visible in the CI log. --- benchmarks/truthbench/generated_code.py | 52 ++++++++++++++++++------- benchmarks/truthbench/runner.py | 7 ++++ tests/truthbench/test_generated_code.py | 47 ++++++++++++++++++++++ 3 files changed, 92 insertions(+), 14 deletions(-) diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py index b2b866f7..317a3dbd 100644 --- a/benchmarks/truthbench/generated_code.py +++ b/benchmarks/truthbench/generated_code.py @@ -91,6 +91,13 @@ def _stderr_excerpt(stderr: str) -> str: return dump[:_CRASH_DUMP_CHARS] +#: A child killed by a signal (negative exit code) is run again this many times. +#: pandas/numpy native code has segfaulted intermittently on CI in otherwise +#: passing generated code; one retry separates that flake from a real crash, +#: and every crash is still recorded in ``GeneratedCodeResult.native_crashes``. +_NATIVE_CRASH_RETRIES = 1 + + @dataclass(frozen=True) class GeneratedCodeResult: """Outcome of one full verification run.""" @@ -101,6 +108,9 @@ class GeneratedCodeResult: stderr: str = "" produced_files: tuple[str, ...] = () stages: tuple[str, ...] = field(default=()) + #: One redacted excerpt per signal exit, including crashes that a retry + #: recovered from, so a flaky pass is never silent. + native_crashes: tuple[str, ...] = () def _allowlist_failures(tree: ast.AST) -> list[str]: @@ -227,32 +237,45 @@ def leaks(label: str, payload: Any) -> None: "LANG": "C.UTF-8", } interpreter = python or sys.executable + + def redacted(text: str) -> str: + return str(scanner.redact(text)) if scanner is not None else text + + native_crashes: list[str] = [] try: - proc = subprocess.run( # noqa: PLW1510 - exit code inspected below - # -X faulthandler: a native crash (negative exit code) dumps the - # Python traceback to stderr, which is redacted and reported. - [interpreter, "-I", "-X", "faulthandler", str(harness_path)], - cwd=workdir, - env=env, - capture_output=True, - text=True, - timeout=timeout, - ) + for attempt in range(1 + _NATIVE_CRASH_RETRIES): + if attempt: + # A crashed attempt may have left the input half-written; + # give the retry the same input contract as the first run. + frame.to_csv(input_path, index=False) + proc = subprocess.run( # noqa: PLW1510 - exit code inspected below + # -X faulthandler: a native crash (negative exit code) dumps + # the Python traceback to stderr, which is redacted and reported. + [interpreter, "-I", "-X", "faulthandler", str(harness_path)], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + if proc.returncode >= 0: + break + native_crashes.append( + f"exited {proc.returncode}: {_stderr_excerpt(redacted(proc.stderr))}" + ) except subprocess.TimeoutExpired: return GeneratedCodeResult( False, (*failures, f"generated code exceeded the {timeout:.0f}s timeout"), stages=(*stages, "execute"), + native_crashes=tuple(native_crashes), ) stages.append("execute") stdout, stderr = proc.stdout, proc.stderr if proc.returncode != 0: - safe_stderr: Any = stderr - if scanner is not None: - safe_stderr = scanner.redact(stderr) failures.append( f"generated code exited {proc.returncode}: " - f"{_stderr_excerpt(str(safe_stderr))}" + f"{_stderr_excerpt(redacted(stderr))}" ) produced = tuple( @@ -297,6 +320,7 @@ def leaks(label: str, payload: Any) -> None: stderr=str(safe_err), produced_files=produced, stages=tuple(stages), + native_crashes=tuple(native_crashes), ) diff --git a/benchmarks/truthbench/runner.py b/benchmarks/truthbench/runner.py index 2f6dba81..0ad203d6 100644 --- a/benchmarks/truthbench/runner.py +++ b/benchmarks/truthbench/runner.py @@ -734,6 +734,13 @@ def run_release( f"{domain}/{surface_name}: {failure}" for failure in outcome.failures ) + # A native crash that a retry recovered from does not fail + # the gate, but it must still show up in the run log. + for crash in outcome.native_crashes: + print( + f"truthbench: {domain}/{surface_name}: generated code {crash}", + file=sys.stderr, + ) observations, failures, parity_ledger = _parity_observations(parity, backends) parity_failures.extend( diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py index 90cc277c..17c4b578 100644 --- a/tests/truthbench/test_generated_code.py +++ b/tests/truthbench/test_generated_code.py @@ -218,3 +218,50 @@ def failed_child(args, **kwargs): result = verify_generated_code(GOOD, _fixture()) [failure] = [f for f in result.failures if "exited" in f] assert failure.endswith("ValueError: bad column\n") + + +_SEGV_DUMP = ( + "Fatal Python error: Segmentation fault\n\n" + "Current thread 0x0000000000000001 (most recent call first):\n" + ' File "pandas/core/tools/numeric.py", line 235 in to_numeric\n' +) + + +def _scripted_children(monkeypatch, outcomes): + """Replace the sandbox child with *outcomes* (return code, stderr) in order.""" + calls = [] + + def child(args, **kwargs): + code, stderr = outcomes[min(len(calls), len(outcomes) - 1)] + calls.append(code) + return subprocess.CompletedProcess(args, code, "", stderr) + + monkeypatch.setattr(gc.subprocess, "run", child) + return calls + + +def test_native_crash_is_retried_once_and_recorded(monkeypatch): + calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP), (0, "")]) + result = verify_generated_code(GOOD, _fixture()) + assert calls == [-signal.SIGSEGV, 0] + assert result.passed, result.failures + [crash] = result.native_crashes + assert f"exited {-signal.SIGSEGV}" in crash + assert "line 235 in to_numeric" in crash + + +def test_repeated_native_crash_still_fails(monkeypatch): + calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP)]) + result = verify_generated_code(GOOD, _fixture()) + assert len(calls) == 1 + gc._NATIVE_CRASH_RETRIES + assert not result.passed + assert any(f"exited {-signal.SIGSEGV}" in f for f in result.failures) + assert len(result.native_crashes) == len(calls) + + +def test_ordinary_failure_is_not_retried(monkeypatch): + calls = _scripted_children(monkeypatch, [(1, "ValueError: bad column\n")]) + result = verify_generated_code(GOOD, _fixture()) + assert calls == [1] + assert not result.passed + assert result.native_crashes == ()