diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py index b2b866f7..317a3dbd 100644 --- a/benchmarks/truthbench/generated_code.py +++ b/benchmarks/truthbench/generated_code.py @@ -91,6 +91,13 @@ def _stderr_excerpt(stderr: str) -> str: return dump[:_CRASH_DUMP_CHARS] +#: A child killed by a signal (negative exit code) is run again this many times. +#: pandas/numpy native code has segfaulted intermittently on CI in otherwise +#: passing generated code; one retry separates that flake from a real crash, +#: and every crash is still recorded in ``GeneratedCodeResult.native_crashes``. +_NATIVE_CRASH_RETRIES = 1 + + @dataclass(frozen=True) class GeneratedCodeResult: """Outcome of one full verification run.""" @@ -101,6 +108,9 @@ class GeneratedCodeResult: stderr: str = "" produced_files: tuple[str, ...] = () stages: tuple[str, ...] = field(default=()) + #: One redacted excerpt per signal exit, including crashes that a retry + #: recovered from, so a flaky pass is never silent. + native_crashes: tuple[str, ...] = () def _allowlist_failures(tree: ast.AST) -> list[str]: @@ -227,32 +237,45 @@ def leaks(label: str, payload: Any) -> None: "LANG": "C.UTF-8", } interpreter = python or sys.executable + + def redacted(text: str) -> str: + return str(scanner.redact(text)) if scanner is not None else text + + native_crashes: list[str] = [] try: - proc = subprocess.run( # noqa: PLW1510 - exit code inspected below - # -X faulthandler: a native crash (negative exit code) dumps the - # Python traceback to stderr, which is redacted and reported. - [interpreter, "-I", "-X", "faulthandler", str(harness_path)], - cwd=workdir, - env=env, - capture_output=True, - text=True, - timeout=timeout, - ) + for attempt in range(1 + _NATIVE_CRASH_RETRIES): + if attempt: + # A crashed attempt may have left the input half-written; + # give the retry the same input contract as the first run. + frame.to_csv(input_path, index=False) + proc = subprocess.run( # noqa: PLW1510 - exit code inspected below + # -X faulthandler: a native crash (negative exit code) dumps + # the Python traceback to stderr, which is redacted and reported. + [interpreter, "-I", "-X", "faulthandler", str(harness_path)], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + if proc.returncode >= 0: + break + native_crashes.append( + f"exited {proc.returncode}: {_stderr_excerpt(redacted(proc.stderr))}" + ) except subprocess.TimeoutExpired: return GeneratedCodeResult( False, (*failures, f"generated code exceeded the {timeout:.0f}s timeout"), stages=(*stages, "execute"), + native_crashes=tuple(native_crashes), ) stages.append("execute") stdout, stderr = proc.stdout, proc.stderr if proc.returncode != 0: - safe_stderr: Any = stderr - if scanner is not None: - safe_stderr = scanner.redact(stderr) failures.append( f"generated code exited {proc.returncode}: " - f"{_stderr_excerpt(str(safe_stderr))}" + f"{_stderr_excerpt(redacted(stderr))}" ) produced = tuple( @@ -297,6 +320,7 @@ def leaks(label: str, payload: Any) -> None: stderr=str(safe_err), produced_files=produced, stages=tuple(stages), + native_crashes=tuple(native_crashes), ) diff --git a/benchmarks/truthbench/runner.py b/benchmarks/truthbench/runner.py index 2f6dba81..0ad203d6 100644 --- a/benchmarks/truthbench/runner.py +++ b/benchmarks/truthbench/runner.py @@ -734,6 +734,13 @@ def run_release( f"{domain}/{surface_name}: {failure}" for failure in outcome.failures ) + # A native crash that a retry recovered from does not fail + # the gate, but it must still show up in the run log. + for crash in outcome.native_crashes: + print( + f"truthbench: {domain}/{surface_name}: generated code {crash}", + file=sys.stderr, + ) observations, failures, parity_ledger = _parity_observations(parity, backends) parity_failures.extend( diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py index 90cc277c..17c4b578 100644 --- a/tests/truthbench/test_generated_code.py +++ b/tests/truthbench/test_generated_code.py @@ -218,3 +218,50 @@ def failed_child(args, **kwargs): result = verify_generated_code(GOOD, _fixture()) [failure] = [f for f in result.failures if "exited" in f] assert failure.endswith("ValueError: bad column\n") + + +_SEGV_DUMP = ( + "Fatal Python error: Segmentation fault\n\n" + "Current thread 0x0000000000000001 (most recent call first):\n" + ' File "pandas/core/tools/numeric.py", line 235 in to_numeric\n' +) + + +def _scripted_children(monkeypatch, outcomes): + """Replace the sandbox child with *outcomes* (return code, stderr) in order.""" + calls = [] + + def child(args, **kwargs): + code, stderr = outcomes[min(len(calls), len(outcomes) - 1)] + calls.append(code) + return subprocess.CompletedProcess(args, code, "", stderr) + + monkeypatch.setattr(gc.subprocess, "run", child) + return calls + + +def test_native_crash_is_retried_once_and_recorded(monkeypatch): + calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP), (0, "")]) + result = verify_generated_code(GOOD, _fixture()) + assert calls == [-signal.SIGSEGV, 0] + assert result.passed, result.failures + [crash] = result.native_crashes + assert f"exited {-signal.SIGSEGV}" in crash + assert "line 235 in to_numeric" in crash + + +def test_repeated_native_crash_still_fails(monkeypatch): + calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP)]) + result = verify_generated_code(GOOD, _fixture()) + assert len(calls) == 1 + gc._NATIVE_CRASH_RETRIES + assert not result.passed + assert any(f"exited {-signal.SIGSEGV}" in f for f in result.failures) + assert len(result.native_crashes) == len(calls) + + +def test_ordinary_failure_is_not_retried(monkeypatch): + calls = _scripted_children(monkeypatch, [(1, "ValueError: bad column\n")]) + result = verify_generated_code(GOOD, _fixture()) + assert calls == [1] + assert not result.passed + assert result.native_crashes == ()