fix(truthbench): retry a generated-code run once after a native crash

benchmarks/truthbench/generated_code.py

@@ -91,6 +91,13 @@ def _stderr_excerpt(stderr: str) -> str:
     return dump[:_CRASH_DUMP_CHARS]
 
 
+#: A child killed by a signal (negative exit code) is run again this many times.
+#: pandas/numpy native code has segfaulted intermittently on CI in otherwise
+#: passing generated code; one retry separates that flake from a real crash,
+#: and every crash is still recorded in ``GeneratedCodeResult.native_crashes``.
+_NATIVE_CRASH_RETRIES = 1
+
+
 @dataclass(frozen=True)
 class GeneratedCodeResult:
     """Outcome of one full verification run."""
@@ -101,6 +108,9 @@ class GeneratedCodeResult:
     stderr: str = ""
     produced_files: tuple[str, ...] = ()
     stages: tuple[str, ...] = field(default=())
+    #: One redacted excerpt per signal exit, including crashes that a retry
+    #: recovered from, so a flaky pass is never silent.
+    native_crashes: tuple[str, ...] = ()
 
 
 def _allowlist_failures(tree: ast.AST) -> list[str]:
@@ -227,32 +237,45 @@ def leaks(label: str, payload: Any) -> None:
             "LANG": "C.UTF-8",
         }
         interpreter = python or sys.executable
+
+        def redacted(text: str) -> str:
+            return str(scanner.redact(text)) if scanner is not None else text
+
+        native_crashes: list[str] = []
         try:
-            proc = subprocess.run(  # noqa: PLW1510 - exit code inspected below
-                # -X faulthandler: a native crash (negative exit code) dumps the
-                # Python traceback to stderr, which is redacted and reported.
-                [interpreter, "-I", "-X", "faulthandler", str(harness_path)],
-                cwd=workdir,
-                env=env,
-                capture_output=True,
-                text=True,
-                timeout=timeout,
-            )
+            for attempt in range(1 + _NATIVE_CRASH_RETRIES):
+                if attempt:
+                    # A crashed attempt may have left the input half-written;
+                    # give the retry the same input contract as the first run.
+                    frame.to_csv(input_path, index=False)
+                proc = subprocess.run(  # noqa: PLW1510 - exit code inspected below
+                    # -X faulthandler: a native crash (negative exit code) dumps
+                    # the Python traceback to stderr, which is redacted and reported.
+                    [interpreter, "-I", "-X", "faulthandler", str(harness_path)],
+                    cwd=workdir,
+                    env=env,
+                    capture_output=True,
+                    text=True,
+                    timeout=timeout,
+                )
+                if proc.returncode >= 0:
+                    break
+                native_crashes.append(
+                    f"exited {proc.returncode}: {_stderr_excerpt(redacted(proc.stderr))}"
+                )
         except subprocess.TimeoutExpired:
             return GeneratedCodeResult(
                 False,
                 (*failures, f"generated code exceeded the {timeout:.0f}s timeout"),
                 stages=(*stages, "execute"),
+                native_crashes=tuple(native_crashes),
             )
         stages.append("execute")
         stdout, stderr = proc.stdout, proc.stderr
         if proc.returncode != 0:
-            safe_stderr: Any = stderr
-            if scanner is not None:
-                safe_stderr = scanner.redact(stderr)
             failures.append(
                 f"generated code exited {proc.returncode}: "
-                f"{_stderr_excerpt(str(safe_stderr))}"
+                f"{_stderr_excerpt(redacted(stderr))}"
             )
 
         produced = tuple(
@@ -297,6 +320,7 @@ def leaks(label: str, payload: Any) -> None:
             stderr=str(safe_err),
             produced_files=produced,
             stages=tuple(stages),
+            native_crashes=tuple(native_crashes),
         )
 
 

benchmarks/truthbench/runner.py

@@ -734,6 +734,13 @@ def run_release(
                         f"{domain}/{surface_name}: {failure}"
                         for failure in outcome.failures
                     )
+                    # A native crash that a retry recovered from does not fail
+                    # the gate, but it must still show up in the run log.
+                    for crash in outcome.native_crashes:
+                        print(
+                            f"truthbench: {domain}/{surface_name}: generated code {crash}",
+                            file=sys.stderr,
+                        )
 
         observations, failures, parity_ledger = _parity_observations(parity, backends)
         parity_failures.extend(

tests/truthbench/test_generated_code.py

@@ -218,3 +218,50 @@ def failed_child(args, **kwargs):
     result = verify_generated_code(GOOD, _fixture())
     [failure] = [f for f in result.failures if "exited" in f]
     assert failure.endswith("ValueError: bad column\n")
+
+
+_SEGV_DUMP = (
+    "Fatal Python error: Segmentation fault\n\n"
+    "Current thread 0x0000000000000001 (most recent call first):\n"
+    '  File "pandas/core/tools/numeric.py", line 235 in to_numeric\n'
+)
+
+
+def _scripted_children(monkeypatch, outcomes):
+    """Replace the sandbox child with *outcomes* (return code, stderr) in order."""
+    calls = []
+
+    def child(args, **kwargs):
+        code, stderr = outcomes[min(len(calls), len(outcomes) - 1)]
+        calls.append(code)
+        return subprocess.CompletedProcess(args, code, "", stderr)
+
+    monkeypatch.setattr(gc.subprocess, "run", child)
+    return calls
+
+
+def test_native_crash_is_retried_once_and_recorded(monkeypatch):
+    calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP), (0, "")])
+    result = verify_generated_code(GOOD, _fixture())
+    assert calls == [-signal.SIGSEGV, 0]
+    assert result.passed, result.failures
+    [crash] = result.native_crashes
+    assert f"exited {-signal.SIGSEGV}" in crash
+    assert "line 235 in to_numeric" in crash
+
+
+def test_repeated_native_crash_still_fails(monkeypatch):
+    calls = _scripted_children(monkeypatch, [(-signal.SIGSEGV, _SEGV_DUMP)])
+    result = verify_generated_code(GOOD, _fixture())
+    assert len(calls) == 1 + gc._NATIVE_CRASH_RETRIES
+    assert not result.passed
+    assert any(f"exited {-signal.SIGSEGV}" in f for f in result.failures)
+    assert len(result.native_crashes) == len(calls)
+
+
+def test_ordinary_failure_is_not_retried(monkeypatch):
+    calls = _scripted_children(monkeypatch, [(1, "ValueError: bad column\n")])
+    result = verify_generated_code(GOOD, _fixture())
+    assert calls == [1]
+    assert not result.passed
+    assert result.native_crashes == ()