fix(truthbench): keep the faulthandler stack in sandbox crash failures

benchmarks/truthbench/generated_code.py

@@ -67,6 +67,29 @@
 
 _INPUT_BASENAME = "your_data.csv"
 
+#: How much of a failed child's stderr a failure message carries.
+_STDERR_TAIL_CHARS = 800
+_CRASH_DUMP_CHARS = 1500
+
+
+def _stderr_excerpt(stderr: str) -> str:
+    """The useful part of a failed child's stderr for its failure message.
+
+    A normal traceback ends at the bottom, so keep the tail.  A faulthandler
+    dump is the opposite: the stack that locates a native crash comes first and
+    a long ``Extension modules: ...`` line comes last, so a tail keeps only the
+    module list.  For a dump, keep it from its header and drop that line.
+    """
+    start = stderr.find("Fatal Python error:")
+    if start == -1:
+        return stderr[-_STDERR_TAIL_CHARS:]
+    dump = "\n".join(
+        line
+        for line in stderr[start:].splitlines()
+        if not line.startswith("Extension modules:")
+    )
+    return dump[:_CRASH_DUMP_CHARS]
+
 
 @dataclass(frozen=True)
 class GeneratedCodeResult:
@@ -228,7 +251,8 @@ def leaks(label: str, payload: Any) -> None:
             if scanner is not None:
                 safe_stderr = scanner.redact(stderr)
             failures.append(
-                f"generated code exited {proc.returncode}: {str(safe_stderr)[-800:]}"
+                f"generated code exited {proc.returncode}: "
+                f"{_stderr_excerpt(str(safe_stderr))}"
             )
 
         produced = tuple(

tests/truthbench/test_generated_code.py

@@ -182,3 +182,39 @@ def crashed_child(args, **kwargs):
     [failure] = [f for f in result.failures if "exited" in f]
     assert f"exited {-signal.SIGSEGV}" in failure
     assert "Segmentation fault" in failure
+
+
+def test_native_crash_failure_keeps_stack_not_module_list(monkeypatch):
+    # faulthandler writes the stack first and a long "Extension modules" line
+    # last. Keeping only the tail of stderr reported just the module list and
+    # lost the frame that located the crash (seen in CI on a copilot case).
+    modules = ", ".join(f"pandas._libs.module_{i}" for i in range(53))
+    dump = (
+        "Fatal Python error: Segmentation fault\n\n"
+        "Current thread 0x0000000000000001 (most recent call first):\n"
+        '  File "pandas/core/tools/numeric.py", line 235 in to_numeric\n'
+        '  File "generated_pipeline.py", line 20 in <module>\n\n'
+        f"Extension modules: {modules} (total: 53)\n"
+    )
+    assert len(dump) > 800
+
+    def crashed_child(args, **kwargs):
+        return subprocess.CompletedProcess(args, -signal.SIGSEGV, "", dump)
+
+    monkeypatch.setattr(gc.subprocess, "run", crashed_child)
+    result = verify_generated_code(GOOD, _fixture())
+    [failure] = [f for f in result.failures if "exited" in f]
+    assert "line 235 in to_numeric" in failure
+    assert "Extension modules" not in failure
+
+
+def test_ordinary_failure_keeps_traceback_tail(monkeypatch):
+    stderr = "noise\n" * 300 + "ValueError: bad column\n"
+
+    def failed_child(args, **kwargs):
+        return subprocess.CompletedProcess(args, 1, "", stderr)
+
+    monkeypatch.setattr(gc.subprocess, "run", failed_child)
+    result = verify_generated_code(GOOD, _fixture())
+    [failure] = [f for f in result.failures if "exited" in f]
+    assert failure.endswith("ValueError: bad column\n")