From c34ce3ea28451cb27f54925814807c58a9520c74 Mon Sep 17 00:00:00 2001
From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com>
Date: Tue, 15 Sep 2026 14:01:37 +0530
Subject: [PATCH] fix(truthbench): keep the faulthandler stack in sandbox crash
 failures

The generated-code sandbox reported a failed child with the last 800
characters of its stderr. faulthandler writes the crashing thread's stack
first and a long "Extension modules: ..." line last, so a native crash
was reported with only the module list and without the frame that
located it. The finance/copilot case on PR #360 failed this way.

Keep a faulthandler dump from its header, drop the module list line and
cap it at 1500 characters. Other failures still keep the traceback tail.
---
 benchmarks/truthbench/generated_code.py | 26 +++++++++++++++++-
 tests/truthbench/test_generated_code.py | 36 +++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py
index 4e2ac742..b2b866f7 100644
--- a/benchmarks/truthbench/generated_code.py
+++ b/benchmarks/truthbench/generated_code.py
@@ -67,6 +67,29 @@
 
 _INPUT_BASENAME = "your_data.csv"
 
+#: How much of a failed child's stderr a failure message carries.
+_STDERR_TAIL_CHARS = 800
+_CRASH_DUMP_CHARS = 1500
+
+
+def _stderr_excerpt(stderr: str) -> str:
+    """The useful part of a failed child's stderr for its failure message.
+
+    A normal traceback ends at the bottom, so keep the tail.  A faulthandler
+    dump is the opposite: the stack that locates a native crash comes first and
+    a long ``Extension modules: ...`` line comes last, so a tail keeps only the
+    module list.  For a dump, keep it from its header and drop that line.
+    """
+    start = stderr.find("Fatal Python error:")
+    if start == -1:
+        return stderr[-_STDERR_TAIL_CHARS:]
+    dump = "\n".join(
+        line
+        for line in stderr[start:].splitlines()
+        if not line.startswith("Extension modules:")
+    )
+    return dump[:_CRASH_DUMP_CHARS]
+
 
 @dataclass(frozen=True)
 class GeneratedCodeResult:
@@ -228,7 +251,8 @@ def leaks(label: str, payload: Any) -> None:
             if scanner is not None:
                 safe_stderr = scanner.redact(stderr)
             failures.append(
-                f"generated code exited {proc.returncode}: {str(safe_stderr)[-800:]}"
+                f"generated code exited {proc.returncode}: "
+                f"{_stderr_excerpt(str(safe_stderr))}"
             )
 
         produced = tuple(
diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py
index 88ae4e5c..90cc277c 100644
--- a/tests/truthbench/test_generated_code.py
+++ b/tests/truthbench/test_generated_code.py
@@ -182,3 +182,39 @@ def crashed_child(args, **kwargs):
     [failure] = [f for f in result.failures if "exited" in f]
     assert f"exited {-signal.SIGSEGV}" in failure
     assert "Segmentation fault" in failure
+
+
+def test_native_crash_failure_keeps_stack_not_module_list(monkeypatch):
+    # faulthandler writes the stack first and a long "Extension modules" line
+    # last. Keeping only the tail of stderr reported just the module list and
+    # lost the frame that located the crash (seen in CI on a copilot case).
+    modules = ", ".join(f"pandas._libs.module_{i}" for i in range(53))
+    dump = (
+        "Fatal Python error: Segmentation fault\n\n"
+        "Current thread 0x0000000000000001 (most recent call first):\n"
+        '  File "pandas/core/tools/numeric.py", line 235 in to_numeric\n'
+        '  File "generated_pipeline.py", line 20 in <module>\n\n'
+        f"Extension modules: {modules} (total: 53)\n"
+    )
+    assert len(dump) > 800
+
+    def crashed_child(args, **kwargs):
+        return subprocess.CompletedProcess(args, -signal.SIGSEGV, "", dump)
+
+    monkeypatch.setattr(gc.subprocess, "run", crashed_child)
+    result = verify_generated_code(GOOD, _fixture())
+    [failure] = [f for f in result.failures if "exited" in f]
+    assert "line 235 in to_numeric" in failure
+    assert "Extension modules" not in failure
+
+
+def test_ordinary_failure_keeps_traceback_tail(monkeypatch):
+    stderr = "noise\n" * 300 + "ValueError: bad column\n"
+
+    def failed_child(args, **kwargs):
+        return subprocess.CompletedProcess(args, 1, "", stderr)
+
+    monkeypatch.setattr(gc.subprocess, "run", failed_child)
+    result = verify_generated_code(GOOD, _fixture())
+    [failure] = [f for f in result.failures if "exited" in f]
+    assert failure.endswith("ValueError: bad column\n")