diff --git a/benchmarks/truthbench/generated_code.py b/benchmarks/truthbench/generated_code.py index 4e2ac742..b2b866f7 100644 --- a/benchmarks/truthbench/generated_code.py +++ b/benchmarks/truthbench/generated_code.py @@ -67,6 +67,29 @@ _INPUT_BASENAME = "your_data.csv" +#: How much of a failed child's stderr a failure message carries. +_STDERR_TAIL_CHARS = 800 +_CRASH_DUMP_CHARS = 1500 + + +def _stderr_excerpt(stderr: str) -> str: + """The useful part of a failed child's stderr for its failure message. + + A normal traceback ends at the bottom, so keep the tail. A faulthandler + dump is the opposite: the stack that locates a native crash comes first and + a long ``Extension modules: ...`` line comes last, so a tail keeps only the + module list. For a dump, keep it from its header and drop that line. + """ + start = stderr.find("Fatal Python error:") + if start == -1: + return stderr[-_STDERR_TAIL_CHARS:] + dump = "\n".join( + line + for line in stderr[start:].splitlines() + if not line.startswith("Extension modules:") + ) + return dump[:_CRASH_DUMP_CHARS] + @dataclass(frozen=True) class GeneratedCodeResult: @@ -228,7 +251,8 @@ def leaks(label: str, payload: Any) -> None: if scanner is not None: safe_stderr = scanner.redact(stderr) failures.append( - f"generated code exited {proc.returncode}: {str(safe_stderr)[-800:]}" + f"generated code exited {proc.returncode}: " + f"{_stderr_excerpt(str(safe_stderr))}" ) produced = tuple( diff --git a/tests/truthbench/test_generated_code.py b/tests/truthbench/test_generated_code.py index 88ae4e5c..90cc277c 100644 --- a/tests/truthbench/test_generated_code.py +++ b/tests/truthbench/test_generated_code.py @@ -182,3 +182,39 @@ def crashed_child(args, **kwargs): [failure] = [f for f in result.failures if "exited" in f] assert f"exited {-signal.SIGSEGV}" in failure assert "Segmentation fault" in failure + + +def test_native_crash_failure_keeps_stack_not_module_list(monkeypatch): + # faulthandler writes the stack first and a long "Extension modules" line + # last. Keeping only the tail of stderr reported just the module list and + # lost the frame that located the crash (seen in CI on a copilot case). + modules = ", ".join(f"pandas._libs.module_{i}" for i in range(53)) + dump = ( + "Fatal Python error: Segmentation fault\n\n" + "Current thread 0x0000000000000001 (most recent call first):\n" + ' File "pandas/core/tools/numeric.py", line 235 in to_numeric\n' + ' File "generated_pipeline.py", line 20 in \n\n' + f"Extension modules: {modules} (total: 53)\n" + ) + assert len(dump) > 800 + + def crashed_child(args, **kwargs): + return subprocess.CompletedProcess(args, -signal.SIGSEGV, "", dump) + + monkeypatch.setattr(gc.subprocess, "run", crashed_child) + result = verify_generated_code(GOOD, _fixture()) + [failure] = [f for f in result.failures if "exited" in f] + assert "line 235 in to_numeric" in failure + assert "Extension modules" not in failure + + +def test_ordinary_failure_keeps_traceback_tail(monkeypatch): + stderr = "noise\n" * 300 + "ValueError: bad column\n" + + def failed_child(args, **kwargs): + return subprocess.CompletedProcess(args, 1, "", stderr) + + monkeypatch.setattr(gc.subprocess, "run", failed_child) + result = verify_generated_code(GOOD, _fixture()) + [failure] = [f for f in result.failures if "exited" in f] + assert failure.endswith("ValueError: bad column\n")