diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f1b8eb9f3a2..2e24fc982f8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,6 +8,17 @@ on: schedule: - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC workflow_dispatch: {} + # TEMPORARY (do not merge): runs the Windows coverage job against a PR, via + # the copy-pr-bot mirror branch. Same trigger as ci.yml. + push: + branches: + - "pull-request/[0-9]+" + +# One coverage run per ref: the Windows leg holds an A100 runner for the whole +# job, so superseded runs must not queue up behind each other. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true env: PY_VER: "3.14" @@ -32,6 +43,8 @@ jobs: coverage-linux: name: Coverage (Linux) needs: [coverage-vars] + # Only Windows crashes, so a PR run skips the Linux leg entirely. + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} runs-on: "linux-amd64-gpu-a100-latest-1" permissions: id-token: write @@ -214,6 +227,22 @@ jobs: - name: Set up MSVC uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 + # Printed for contrast with the same step in the test job: wheels are + # built here and run there, and it is the copy on the *test* machine that + # decides whether std::mutex works. A plain `pip wheel` performs no + # delvewheel repair, so nothing from this machine travels with the wheel. + - name: Report the C++ runtime on the build machine + continue-on-error: true + shell: pwsh + run: | + foreach ($name in "msvcp140.dll", "vcruntime140.dll") { + $path = Join-Path $env:WINDIR "System32\$name" + if (Test-Path $path) { + "{0,-20} {1}" -f $name, (Get-Item $path).VersionInfo.FileVersion + } + } + "cl.exe {0}" -f (Get-Command cl.exe -ErrorAction SilentlyContinue).Source + - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false @@ -235,13 +264,88 @@ jobs: cd cuda_bindings ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ + # TEMPORARY (do not merge): build against the cuda-bindings wheel built a + # step ago, not whatever PyPI offers. + # + # PIP_PRE is here so pip will consider the locally built wheel at all -- + # it carries a .devN version, which is a pre-release. The side effect is + # that PyPI's pre-releases become eligible too, and cuda-bindings + # 13.4.0b1, published 2026-07-29, outranks the local build for + # `cuda-bindings==13.*`. Its cydriver.pxd is generated from CTK 13.4 + # headers, so CUmemLocation gains a `localized` field, while cuda.core + # compiles against the 13.3.0 mini-CTK that has no such field. Cython + # generates a struct converter over every field, and the build dies: + # + # _managed_memory_ops.cpp(18447): error C2039: 'localized' is not a + # member of 'CUmemLocation_st' + # + # Windows coverage has produced nothing since; the job fails here and + # Coverage (Windows) is skipped. cibuildwheel escapes it in + # build-wheel.yml only because it sets PIP_FIND_LINKS without PIP_PRE. + # + # Pinning the exact local version keeps what PIP_PRE was for and drops + # what it let in. The real fix belongs upstream; this only exists so the + # crash diagnostics on this branch can run before that lands. - name: Build cuda.core wheel run: | export PIP_FIND_LINKS="$(pwd)/wheels" export PIP_PRE=1 + bindings_whl="$(ls ./wheels/cuda_bindings-*.whl | head -1)" + bindings_ver="$(basename "$bindings_whl" | cut -d- -f2)" + # Drop the local segment: PEP 440 says a public `==` specifier ignores + # a candidate's local label, so `==13.3.2.dev139` still selects + # 13.3.2.dev139+g0123456 while avoiding local versions in a + # constraints file, which pip has been unhappy with before. + bindings_ver="${bindings_ver%%+*}" + echo "cuda-bindings==${bindings_ver}" > "$GITHUB_WORKSPACE/constraints.txt" + echo "=== pip constraints ===" + cat "$GITHUB_WORKSPACE/constraints.txt" + export PIP_CONSTRAINT="$GITHUB_WORKSPACE/constraints.txt" cd cuda_core ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ + # The step every other Windows build does and this one never did. + # + # cibuildwheel runs `repair-wheel-command` out of each package's + # pyproject.toml after building, which on Windows is delvewheel: it + # copies the dependent DLLs into .libs under content-hashed + # names, rewrites the .pyd import tables to reference those names, and + # adds an os.add_dll_directory call to the package __init__. A plain + # `pip wheel` knows nothing about [tool.cibuildwheel], so this build has + # always shipped .pyd files that import a bare "MSVCP140.dll" and + # resolve it to whatever the test machine happens to have in System32 -- + # which on that runner is 14.00.24215.1, from 2015. + # + # _resource_handles.pyd, compiled by MSVC 14.44, takes only _Mtx_lock + # and _Mtx_unlock from that DLL and never _Mtx_init_in_situ, because + # std::mutex has had a constexpr constructor since VS 2022 17.10. The + # 2015 runtime still expects that initialisation to have happened and + # dereferences a null handle on the first lock, which _stream.pyx:422 + # takes during import. + # + # Vendoring 14.51 from this build machine removes the dependency on what + # the test machine has. --namespace-pkg cuda is required: `cuda` is a + # namespace package, and without it the add_dll_directory patch lands in + # the wrong __init__. + - name: Vendor the C++ runtime into the wheels + run: | + .venv/Scripts/pip install delvewheel + mkdir -p wheels-repaired + for whl in ./wheels/cuda_bindings-*.whl ./wheels/cuda_core-*.whl; do + echo "=== repairing $whl ===" + .venv/Scripts/delvewheel repair --namespace-pkg cuda \ + --exclude "torch_cpu.dll;torch_python.dll" \ + -w ./wheels-repaired "$whl" + done + # Replace the originals in place: everything downstream, including + # the artifact the test job installs from, reads ./wheels. + mv -f ./wheels-repaired/*.whl ./wheels/ + echo "=== what got vendored ===" + for whl in ./wheels/cuda_bindings-*.whl ./wheels/cuda_core-*.whl; do + echo "--- $whl" + unzip -l "$whl" | grep -i "\.libs/" || echo " (nothing vendored)" + done + - name: List wheel artifacts run: | echo "=== Windows wheel artifacts ===" @@ -268,6 +372,15 @@ jobs: ARCH: "amd64" CUDA_PYTHON_COVERAGE: "1" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} + # Diagnostics for the recurring exit-139 crash in the cuda.core step. + # faulthandler names the Windows exception (access violation vs stack + # overflow) and dumps every thread's Python stack; unlike pytest's + # built-in faulthandler plugin, the env var is inherited by tests that + # spawn their own `python -c` children, which crash too. Unbuffered + # stdio keeps the last lines before a hard crash from dying in the + # block buffer of a non-tty pipe. + PYTHONFAULTHANDLER: "1" + PYTHONUNBUFFERED: "1" defaults: run: shell: bash --noprofile --norc -xeuo pipefail {0} @@ -295,6 +408,28 @@ jobs: run: | nvidia-smi + # The crash is a null read inside msvcp140's mutex lock, and the copy + # this machine loads was built before VS 2022 17.10 made std::mutex's + # constructor constexpr -- code compiled after that change no longer + # calls _Mtx_init, which the older runtime still expects. The version is + # currently only inferred from a TimeDateStamp in a minidump, so read it + # from the file itself; the number is what a report to the runner owners + # needs. The build machine's copy is printed for contrast: wheels are + # built there, tested here. + - name: Report the C++ runtime this machine loads + continue-on-error: true + shell: pwsh + run: | + foreach ($name in "msvcp140.dll", "vcruntime140.dll", "vcruntime140_1.dll") { + $path = Join-Path $env:WINDIR "System32\$name" + if (Test-Path $path) { + $info = (Get-Item $path).VersionInfo + "{0,-20} {1,-16} {2}" -f $name, $info.FileVersion, $path + } else { + "{0,-20} " -f $name + } + } + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -365,14 +500,217 @@ jobs: --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ "$GITHUB_WORKSPACE/cuda_bindings/tests" + # The whole crash reduces to one `import cuda.core` under coverage, with + # no pytest, no conftest and no tests: the linetrace wheel and the + # ctracer are the two conditions `test-wheel-windows.yml` lacks on this + # same runner. Importing submodules one at a time was tried first and + # measured nothing -- every one of them executes cuda/core/__init__.py + # on the way in, so all six probes walked the same path and all six + # failed identically. + # + # What is missing is the native half. The fault happens inside an + # extension module's initialiser, and faulthandler's report ends at + # ``. The usual ways to see past + # that are all closed here: cdb is not installed (the SDK ships without + # the Debugging Tools feature), procdump would need an outbound + # download, a postmortem debugger under AeDebug never fires because this + # image does not record user-process crashes with WER, and there are no + # PDBs because build_hooks.py refuses debuggable builds on Windows. + # + # dbgcore.dll in System32 is always present though, so the process + # reports on itself. ci/tools/native_crash_handler.py catches the fault + # first-chance and writes the faulting address resolved to module + RVA, + # the native return chain resolved the same way, the module map, and a + # minidump, then lets the process die exactly as it would have. Which + # binary faults is the open question, and module + RVA answers it + # without symbols. + # + # Five runs, each answering something the others cannot: + # + # selftest faults on purpose, so a clean report proves the handler + # works here and an empty one is not mistaken for "no + # crash" + # plain the import with no coverage and no trace function at + # all. The fault is a null read inside msvcp140's mutex + # lock, taken unconditionally by _stream.pyx:422, so this + # should crash as well -- and if it does, instrumentation + # was never a condition + # coverage the nightly's own conditions, on the 8 MB stack the + # pytest wrapper uses + # breadcrumb the same import with a bare trace function instead of + # coverage. It arms Cython's linetrace the same way, so + # a crash here means any tracer will do rather than + # coverage specifically -- and the last line of its log + # names the .pyx statement that was executing + # fingerprint the environment around the import: driver, CPU, and + # the loaded-module map, which is what a comparison + # against a machine that does not crash needs + # + # COVERAGE_FILE keeps all of this out of the real .coverage, which + # `coverage run` would otherwise truncate. + - name: Record the native side of the fault + if: always() + continue-on-error: true + env: + COVERAGE_FILE: ${{ github.workspace }}/.coverage.probe + run: | + probe="$GITHUB_WORKSPACE/ci/tools/crash_probe.py" + python="$GITHUB_WORKSPACE/.venv/Scripts/python" + cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" + + rc=0 + "$python" "$probe" --selftest \ + --native-log "$GITHUB_WORKSPACE/native-selftest.txt" \ + --dump "$GITHUB_WORKSPACE/selftest.dmp" || rc=$? + echo "SELFTEST rc=$rc" + head -n 10 "$GITHUB_WORKSPACE/native-selftest.txt" || true + + # No coverage, no trace function, nothing but the import. The fault + # was named as a null read inside msvcp140's mutex lock, reached from + # a std::lock_guard that _stream.pyx:422 takes unconditionally, so + # instrumentation should have nothing to do with it. If this crashes + # too, "linetrace plus a tracer" was never the condition -- the cov + # wheels simply are not built through delvewheel, so they load the + # system msvcp140.dll instead of a vendored one, and that copy on + # this image predates the constexpr std::mutex change. + rc=0 + "$python" "$probe" --stack-mb 8 \ + --native-log "$GITHUB_WORKSPACE/native-plain.txt" \ + --dump "$GITHUB_WORKSPACE/crash-plain.dmp" || rc=$? + echo "PLAIN PROBE (no coverage, no tracer) rc=$rc" + cat "$GITHUB_WORKSPACE/native-plain.txt" || true + + rc=0 + "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \ + "$probe" --stack-mb 8 \ + --native-log "$GITHUB_WORKSPACE/native-coverage.txt" \ + --dump "$GITHUB_WORKSPACE/crash-coverage.dmp" || rc=$? + echo "COVERAGE PROBE rc=$rc" + cat "$GITHUB_WORKSPACE/native-coverage.txt" || true + + rc=0 + "$python" "$probe" --stack-mb 8 \ + --breadcrumb "$GITHUB_WORKSPACE/breadcrumb.txt" \ + --native-log "$GITHUB_WORKSPACE/native-breadcrumb.txt" \ + --dump "$GITHUB_WORKSPACE/crash-breadcrumb.dmp" || rc=$? + echo "BREADCRUMB PROBE rc=$rc" + echo "=== breadcrumb: $(wc -l < "$GITHUB_WORKSPACE/breadcrumb.txt") lines, last 25 ===" + tail -n 25 "$GITHUB_WORKSPACE/breadcrumb.txt" || true + + rc=0 + "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \ + "$GITHUB_WORKSPACE/ci/tools/env_fingerprint.py" --stack-mb 8 \ + "$GITHUB_WORKSPACE/fingerprint-windows.txt" || rc=$? + echo "FINGERPRINT rc=$rc" + + # Only this step crashes. cuda.bindings runs the same wrapper, the same + # 8 MB thread and the same coverage wheel, and finishes normally, so the + # extra diagnostics are scoped to cuda.core alone. + # + # The crash log narrows the fault to "somewhere before pytest collects". + # PYTHONVERBOSE narrows it to a module: the interpreter announces every + # import as it begins one, so the last line written before the process + # dies names the module it died in. That output is stderr, which is why + # the session is redirected to a file rather than left in the step log -- + # a crash takes the tail of a pipe with it, while each unbuffered write + # to a file has already reached the OS. - name: Run cuda.core tests (with 8MB stack) continue-on-error: true + env: + PYTEST_CRASHLOG: ${{ github.workspace }}/crashlog-core.txt + PYTEST_FAULTLOG: ${{ github.workspace }}/faulthandler-core.txt + PYTHONVERBOSE: "1" run: | "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ + --isolate \ --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ -v --cov=./cuda --cov-append --cov-context=test \ --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ - "$GITHUB_WORKSPACE/cuda_core/tests" + "$GITHUB_WORKSPACE/cuda_core/tests" \ + > "$GITHUB_WORKSPACE/verbose-core.txt" 2>&1 + + # The step above has been exiting 139 on every run, and Git Bash reports + # that as a bare "Segmentation fault". The crash log's final line says + # how far the session actually got. + - name: Report how far the cuda.core session got + if: always() + continue-on-error: true + run: | + log="$GITHUB_WORKSPACE/crashlog-core.txt" + if [ -f "$log" ]; then + echo "=== crashlog: $(wc -l < "$log") lines, last 20 ===" + tail -n 20 "$log" + else + echo "No crash log: died before pytest started." + fi + + echo "" + fault="$GITHUB_WORKSPACE/faulthandler-core.txt" + if [ -s "$fault" ]; then + echo "=== faulthandler ===" + cat "$fault" + else + # Empty is itself a result: the fault never reached Python's + # handler, which points at native code running outside any Python + # frame -- an extension module's initialisation, for instance. + echo "=== faulthandler: no output ===" + fi + + echo "" + verbose="$GITHUB_WORKSPACE/verbose-core.txt" + if [ -f "$verbose" ]; then + echo "=== PYTHONVERBOSE: $(wc -l < "$verbose") lines, last 40 ===" + tail -n 40 "$verbose" + fi + + # Windows records an Application Error entry for every process it kills, + # naming the faulting module and the offset within it, and it does so + # whether or not crash dumps are configured. That is the one fact no + # Python-level log can produce, and reading the log changes no machine + # state, so nothing has to be restored afterwards. + - name: Read the crash record from the Windows event log + if: always() + continue-on-error: true + shell: powershell + run: | + $events = Get-WinEvent -FilterHashtable @{ + LogName = 'Application' + ProviderName = 'Application Error', 'Windows Error Reporting' + StartTime = (Get-Date).AddHours(-2) + } -ErrorAction SilentlyContinue + if (-not $events) { + Write-Output "No Application Error / WER records in the last 2 hours." + Write-Output "(If this image keeps no such log, a minidump is the fallback.)" + exit 0 + } + foreach ($e in $events | Select-Object -First 15) { + Write-Output "===== $($e.TimeCreated.ToString('u')) $($e.ProviderName) Id=$($e.Id) =====" + Write-Output $e.Message + Write-Output "" + } + + - name: Upload crash diagnostics + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: crash-diagnostics-windows + path: | + crashlog-core.txt + faulthandler-core.txt + verbose-core.txt + native-selftest.txt + native-plain.txt + native-coverage.txt + native-breadcrumb.txt + breadcrumb.txt + fingerprint-windows.txt + selftest.dmp + crash-plain.dmp + crash-coverage.dmp + crash-breadcrumb.dmp + retention-days: 7 + if-no-files-found: warn - name: Copy Windows coverage file to workspace run: | @@ -393,7 +731,10 @@ jobs: name: Combine Coverage and Deploy needs: [coverage-linux, coverage-windows] runs-on: ubuntu-latest - if: always() + # always() alone would still run this after coverage-linux is skipped, and + # it would both fail on the missing .coverage.linux and publish a half + # report to gh-pages. + if: ${{ always() && !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: id-token: write contents: write diff --git a/ci/tools/crash_probe.py b/ci/tools/crash_probe.py new file mode 100644 index 00000000000..0b907e1828b --- /dev/null +++ b/ci/tools/crash_probe.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Import cuda.core under full instrumentation and record where it dies. + +The crash reduces to a single import, on a thread the size of the one +run_pytest_with_stack.py creates. Two records are taken around it, and they +answer different halves of the same question: + + the breadcrumb the last .pyx line that executed, i.e. which source + statement was running -- written by our own trace + function, which also arms Cython's linetrace hooks the + way coverage does, so the fault still triggers + the native log which module faulted and at what offset, from + native_crash_handler + +A run under coverage answers "does it still crash", and a run with the +breadcrumb instead of coverage answers "is coverage necessary, or does any +tracer do". Only one trace function can be installed at a time, so those are +separate invocations rather than one. + + crash_probe.py --stack-mb 8 --native-log native.txt --dump crash.dmp + crash_probe.py --stack-mb 8 --breadcrumb lines.txt --native-log native.txt + crash_probe.py --selftest --native-log native.txt + +--selftest writes to address zero on purpose. It is how the whole chain gets +verified somewhere that does not crash on its own: if the handler, the module +map, the dump and the artifact upload all work there, then the one gated run +on the runner that does crash is not spent discovering a typo. +""" + +from __future__ import annotations + +import argparse +import ctypes +import importlib +import os +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import native_crash_handler + + +def install_breadcrumb(path: str, needle: str) -> None: + """Log every executed line whose file matches `needle`, line buffered. + + Cython emits __Pyx_TraceLine for module-level statements too, which is the + only reason this can see anything at all: the fault happens while a module + body is still executing, before any function in it has been called. + """ + log = open( # noqa: SIM115 - must outlive this function + os.path.abspath(path), "w", buffering=1, encoding="utf-8", errors="backslashreplace" + ) + + def tracer(frame, event, _arg): + if event == "line": + filename = frame.f_code.co_filename + if needle in filename: + log.write(f"{filename}:{frame.f_lineno}\n") + return tracer + + # Only one trace function exists per thread, so arming this one under + # `coverage run` would quietly switch coverage off and leave a run that + # looks like it measured something. Say so rather than let it pass. + displaced = sys.gettrace() + if displaced is not None: + message = f"# WARNING: replaced an existing trace function: {displaced!r}" + log.write(message + "\n") + print(message, flush=True) + + log.write(f"# breadcrumb armed, filter={needle!r}\n") + sys.settrace(tracer) + + +def selftest() -> None: + """Deliberately fault, to prove the handler reports what it should.""" + print("selftest: writing to address 0", flush=True) + ctypes.memset(0, 0, 1) + print("selftest: still alive -- the write did NOT fault", flush=True) + + +def body(args) -> None: + if args.breadcrumb: + install_breadcrumb(args.breadcrumb, args.breadcrumb_filter) + if args.selftest: + selftest() + return + started = time.perf_counter() + importlib.import_module(args.module) + print(f"PROBE OK {args.module} in {time.perf_counter() - started:.2f}s", flush=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--module", default="cuda.core") + parser.add_argument("--stack-mb", type=float, default=0.0) + parser.add_argument("--native-log", default="crash-native.txt") + parser.add_argument("--dump") + parser.add_argument("--breadcrumb") + parser.add_argument("--breadcrumb-filter", default="cuda") + parser.add_argument("--selftest", action="store_true") + args = parser.parse_args() + + # Process-wide, so arming it here covers the worker thread as well. + native_crash_handler.install(args.native_log, dump_path=args.dump) + + if not args.stack_mb: + body(args) + return 0 + + # sys.settrace only affects the thread that calls it, so the breadcrumb is + # installed inside body(), on the thread that does the import. + threading.stack_size(int(args.stack_mb * 1024 * 1024)) + worker = threading.Thread(target=body, args=(args,), name=f"stack-{args.stack_mb:g}mb") + worker.start() + worker.join() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tools/env_fingerprint.py b/ci/tools/env_fingerprint.py new file mode 100644 index 00000000000..1ed85f2b5fa --- /dev/null +++ b/ci/tools/env_fingerprint.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# Environment fingerprint taken around `import cuda.core`. +# +# docs/WINDOWS_COVERAGE_SEGFAULT.md section 19.1 reduced the nightly crash to a +# single import under coverage, and section 17.7 ranked what still differs +# between the NVIDIA runner and ours. Nothing in that ranking has ever been +# measured on our side beyond the A100 box's `nvidia-smi`, so this script +# records the whole comparison surface in one pass: +# +# * host: OS build, CPU, RAM, account, session id +# * GPU: driver version and TCC/MCDM mode +# * tracer: which coverage core is actually installed once tracing starts -- +# if it is not ctrace, `__Pyx_TraceLine` never calls back into Python and +# every "we cannot reproduce it" run so far was testing the wrong thing +# * modules: the loaded DLL map before and after the import, with base +# addresses, which is the direct fingerprint for a fault that happens in +# extension-module initialisation +# +# The module delta also answers a question the source could not: whether +# `import cuda.core` pulls in nvcuda.dll at all. If it does not, the driver +# version and TCC/MCDM mode drop out of the suspect list and the repro matrix +# opens up to any Windows box. +# +# Run it the same way the minimal repro runs: +# +# coverage run --rcfile=/.coveragerc env_fingerprint.py [out.txt] +# +# Exits 0 whether or not the import succeeds -- a crash is a native fault that +# kills the process outright, so any Python-level exception is reported and +# then swallowed, keeping the fingerprint itself readable. + +from __future__ import annotations + +import ctypes +import ctypes.wintypes as wintypes +import os +import platform +import struct +import subprocess +import sys +import threading +import time + +_out_lines: list[str] = [] + + +def emit(line: str = "") -> None: + print(line, flush=True) + _out_lines.append(line) + + +def section(title: str) -> None: + emit() + emit(f"===== {title} =====") + + +def safe(title: str, fn) -> None: + """Run one collector. A collector that fails must not lose the rest.""" + section(title) + try: + fn() + except Exception as exc: # diagnostics must never abort + emit(f"") + + +# -------------------------------------------------------------------------- +# Win32 helpers +# -------------------------------------------------------------------------- + +_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +_psapi = ctypes.WinDLL("psapi", use_last_error=True) +_advapi32 = ctypes.WinDLL("advapi32", use_last_error=True) + + +class MODULEINFO(ctypes.Structure): + _fields_ = [ + ("lpBaseOfDll", ctypes.c_void_p), + ("SizeOfImage", wintypes.DWORD), + ("EntryPoint", ctypes.c_void_p), + ] + + +class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ("dwLength", wintypes.DWORD), + ("dwMemoryLoad", wintypes.DWORD), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + +_kernel32.GetCurrentProcess.restype = wintypes.HANDLE +_psapi.EnumProcessModules.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(ctypes.c_void_p), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), +] +_psapi.GetModuleFileNameExW.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + wintypes.LPWSTR, + wintypes.DWORD, +] +_psapi.GetModuleInformation.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + ctypes.POINTER(MODULEINFO), + wintypes.DWORD, +] + + +def loaded_modules() -> dict[str, tuple[int, int]]: + """path -> (base address, image size) for every DLL mapped right now.""" + handle = _kernel32.GetCurrentProcess() + # Ask twice: the first call reports how much room the list actually needs. + needed = wintypes.DWORD() + arr = (ctypes.c_void_p * 1024)() + if not _psapi.EnumProcessModules(handle, arr, ctypes.sizeof(arr), ctypes.byref(needed)): + raise ctypes.WinError(ctypes.get_last_error()) + count = needed.value // ctypes.sizeof(ctypes.c_void_p) + if count > len(arr): + arr = (ctypes.c_void_p * count)() + if not _psapi.EnumProcessModules(handle, arr, ctypes.sizeof(arr), ctypes.byref(needed)): + raise ctypes.WinError(ctypes.get_last_error()) + count = needed.value // ctypes.sizeof(ctypes.c_void_p) + + out: dict[str, tuple[int, int]] = {} + name = ctypes.create_unicode_buffer(32768) + for i in range(count): + hmod = arr[i] + if not _psapi.GetModuleFileNameExW(handle, hmod, name, len(name)): + continue + info = MODULEINFO() + if not _psapi.GetModuleInformation(handle, hmod, ctypes.byref(info), ctypes.sizeof(info)): + continue + out[name.value] = (info.lpBaseOfDll or 0, info.SizeOfImage) + return out + + +def format_modules(mods: dict[str, tuple[int, int]]) -> list[str]: + rows = sorted(mods.items(), key=lambda kv: kv[1][0]) + return [f"0x{base:016x} {size:>9} {path}" for path, (base, size) in rows] + + +# -------------------------------------------------------------------------- +# Collectors +# -------------------------------------------------------------------------- + + +def collect_host() -> None: + emit(f"hostname = {platform.node()}") + emit(f"platform = {platform.platform()}") + emit(f"win32_ver = {platform.win32_ver()}") + emit(f"win32_edition = {platform.win32_edition()}") + emit(f"is_server_core = {platform.win32_is_iot()!r} (win32_is_iot; edition string above is the real signal)") + emit(f"machine = {platform.machine()}") + emit(f"processor = {platform.processor()}") + emit(f"cpu_count = {os.cpu_count()}") + + status = MEMORYSTATUSEX() + status.dwLength = ctypes.sizeof(MEMORYSTATUSEX) + if _kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + gb = 1024**3 + emit(f"ram_total_gb = {status.ullTotalPhys / gb:.1f}") + emit(f"ram_avail_gb = {status.ullAvailPhys / gb:.1f}") + + user = ctypes.create_unicode_buffer(256) + size = wintypes.DWORD(len(user)) + if _advapi32.GetUserNameW(user, ctypes.byref(size)): + emit(f"account = {user.value}") + + session = wintypes.DWORD() + if _kernel32.ProcessIdToSessionId(os.getpid(), ctypes.byref(session)): + # The nightly runs as a service; session 0 means no interactive desktop. + emit(f"session_id = {session.value}") + + emit(f"cwd = {os.getcwd()}") + + +def collect_gpu() -> None: + # -q carries the fields the plain table hides: the TCC/MCDM mode, and on + # r600+ the split KMD / CUDA UMD versions (see section 17.6). + proc = subprocess.run( + ["nvidia-smi", "-q"], # noqa: S607 - resolved via PATH on purpose + capture_output=True, + text=True, + timeout=120, + check=False, + ) + if proc.returncode != 0: + emit(f"") + emit(proc.stderr.strip()[:500]) + return + wanted = ( + "Driver Version", + "KMD Version", + "CUDA Version", + "CUDA UMD Version", + "Product Name", + "Compute Mode", + "GPU UUID", + ) + # "Current"/"Pending" appear under a dozen unrelated headings (temperature, + # page retirement, PCIe width), so they are only taken while inside the + # Driver Model block -- that pair is the TCC/MCDM/WDDM answer. + in_driver_model = 0 + for line in proc.stdout.splitlines(): + stripped = line.strip() + if stripped.startswith("Driver Model"): + emit(line.rstrip()) + in_driver_model = 2 + continue + if in_driver_model and (stripped.startswith(("Current", "Pending"))): + emit(line.rstrip()) + in_driver_model -= 1 + continue + in_driver_model = 0 + if any(key in line for key in wanted): + emit(line.rstrip()) + + +def pe_stack_reserve(path: str) -> tuple[int, int]: + """(reserve, commit) for the main thread, from the executable's PE header. + + The main thread's stack is whatever the linker wrote here; only threads + created later can pick their own size. python.org builds link with + /STACK:3000000 (2.86 MB); other redistributions need not, and a Cython + linetrace build's module-init frames are large enough for the difference + to decide whether `import cuda.core` survives. + """ + with open(path, "rb") as fh: + data = fh.read(1024) + pe = struct.unpack_from(" None: + emit(f"executable = {sys.executable}") + emit(f"version = {sys.version}") + emit(f"bits = {ctypes.sizeof(ctypes.c_void_p) * 8}") + try: + reserve, commit = pe_stack_reserve(sys.executable) + emit(f"main_stack_mb = {reserve / 1024 / 1024:.2f} (PE SizeOfStackReserve)") + emit(f"main_stack_commit_kb = {commit / 1024:.0f}") + except Exception as exc: + emit(f"main_stack_mb = <{type(exc).__name__}: {exc}>") + current = threading.current_thread() + emit(f"thread = {current.name} (main={current is threading.main_thread()})") + emit(f"thread_stack_mb = {threading.stack_size() / 1024 / 1024:.2f} (0.00 = OS default)") + for dist in ("coverage", "Cython", "pytest", "numpy"): + try: + from importlib.metadata import version as _version + + emit(f"{dist:<15} = {_version(dist)}") + except Exception as exc: + emit(f"{dist:<15} = <{type(exc).__name__}>") + + +def collect_wheel() -> None: + """Prove the installed cuda.core is a linetrace build, without importing it. + + Necessary condition 1 (section 4) is the cov wheel, and a package name in + the registry is not evidence of it -- a nocov set has been published under + the cov name before. Two physical markers settle it: setup.py's build_py + only ships .pyx/.cpp alongside the .pyd when CUDA_PYTHON_COVERAGE=1, and + the generated .cpp carries the compile flags in its Cython Metadata header. + + Uses find_spec so the parent namespace package is all that gets imported; + `import cuda.core` is the thing that crashes, and it happens later. + """ + import importlib.util + + spec = importlib.util.find_spec("cuda.core") + if spec is None or not spec.submodule_search_locations: + emit("") + return + pkg_dir = next(iter(spec.submodule_search_locations)) + emit(f"package_dir = {pkg_dir}") + + for name in ("_stream.pyx", "_stream.cpp", "_resource_handles.pyx"): + path = os.path.join(pkg_dir, name) + emit(f"{name:<22} {'present' if os.path.exists(path) else 'MISSING'}") + + cpp = os.path.join(pkg_dir, "_stream.cpp") + if os.path.exists(cpp): + # The metadata block is the first ~30 lines; the flags live in it. + with open(cpp, encoding="utf-8", errors="replace") as fh: + head = [next(fh, "") for _ in range(40)] + for line in head: + if "CYTHON_TRACE" in line or "std:c++" in line or "USE_SYS_MONITORING" in line: + emit(f"compile_arg = {line.strip()}") + emit( + "linetrace_build = " + + str(os.path.exists(os.path.join(pkg_dir, "_stream.pyx"))) + + " (sources shipped == built with CUDA_PYTHON_COVERAGE=1)" + ) + + +def collect_tracer() -> None: + """Which tracer is armed -- the whole crash hinges on it being ctrace. + + The cov wheels are built with CYTHON_USE_SYS_MONITORING=0, so the + `__Pyx_TraceLine` hooks only fire through the legacy sys.settrace path. If + coverage quietly selected sysmon here, our runs never armed the trigger and + every negative result on this runner means nothing. + """ + emit(f"sys.gettrace() = {sys.gettrace()!r}") + emit(f"sys.monitoring in use = {getattr(sys, 'monitoring', None) is not None}") + try: + import coverage + + cov = coverage.Coverage.current() + emit(f"coverage.current() = {cov!r}") + if cov is not None: + collector = getattr(cov, "_collector", None) + emit(f"collector = {collector!r}") + for attr in ("tracer_name", "core_name"): + fn = getattr(collector, attr, None) + if callable(fn): + emit(f"{attr}() = {fn()}") + elif fn is not None: + emit(f"{attr} = {fn}") + emit(f"plugins = {list(getattr(cov, '_plugins', []) or [])}") + except Exception as exc: + emit(f"") + + +def run(out_path: str | None) -> None: + emit("cuda.core import fingerprint") + emit(f"timestamp = {time.strftime('%Y-%m-%d %H:%M:%S')}") + emit(f"argv = {sys.argv}") + + safe("host", collect_host) + safe("gpu", collect_gpu) + safe("python", collect_python) + safe("wheel", collect_wheel) + safe("tracer (before import)", collect_tracer) + + before = loaded_modules() + section("modules before import") + emit(f"count = {len(before)}") + + section("import cuda.core") + started = time.perf_counter() + failure = None + try: + import cuda.core + + emit(f"import OK, version = {cuda.core.__version__}") + except BaseException as exc: # report, do not propagate + failure = exc + emit(f"import FAILED: {type(exc).__name__}: {exc}") + emit(f"elapsed_s = {time.perf_counter() - started:.3f}") + + after = loaded_modules() + section("modules loaded by the import") + new = {path: info for path, info in after.items() if path not in before} + emit(f"count before = {len(before)}, after = {len(after)}, new = {len(new)}") + for row in format_modules(new): + emit(row) + + section("driver DLLs anywhere in the process") + # F1: does the import touch the driver at all? If nvcuda.dll never shows + # up, driver version and TCC/MCDM cannot be the trigger. + for needle in ("nvcuda", "nvml", "cudart", "nvrtc", "nvJitLink", "nvvm"): + hits = [p for p in after if needle.lower() in os.path.basename(p).lower()] + emit(f"{needle:<10} -> {hits if hits else ''}") + + safe("tracer (after import)", collect_tracer) + + section("full module map after import") + for row in format_modules(after): + emit(row) + + section("result") + emit(f"import_failed = {failure is not None}") + + if out_path: + with open(out_path, "w", encoding="utf-8") as fh: + fh.write("\n".join(_out_lines) + "\n") + print(f"fingerprint written to {out_path}", flush=True) + + +def main() -> int: + argv = sys.argv[1:] + stack_mb = 0.0 + if "--stack-mb" in argv: + i = argv.index("--stack-mb") + stack_mb = float(argv[i + 1]) + del argv[i : i + 2] + out_path = argv[0] if argv else None + + if not stack_mb: + run(out_path) + return 0 + + # The nightly's cuda.core step runs everything inside run_pytest_with_stack.py's + # worker, so an import on the main thread is not the same measurement: on a + # build whose PE stack reserve is small, it dies of stack exhaustion long + # before reaching whatever the nightly hits. Matching the wrapper's thread + # size is what makes the two comparable. + threading.stack_size(int(stack_mb * 1024 * 1024)) + worker = threading.Thread(target=run, args=(out_path,), name=f"stack-{stack_mb:g}mb") + worker.start() + worker.join() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/tools/native_crash_handler.py b/ci/tools/native_crash_handler.py new file mode 100644 index 00000000000..bcedb5955e2 --- /dev/null +++ b/ci/tools/native_crash_handler.py @@ -0,0 +1,327 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Record the native side of a Windows fault, from inside the faulting process. + +faulthandler names the exception and prints Python frames, but the fault we are +chasing happens inside an extension module's initialiser, so the interesting +frames are the ones it cannot see: ``. + +Nothing that would normally fill that gap is available on the runner this has +to work on. A postmortem debugger registered under AeDebug never fires, +because that Server Core image does not record user-process crashes with WER -- +that is why the crash-dumps directory came back empty the last time. cdb is +not installed: the SDK is there but without the Debugging Tools feature, so +`Windows Kits\\10\\Debuggers\\x64` holds the DLLs and no debugger executables. +Downloading procdump means an outbound request on a runner whose whole job is +to police outbound requests. And there are no PDBs either way, since +build_hooks.py refuses debuggable builds on Windows. + +What *is* always present is dbgcore.dll in System32, exporting +MiniDumpWriteDump. So the process reports on itself: a vectored exception +handler catches the fault first-chance and writes, in order, + + the exception record code, flags, faulting address, and for an access + violation the operation and target address + the module map base and size for every loaded module, so an + address becomes module + RVA + the native return chain RtlCaptureStackBackTrace, each frame resolved the + same way + a minidump for reading back with a real debugger later + +then returns EXCEPTION_CONTINUE_SEARCH so the process still dies exactly as it +would have. Symbols are absent, so the output is addresses; module + RVA is +enough to say which binary faulted, which is the question being asked. + +Usage: + + import native_crash_handler + native_crash_handler.install("crash-native.txt", dump_path="crash.dmp") +""" + +from __future__ import annotations + +import contextlib +import ctypes +import ctypes.wintypes as wintypes +import os + +EXCEPTION_CONTINUE_SEARCH = 0 + +# Only genuinely fatal codes. A process raises first-chance exceptions all the +# time -- 0xE06D7363 for every C++ throw, 0x406D1388 when a debugger names a +# thread -- and reporting those would bury the one that matters. +FATAL_CODES = { + 0xC0000005: "EXCEPTION_ACCESS_VIOLATION", + 0xC00000FD: "EXCEPTION_STACK_OVERFLOW", + 0xC0000374: "STATUS_HEAP_CORRUPTION", + 0xC0000409: "STATUS_STACK_BUFFER_OVERRUN", + 0xC000041D: "STATUS_FATAL_USER_CALLBACK_EXCEPTION", + 0xC000001D: "EXCEPTION_ILLEGAL_INSTRUCTION", + 0x80000003: "STATUS_BREAKPOINT", +} + +# MiniDumpNormal plus the extras that make a stack walk possible. Deliberately +# not MiniDumpWithFullMemory: python.exe with ~120 modules loaded dumps +# hundreds of megabytes that way, and the stacks are what we came for. +MINIDUMP_TYPE = 0x0000 | 0x0040 | 0x0100 | 0x1000 + +_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +_psapi = ctypes.WinDLL("psapi", use_last_error=True) +_ntdll = ctypes.WinDLL("ntdll", use_last_error=True) + + +class EXCEPTION_RECORD(ctypes.Structure): # noqa: N801 - mirrors the Win32 name + pass + + +EXCEPTION_RECORD._fields_ = [ + ("ExceptionCode", wintypes.DWORD), + ("ExceptionFlags", wintypes.DWORD), + ("ExceptionRecord", ctypes.POINTER(EXCEPTION_RECORD)), + ("ExceptionAddress", ctypes.c_void_p), + ("NumberParameters", wintypes.DWORD), + ("ExceptionInformation", ctypes.c_size_t * 15), +] + + +class EXCEPTION_POINTERS(ctypes.Structure): # noqa: N801 - mirrors the Win32 name + _fields_ = [ + ("ExceptionRecord", ctypes.POINTER(EXCEPTION_RECORD)), + ("ContextRecord", ctypes.c_void_p), + ] + + +class MODULEINFO(ctypes.Structure): + _fields_ = [ + ("lpBaseOfDll", ctypes.c_void_p), + ("SizeOfImage", wintypes.DWORD), + ("EntryPoint", ctypes.c_void_p), + ] + + +class MINIDUMP_EXCEPTION_INFORMATION(ctypes.Structure): # noqa: N801 - mirrors the Win32 name + _fields_ = [ + ("ThreadId", wintypes.DWORD), + ("ExceptionPointers", ctypes.POINTER(EXCEPTION_POINTERS)), + ("ClientPointers", wintypes.BOOL), + ] + + +PVECTORED_EXCEPTION_HANDLER = ctypes.WINFUNCTYPE(ctypes.c_long, ctypes.POINTER(EXCEPTION_POINTERS)) + +# argtypes are not optional here. Without them ctypes marshals a Python int +# as a C int, and every HANDLE and module base on win64 is wider than that: +# the truncated value comes back as a failed call, inside an exception handler +# where the failure has nowhere to surface. +_kernel32.GetCurrentProcess.restype = wintypes.HANDLE +_kernel32.AddVectoredExceptionHandler.argtypes = [wintypes.ULONG, PVECTORED_EXCEPTION_HANDLER] +_kernel32.AddVectoredExceptionHandler.restype = ctypes.c_void_p +_psapi.EnumProcessModules.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(ctypes.c_void_p), + wintypes.DWORD, + ctypes.POINTER(wintypes.DWORD), +] +_psapi.GetModuleFileNameExW.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + wintypes.LPWSTR, + wintypes.DWORD, +] +_psapi.GetModuleInformation.argtypes = [ + wintypes.HANDLE, + ctypes.c_void_p, + ctypes.POINTER(MODULEINFO), + wintypes.DWORD, +] +_ntdll.RtlCaptureStackBackTrace.argtypes = [ + wintypes.ULONG, + wintypes.ULONG, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(wintypes.ULONG), +] +_ntdll.RtlCaptureStackBackTrace.restype = wintypes.USHORT + +# Held at module scope on purpose. ctypes does not keep the callback alive, +# and a collected handler is a crash of its own; the log handle must outlive +# install() for the same reason. +_handler_ref = None +_log = None +_dump_path = None + +# A vectored handler sees first-chance exceptions, including ones somebody is +# about to handle -- ctypes turns an access violation into OSError exactly +# that way, which is how --selftest reports rather than dies. Reporting only +# the first one would let a benign, handled fault earlier in the import +# consume the single slot and leave the real crash unrecorded, so a few are +# allowed through and each is numbered. +_MAX_REPORTS = 3 +_reports = 0 + + +def _modules() -> list[tuple[int, int, str]]: + """(base, size, path), sorted by base, for every module mapped right now.""" + process = _kernel32.GetCurrentProcess() + needed = wintypes.DWORD() + arr = (ctypes.c_void_p * 1024)() + if not _psapi.EnumProcessModules(process, arr, ctypes.sizeof(arr), ctypes.byref(needed)): + return [] + count = min(needed.value // ctypes.sizeof(ctypes.c_void_p), len(arr)) + name = ctypes.create_unicode_buffer(32768) + out = [] + for i in range(count): + hmod = arr[i] + if not _psapi.GetModuleFileNameExW(process, hmod, name, len(name)): + continue + info = MODULEINFO() + if not _psapi.GetModuleInformation(process, hmod, ctypes.byref(info), ctypes.sizeof(info)): + continue + out.append((info.lpBaseOfDll or 0, info.SizeOfImage, name.value)) + out.sort() + return out + + +def _resolve(address: int, modules: list[tuple[int, int, str]]) -> str: + """address -> "module+0xRVA", the closest thing to a symbol available here.""" + for base, size, path in modules: + if base <= address < base + size: + return f"{os.path.basename(path)}+0x{address - base:x}" + return "" + + +def _write_dump(exception_pointers, index: int) -> None: + if not _dump_path: + return + # One file per report, so a second fault cannot overwrite the first. + root, ext = os.path.splitext(_dump_path) + dump_path = _dump_path if index == 1 else f"{root}.{index}{ext}" + try: + import msvcrt + + # MiniDumpWriteDump lives in dbgcore.dll, and lived in dbghelp.dll + # before that; both are system DLLs, so at least one is present on + # anything this could run on. If neither loads, the text report above + # already carries the answer -- the dump is the convenience, not the + # evidence -- so this says so and returns instead of failing. + dbgcore = None + for name in ("dbgcore", "dbghelp"): + try: + dbgcore = ctypes.WinDLL(name, use_last_error=True) + _log.write(f"dump provider: {name}.dll\n") + break + except OSError: + continue + if dbgcore is None: + _log.write("no dbgcore.dll or dbghelp.dll; dump skipped\n") + return + dbgcore.MiniDumpWriteDump.argtypes = [ + wintypes.HANDLE, + wintypes.DWORD, + wintypes.HANDLE, + wintypes.DWORD, + ctypes.c_void_p, # PMINIDUMP_EXCEPTION_INFORMATION, or NULL + ctypes.c_void_p, + ctypes.c_void_p, + ] + dbgcore.MiniDumpWriteDump.restype = wintypes.BOOL + info = MINIDUMP_EXCEPTION_INFORMATION( + ThreadId=_kernel32.GetCurrentThreadId(), + ExceptionPointers=exception_pointers, + ClientPointers=False, + ) + # Attaching the exception record is worth a try -- it is what makes a + # debugger open the dump on the faulting instruction -- but writing it + # from inside a vectored handler has been seen to come back as + # ERROR_NOACCESS (998). The thread stacks are the reason for the dump + # and they are present either way, and the exception itself is already + # recorded above in text, so a refusal falls back rather than gives up. + for exception_param in (ctypes.byref(info), None): + with open(dump_path, "wb") as fh: + ok = dbgcore.MiniDumpWriteDump( + _kernel32.GetCurrentProcess(), + _kernel32.GetCurrentProcessId(), + wintypes.HANDLE(msvcrt.get_osfhandle(fh.fileno())), + MINIDUMP_TYPE, + exception_param, + None, + None, + ) + if ok: + size = os.path.getsize(dump_path) + kind = "with exception record" if exception_param is not None else "no exception record" + _log.write(f"dump written: {_dump_path} ({size} bytes, {kind})\n") + return + _log.write( + f"MiniDumpWriteDump failed (exception_param={exception_param is not None}), " + f"GetLastError={ctypes.get_last_error() & 0xFFFFFFFF}\n" + ) + except Exception as exc: # a failed dump must not hide the report + _log.write(f"dump failed: {type(exc).__name__}: {exc}\n") + + +def _on_exception(pointers): + global _reports + try: + record = pointers[0].ExceptionRecord[0] + code = record.ExceptionCode & 0xFFFFFFFF + if code not in FATAL_CODES or _reports >= _MAX_REPORTS: + return EXCEPTION_CONTINUE_SEARCH + # Counted before doing any work: if the reporting itself faults, the + # re-entry must fall through instead of recursing forever. + _reports += 1 + + # Header first, before anything that can fail: if module enumeration + # or the dump goes wrong, the record of the fault itself survives. + address = record.ExceptionAddress or 0 + _log.write(f"=== native fault #{_reports} ===\n") + _log.write(f"code = 0x{code:08X} ({FATAL_CODES[code]})\n") + modules = _modules() + _log.write(f"flags = 0x{record.ExceptionFlags:08X}\n") + _log.write(f"address = 0x{address:016x} {_resolve(address, modules)}\n") + if code == 0xC0000005 and record.NumberParameters >= 2: + op = {0: "read", 1: "write", 8: "DEP"}.get(record.ExceptionInformation[0], "?") + target = record.ExceptionInformation[1] + _log.write(f"operation = {op} at 0x{target:016x} {_resolve(target, modules)}\n") + _log.write(f"thread = {_kernel32.GetCurrentThreadId()}\n") + + frames = (ctypes.c_void_p * 62)() + n = _ntdll.RtlCaptureStackBackTrace(0, 62, frames, None) + _log.write(f"--- native backtrace ({n} frames, innermost first) ---\n") + for i in range(n): + frame = frames[i] or 0 + _log.write(f" [{i:02d}] 0x{frame:016x} {_resolve(frame, modules)}\n") + + _log.write(f"--- modules ({len(modules)}) ---\n") + for base, size, path in modules: + _log.write(f" 0x{base:016x} {size:>9} {path}\n") + + _write_dump(pointers, _reports) + _log.write(f"=== end native fault #{_reports} ===\n") + except BaseException as exc: # never let reporting change the outcome + # Reported, not swallowed: a silent handler is indistinguishable from + # one that never ran, and that ambiguity costs a whole CI round. The + # log is what would have failed, so even saying so is allowed to fail. + with contextlib.suppress(Exception): + _log.write(f"handler error: {type(exc).__name__}: {exc}\n") + # Always continue searching: the process must die the way it would have, + # so the exit code the CI sees is unchanged. + return EXCEPTION_CONTINUE_SEARCH + + +def install(log_path: str, dump_path: str | None = None) -> None: + """Arm the handler. Call once, as early as possible, from any thread.""" + global _handler_ref, _log, _dump_path + + # Line buffered: a hard fault takes the process down without flushing, so + # every line has to reach the OS as it is written. + _log = open( # noqa: SIM115 - must outlive this function + os.path.abspath(log_path), "w", buffering=1, encoding="utf-8", errors="backslashreplace" + ) + _dump_path = os.path.abspath(dump_path) if dump_path else None + _handler_ref = PVECTORED_EXCEPTION_HANDLER(_on_exception) + if not _kernel32.AddVectoredExceptionHandler(1, _handler_ref): + _log.write("AddVectoredExceptionHandler failed\n") + return + _log.write(f"handler armed, pid={os.getpid()}, dump={_dump_path or ''}\n") diff --git a/ci/tools/pytest_crashlog.py b/ci/tools/pytest_crashlog.py new file mode 100644 index 00000000000..c5df2306b1f --- /dev/null +++ b/ci/tools/pytest_crashlog.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Record session progress to a file so a hard crash names where it hit. + +When the interpreter dies from a native fault there is no report, no +summary, and often no console output at all: pytest's terminal writer goes +through the capture manager and through a block-buffered pipe, both of +which lose whatever had not been flushed. Writing progress to a +line-buffered file instead pushes each line to the OS before the next step +begins, so the record survives the process. + +The log covers the whole session, not just the test loop, because a crash +during collection looks exactly like a crash before startup otherwise: + + (file absent) crashed before open_log(), i.e. during interpreter + startup or `import pytest` + open ... entering crashed while pytest loaded its initial conftests + -- for cuda_core that means importing cuda.core + COLLECT crashed importing or collecting that module + START crashed inside that test + closed cleanly crashed after the session finished, e.g. during + interpreter shutdown or coverage's atexit write + +Enabled by setting PYTEST_CRASHLOG to the destination path; a no-op +otherwise. ci/tools/run_pytest_with_stack.py calls open_log() before +handing control to pytest, then loads this module as a plugin. +""" + +import os +import sys + +_log = None +_disabled = False + + +def open_log(path=None): + """Open the crash log. Safe to call before pytest starts, and twice.""" + global _log, _disabled + if _log is not None or _disabled: + return + path = path or os.environ.get("PYTEST_CRASHLOG") + if not path: + return + try: + # buffering=1 is line buffering: every newline reaches the OS, so a + # crash cannot take the preceding lines down with it. The handle + # deliberately outlives this function, so no context manager. + _log = open( # noqa: SIM115 + os.path.abspath(path), "w", buffering=1, encoding="utf-8", errors="backslashreplace" + ) + except OSError as exc: + # A diagnostic must never break the run it is diagnosing. + _disabled = True + print(f"[crashlog] disabled, cannot write {path}: {exc}", file=sys.stderr, flush=True) + return + _log.write(f"open pid={os.getpid()}\n") + + +def note(message): + """Append one line to the crash log, if it is open.""" + if _log is not None: + _log.write(f"{message}\n") + + +class _ImportNoter: + """A meta path finder that records a name and then steps aside. + + Returning None means "I cannot supply this module", so the real finders + run right after and behaviour is unchanged. What it buys is a line in + the log for every import the interpreter begins, in the order it begins + them -- and because the log is line buffered, the innermost name is on + disk before the module it names starts executing. A crash inside an + extension module's init therefore leaves that module as the last line. + + This exists because PYTHONVERBOSE could not answer the same question: + its output goes through the interpreter's C-level stdio, which buffers + when redirected to a file, so a fault takes the last several hundred + lines with it. + """ + + @staticmethod + def find_spec(fullname, path=None, target=None): # noqa: ARG004 - MetaPathFinder signature + note(f"IMPORT {fullname}") + return None + + +def _note_unraisable(unraisable): + """Record what would otherwise scroll past as `Exception ignored in:`. + + A Cython `cdef ... noexcept` function cannot propagate an exception, so + one raised inside it is printed and cleared, and whatever it was + computing is left at its default -- a NULL function pointer, in the case + of cuda.core's driver-pointer initialisers, which faults only later when + something calls through it. Routing unraisables into this log puts them + in sequence with the IMPORT lines, so a swallowed error and the module + that swallowed it appear together. + """ + exc_type = getattr(unraisable.exc_type, "__name__", unraisable.exc_type) + note(f"UNRAISABLE {exc_type}: {unraisable.exc_value!r} in {unraisable.object!r}") + if _prev_unraisablehook is not None: + _prev_unraisablehook(unraisable) + + +_prev_unraisablehook = None +_tracing = False + + +def enable_tracing(): + """Start recording imports and unraisable exceptions. Idempotent.""" + global _prev_unraisablehook, _tracing + if _tracing or _log is None: + return + _tracing = True + # First in the list, so the name is recorded before any real finder can + # start loading it. + sys.meta_path.insert(0, _ImportNoter) + _prev_unraisablehook = sys.unraisablehook + sys.unraisablehook = _note_unraisable + note("tracing enabled") + + +def pytest_configure(): + # Covers direct `-p pytest_crashlog` use, where nothing called open_log(). + # By the time this runs the initial conftests have already been imported, + # which is why the wrapper opens the log earlier. + open_log() + note("configured") + + +def pytest_unconfigure(): + global _log + if _log is not None: + _log.write("closed cleanly\n") + _log.close() + _log = None + + +def pytest_collectstart(collector): + note(f"COLLECT {collector.nodeid or ''}") + + +def pytest_collection_finish(session): + note(f"collected {len(session.items)} items") + + +def pytest_runtest_logstart(nodeid): + note(f"START {nodeid}") + + +def pytest_runtest_logfinish(nodeid): + note(f"END {nodeid}") diff --git a/ci/tools/run_pytest_with_stack.py b/ci/tools/run_pytest_with_stack.py index e1e1f782ba1..bd73988c8a4 100644 --- a/ci/tools/run_pytest_with_stack.py +++ b/ci/tools/run_pytest_with_stack.py @@ -11,18 +11,66 @@ a configurable stack (default 8 MB) so the rest of the CI workflow stays readable. +With --isolate, pytest runs in a child process instead and the parent +reports the child's raw exit status. On Windows the shell only ever +shows a translation of that status -- Git Bash turns an access violation +into "Segmentation fault" and exit 139 -- which loses the distinction +between, say, a use-after-free and a blown stack. A surviving parent +names the NTSTATUS outright, so nothing has to be inferred. + Usage: - python run_pytest_with_stack.py [--stack-mb N] [--cwd DIR] [pytest args ...] + python run_pytest_with_stack.py [--stack-mb N] [--cwd DIR] [--isolate] + [pytest args ...] """ import argparse import concurrent.futures import os +import subprocess import sys import threading import pytest +# Windows exception codes worth naming in a CI log. Anything else is +# printed as a bare hex status. +_NTSTATUS = { + 0xC0000005: "EXCEPTION_ACCESS_VIOLATION", + 0xC000001D: "EXCEPTION_ILLEGAL_INSTRUCTION", + 0xC0000094: "EXCEPTION_INT_DIVIDE_BY_ZERO", + 0xC00000FD: "EXCEPTION_STACK_OVERFLOW", + 0xC0000374: "STATUS_HEAP_CORRUPTION", + 0xC000041D: "STATUS_FATAL_USER_CALLBACK_EXCEPTION", + 0xC0000409: "STATUS_STACK_BUFFER_OVERRUN", +} + + +def _describe_exit(code): + """Render an abnormal child exit status, or None if it looks normal.""" + if 0 <= code <= 255: + return None + if code < 0: # POSIX: killed by signal -code + return f"killed by signal {-code}" + status = code & 0xFFFFFFFF + return f"0x{status:08X} ({_NTSTATUS.get(status, 'unrecognized status')})" + + +def _run_isolated(stack_mb, cwd, pytest_args): + """Re-run this script as a child so the parent survives a native crash.""" + cmd = [sys.executable, os.path.abspath(__file__), f"--stack-mb={stack_mb}"] + if cwd: + cmd += ["--cwd", cwd] + cmd += pytest_args + + print(f"[stack-wrapper] launching child: {cmd}", flush=True) + code = subprocess.call(cmd) # noqa: S603 - cmd is this interpreter re-running this script + described = _describe_exit(code) + if described is None: + print(f"[stack-wrapper] child exited with {code}", flush=True) + else: + print(f"[stack-wrapper] child died abnormally: {described}", flush=True) + return code + def main(): parser = argparse.ArgumentParser(description=__doc__) @@ -37,15 +85,66 @@ def main(): default=None, help="Working directory for the test run", ) + parser.add_argument( + "--isolate", + action="store_true", + help="Run pytest in a child process and report its raw exit status", + ) args, pytest_args = parser.parse_known_args() + if args.isolate: + sys.exit(_run_isolated(args.stack_mb, args.cwd, pytest_args)) + + # PYTHONFAULTHANDLER writes to stderr, and the nightly's stderr has so far + # carried nothing at all -- which leaves two very different readings: the + # handler produced a traceback that the dying process failed to flush, or + # it never ran because the fault happened outside any Python frame. A + # dedicated unbuffered file separates them: still empty here means the + # handler genuinely had nothing to say. The handle is deliberately kept + # alive for the life of the process; faulthandler writes through its fd. + if os.environ.get("PYTEST_FAULTLOG"): + import faulthandler + + try: + _fault_fp = open(os.environ["PYTEST_FAULTLOG"], "w", buffering=1) # noqa: SIM115 + except OSError as exc: + print(f"[stack-wrapper] cannot open fault log: {exc}", file=sys.stderr, flush=True) + else: + faulthandler.enable(file=_fault_fp, all_threads=True) + + plugins = [] + if os.environ.get("PYTEST_CRASHLOG"): + # Load the progress logger from this script's directory, which is not + # otherwise importable from the install root we chdir into below. + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import pytest_crashlog + + # Open the log here rather than from the plugin's pytest_configure: + # pytest imports the initial conftests before configure runs, and for + # cuda_core that import pulls in cuda.core itself. A crash there + # would otherwise leave no file at all, indistinguishable from a crash + # before pytest ever started. + pytest_crashlog.open_log() + # The nightly's log has so far ended at "entering pytest.main", which + # says the fault is somewhere in the initial conftest import but not + # which module. Tracing imports through the same line-buffered file + # answers that: the last IMPORT line names the module that was being + # loaded. Enabled here so it also covers pytest's own startup, since + # the fault happens before pytest_configure would run. + pytest_crashlog.enable_tracing() + pytest_crashlog.note("entering pytest.main") + # Hand pytest the imported module rather than "-p pytest_crashlog": + # naming it would make pytest import it a second time and warn that + # it can no longer rewrite the module's assertions. + plugins.append(pytest_crashlog) + if args.cwd: os.chdir(args.cwd) threading.stack_size(args.stack_mb * 1024 * 1024) with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - code = pool.submit(pytest.main, pytest_args).result() + code = pool.submit(pytest.main, pytest_args, plugins).result() sys.exit(code)