coverage: capture what the Windows coverage crash actually is

.github/workflows/coverage.yml

@@ -8,6 +8,17 @@ on:
   schedule:
     - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC
   workflow_dispatch: {}
+  # TEMPORARY (do not merge): runs the Windows coverage job against a PR, via
+  # the copy-pr-bot mirror branch.  Same trigger as ci.yml.
+  push:
+    branches:
+      - "pull-request/[0-9]+"
+
+# One coverage run per ref: the Windows leg holds an A100 runner for the whole
+# job, so superseded runs must not queue up behind each other.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
 
 env:
   PY_VER: "3.14"
@@ -32,6 +43,8 @@ jobs:
   coverage-linux:
     name: Coverage (Linux)
     needs: [coverage-vars]
+    # Only Windows crashes, so a PR run skips the Linux leg entirely.
+    if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
     runs-on: "linux-amd64-gpu-a100-latest-1"
     permissions:
       id-token: write
@@ -268,6 +281,15 @@ jobs:
       ARCH: "amd64"
       CUDA_PYTHON_COVERAGE: "1"
       CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }}
+      # Diagnostics for the recurring exit-139 crash in the cuda.core step.
+      # faulthandler names the Windows exception (access violation vs stack
+      # overflow) and dumps every thread's Python stack; unlike pytest's
+      # built-in faulthandler plugin, the env var is inherited by tests that
+      # spawn their own `python -c` children, which crash too.  Unbuffered
+      # stdio keeps the last lines before a hard crash from dying in the
+      # block buffer of a non-tty pipe.
+      PYTHONFAULTHANDLER: "1"
+      PYTHONUNBUFFERED: "1"
     defaults:
       run:
         shell: bash --noprofile --norc -xeuo pipefail {0}
@@ -365,14 +387,195 @@ jobs:
             --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
             "$GITHUB_WORKSPACE/cuda_bindings/tests"
 
+      # The whole crash reduces to one `import cuda.core` under coverage, with
+      # no pytest, no conftest and no tests: the linetrace wheel and the
+      # ctracer are the two conditions `test-wheel-windows.yml` lacks on this
+      # same runner.  Importing submodules one at a time was tried first and
+      # measured nothing -- every one of them executes cuda/core/__init__.py
+      # on the way in, so all six probes walked the same path and all six
+      # failed identically.
+      #
+      # What is missing is the native half.  The fault happens inside an
+      # extension module's initialiser, and faulthandler's report ends at
+      # `<cannot get C stack on this system>`.  The usual ways to see past
+      # that are all closed here: cdb is not installed (the SDK ships without
+      # the Debugging Tools feature), procdump would need an outbound
+      # download, a postmortem debugger under AeDebug never fires because this
+      # image does not record user-process crashes with WER, and there are no
+      # PDBs because build_hooks.py refuses debuggable builds on Windows.
+      #
+      # dbgcore.dll in System32 is always present though, so the process
+      # reports on itself.  ci/tools/native_crash_handler.py catches the fault
+      # first-chance and writes the faulting address resolved to module + RVA,
+      # the native return chain resolved the same way, the module map, and a
+      # minidump, then lets the process die exactly as it would have.  Which
+      # binary faults is the open question, and module + RVA answers it
+      # without symbols.
+      #
+      # Four runs, each answering something the others cannot:
+      #
+      #   selftest     faults on purpose, so a clean report proves the handler
+      #                works here and an empty one is not mistaken for "no
+      #                crash"
+      #   coverage     the nightly's own conditions, on the 8 MB stack the
+      #                pytest wrapper uses
+      #   breadcrumb   the same import with a bare trace function instead of
+      #                coverage.  It arms Cython's linetrace the same way, so
+      #                a crash here means any tracer will do rather than
+      #                coverage specifically -- and the last line of its log
+      #                names the .pyx statement that was executing
+      #   fingerprint  the environment around the import: driver, CPU, and
+      #                the loaded-module map, which is what a comparison
+      #                against a machine that does not crash needs
+      #
+      # COVERAGE_FILE keeps all of this out of the real .coverage, which
+      # `coverage run` would otherwise truncate.
+      - name: Record the native side of the fault
+        if: always()
+        continue-on-error: true
+        env:
+          COVERAGE_FILE: ${{ github.workspace }}/.coverage.probe
+        run: |
+          probe="$GITHUB_WORKSPACE/ci/tools/crash_probe.py"
+          python="$GITHUB_WORKSPACE/.venv/Scripts/python"
+          cd "${{ steps.install-root.outputs.INSTALL_ROOT }}"
+
+          rc=0
+          "$python" "$probe" --selftest \
+            --native-log "$GITHUB_WORKSPACE/native-selftest.txt" \
+            --dump "$GITHUB_WORKSPACE/selftest.dmp" || rc=$?
+          echo "SELFTEST rc=$rc"
+          head -n 10 "$GITHUB_WORKSPACE/native-selftest.txt" || true
+
+          rc=0
+          "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \
+            "$probe" --stack-mb 8 \
+            --native-log "$GITHUB_WORKSPACE/native-coverage.txt" \
+            --dump "$GITHUB_WORKSPACE/crash-coverage.dmp" || rc=$?
+          echo "COVERAGE PROBE rc=$rc"
+          cat "$GITHUB_WORKSPACE/native-coverage.txt" || true
+
+          rc=0
+          "$python" "$probe" --stack-mb 8 \
+            --breadcrumb "$GITHUB_WORKSPACE/breadcrumb.txt" \
+            --native-log "$GITHUB_WORKSPACE/native-breadcrumb.txt" \
+            --dump "$GITHUB_WORKSPACE/crash-breadcrumb.dmp" || rc=$?
+          echo "BREADCRUMB PROBE rc=$rc"
+          echo "=== breadcrumb: $(wc -l < "$GITHUB_WORKSPACE/breadcrumb.txt") lines, last 25 ==="
+          tail -n 25 "$GITHUB_WORKSPACE/breadcrumb.txt" || true
+
+          rc=0
+          "$python" -m coverage run --rcfile="$GITHUB_WORKSPACE/.coveragerc" \
+            "$GITHUB_WORKSPACE/ci/tools/env_fingerprint.py" --stack-mb 8 \
+            "$GITHUB_WORKSPACE/fingerprint-windows.txt" || rc=$?
+          echo "FINGERPRINT rc=$rc"
+
+      # Only this step crashes.  cuda.bindings runs the same wrapper, the same
+      # 8 MB thread and the same coverage wheel, and finishes normally, so the
+      # extra diagnostics are scoped to cuda.core alone.
+      #
+      # The crash log narrows the fault to "somewhere before pytest collects".
+      # PYTHONVERBOSE narrows it to a module: the interpreter announces every
+      # import as it begins one, so the last line written before the process
+      # dies names the module it died in.  That output is stderr, which is why
+      # the session is redirected to a file rather than left in the step log --
+      # a crash takes the tail of a pipe with it, while each unbuffered write
+      # to a file has already reached the OS.
       - name: Run cuda.core tests (with 8MB stack)
         continue-on-error: true
+        env:
+          PYTEST_CRASHLOG: ${{ github.workspace }}/crashlog-core.txt
+          PYTEST_FAULTLOG: ${{ github.workspace }}/faulthandler-core.txt
+          PYTHONVERBOSE: "1"
         run: |
           "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \
+            --isolate \
             --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \
             -v --cov=./cuda --cov-append --cov-context=test \
             --cov-config="$GITHUB_WORKSPACE/.coveragerc" \
-            "$GITHUB_WORKSPACE/cuda_core/tests"
+            "$GITHUB_WORKSPACE/cuda_core/tests" \
+            > "$GITHUB_WORKSPACE/verbose-core.txt" 2>&1
+
+      # The step above has been exiting 139 on every run, and Git Bash reports
+      # that as a bare "Segmentation fault".  The crash log's final line says
+      # how far the session actually got.
+      - name: Report how far the cuda.core session got
+        if: always()
+        continue-on-error: true
+        run: |
+          log="$GITHUB_WORKSPACE/crashlog-core.txt"
+          if [ -f "$log" ]; then
+            echo "=== crashlog: $(wc -l < "$log") lines, last 20 ==="
+            tail -n 20 "$log"
+          else
+            echo "No crash log: died before pytest started."
+          fi
+
+          echo ""
+          fault="$GITHUB_WORKSPACE/faulthandler-core.txt"
+          if [ -s "$fault" ]; then
+            echo "=== faulthandler ==="
+            cat "$fault"
+          else
+            # Empty is itself a result: the fault never reached Python's
+            # handler, which points at native code running outside any Python
+            # frame -- an extension module's initialisation, for instance.
+            echo "=== faulthandler: no output ==="
+          fi
+
+          echo ""
+          verbose="$GITHUB_WORKSPACE/verbose-core.txt"
+          if [ -f "$verbose" ]; then
+            echo "=== PYTHONVERBOSE: $(wc -l < "$verbose") lines, last 40 ==="
+            tail -n 40 "$verbose"
+          fi
+
+      # Windows records an Application Error entry for every process it kills,
+      # naming the faulting module and the offset within it, and it does so
+      # whether or not crash dumps are configured.  That is the one fact no
+      # Python-level log can produce, and reading the log changes no machine
+      # state, so nothing has to be restored afterwards.
+      - name: Read the crash record from the Windows event log
+        if: always()
+        continue-on-error: true
+        shell: powershell
+        run: |
+          $events = Get-WinEvent -FilterHashtable @{
+            LogName      = 'Application'
+            ProviderName = 'Application Error', 'Windows Error Reporting'
+            StartTime    = (Get-Date).AddHours(-2)
+          } -ErrorAction SilentlyContinue
+          if (-not $events) {
+            Write-Output "No Application Error / WER records in the last 2 hours."
+            Write-Output "(If this image keeps no such log, a minidump is the fallback.)"
+            exit 0
+          }
+          foreach ($e in $events | Select-Object -First 15) {
+            Write-Output "===== $($e.TimeCreated.ToString('u'))  $($e.ProviderName)  Id=$($e.Id) ====="
+            Write-Output $e.Message
+            Write-Output ""
+          }
+
+      - name: Upload crash diagnostics
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: crash-diagnostics-windows
+          path: |
+            crashlog-core.txt
+            faulthandler-core.txt
+            verbose-core.txt
+            native-selftest.txt
+            native-coverage.txt
+            native-breadcrumb.txt
+            breadcrumb.txt
+            fingerprint-windows.txt
+            selftest.dmp
+            crash-coverage.dmp
+            crash-breadcrumb.dmp
+          retention-days: 7
+          if-no-files-found: warn
 
       - name: Copy Windows coverage file to workspace
         run: |
@@ -393,7 +596,10 @@ jobs:
     name: Combine Coverage and Deploy
     needs: [coverage-linux, coverage-windows]
     runs-on: ubuntu-latest
-    if: always()
+    # always() alone would still run this after coverage-linux is skipped, and
+    # it would both fail on the missing .coverage.linux and publish a half
+    # report to gh-pages.
+    if: ${{ always() && !startsWith(github.ref, 'refs/heads/pull-request/') }}
     permissions:
       id-token: write
       contents: write

ci/tools/crash_probe.py

@@ -0,0 +1,125 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""Import cuda.core under full instrumentation and record where it dies.
+
+The crash reduces to a single import, on a thread the size of the one
+run_pytest_with_stack.py creates.  Two records are taken around it, and they
+answer different halves of the same question:
+
+    the breadcrumb   the last .pyx line that executed, i.e. which source
+                     statement was running -- written by our own trace
+                     function, which also arms Cython's linetrace hooks the
+                     way coverage does, so the fault still triggers
+    the native log   which module faulted and at what offset, from
+                     native_crash_handler
+
+A run under coverage answers "does it still crash", and a run with the
+breadcrumb instead of coverage answers "is coverage necessary, or does any
+tracer do".  Only one trace function can be installed at a time, so those are
+separate invocations rather than one.
+
+    crash_probe.py --stack-mb 8 --native-log native.txt --dump crash.dmp
+    crash_probe.py --stack-mb 8 --breadcrumb lines.txt --native-log native.txt
+    crash_probe.py --selftest --native-log native.txt
+
+--selftest writes to address zero on purpose.  It is how the whole chain gets
+verified somewhere that does not crash on its own: if the handler, the module
+map, the dump and the artifact upload all work there, then the one gated run
+on the runner that does crash is not spent discovering a typo.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+import importlib
+import os
+import sys
+import threading
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import native_crash_handler
+
+
+def install_breadcrumb(path: str, needle: str) -> None:
+    """Log every executed line whose file matches `needle`, line buffered.
+
+    Cython emits __Pyx_TraceLine for module-level statements too, which is the
+    only reason this can see anything at all: the fault happens while a module
+    body is still executing, before any function in it has been called.
+    """
+    log = open(  # noqa: SIM115 - must outlive this function
+        os.path.abspath(path), "w", buffering=1, encoding="utf-8", errors="backslashreplace"
+    )
+
+    def tracer(frame, event, _arg):
+        if event == "line":
+            filename = frame.f_code.co_filename
+            if needle in filename:
+                log.write(f"{filename}:{frame.f_lineno}\n")
+        return tracer
+
+    # Only one trace function exists per thread, so arming this one under
+    # `coverage run` would quietly switch coverage off and leave a run that
+    # looks like it measured something.  Say so rather than let it pass.
+    displaced = sys.gettrace()
+    if displaced is not None:
+        message = f"# WARNING: replaced an existing trace function: {displaced!r}"
+        log.write(message + "\n")
+        print(message, flush=True)
+
+    log.write(f"# breadcrumb armed, filter={needle!r}\n")
+    sys.settrace(tracer)
+
+
+def selftest() -> None:
+    """Deliberately fault, to prove the handler reports what it should."""
+    print("selftest: writing to address 0", flush=True)
+    ctypes.memset(0, 0, 1)
+    print("selftest: still alive -- the write did NOT fault", flush=True)
+
+
+def body(args) -> None:
+    if args.breadcrumb:
+        install_breadcrumb(args.breadcrumb, args.breadcrumb_filter)
+    if args.selftest:
+        selftest()
+        return
+    started = time.perf_counter()
+    importlib.import_module(args.module)
+    print(f"PROBE OK {args.module} in {time.perf_counter() - started:.2f}s", flush=True)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--module", default="cuda.core")
+    parser.add_argument("--stack-mb", type=float, default=0.0)
+    parser.add_argument("--native-log", default="crash-native.txt")
+    parser.add_argument("--dump")
+    parser.add_argument("--breadcrumb")
+    parser.add_argument("--breadcrumb-filter", default="cuda")
+    parser.add_argument("--selftest", action="store_true")
+    args = parser.parse_args()
+
+    # Process-wide, so arming it here covers the worker thread as well.
+    native_crash_handler.install(args.native_log, dump_path=args.dump)
+
+    if not args.stack_mb:
+        body(args)
+        return 0
+
+    # sys.settrace only affects the thread that calls it, so the breadcrumb is
+    # installed inside body(), on the thread that does the import.
+    threading.stack_size(int(args.stack_mb * 1024 * 1024))
+    worker = threading.Thread(target=body, args=(args,), name=f"stack-{args.stack_mb:g}mb")
+    worker.start()
+    worker.join()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())