fix(detector): fail closed on GPU health check errors

gpustack_runtime/detector/amd.py

@@ -186,28 +186,14 @@ def detect_info(self) -> Devices | None:
                             dev_asic_family_id = dev_gpudev_info.family_id
 
                 dev_mem = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 try:
                     dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
                     dev_mem = dev_gpu_vram_usage.get("vram_total")
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
-                            dev,
-                            pyamdsmi.AmdSmiGpuBlock.UMC,
-                        )
-                        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
                 except pyamdsmi.AmdSmiException:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
                     )
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                            dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
-                                dev_idx,
-                            )
-                            if dev_ecc_count.uncorrectable_err > 0:
-                                dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev, dev_idx)
 
                 # The power limit is inventory, so it stays here, while the used
                 # power the same call carries belongs to the usage query.
@@ -331,34 +317,20 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None:
 
                 dev_mem = 0
                 dev_mem_used = 0
-                # Health is reported by both queries, as the operator reports it
-                # from DetectAccelerator and MonitorAccelerator alike.
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 try:
                     dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
                     dev_mem = dev_gpu_vram_usage.get("vram_total")
                     dev_mem_used = dev_gpu_vram_usage.get("vram_used")
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
-                            dev,
-                            pyamdsmi.AmdSmiGpuBlock.UMC,
-                        )
-                        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
                 except pyamdsmi.AmdSmiException:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
                     )
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
                     )
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                            dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
-                                dev_idx,
-                            )
-                            if dev_ecc_count.uncorrectable_err > 0:
-                                dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                # Health is reported by both queries, as the operator reports it
+                # from DetectAccelerator and MonitorAccelerator alike.
+                dev_mem_status = _get_memory_status(dev, dev_idx)
 
                 # A sentinel reading means the same as a failed call -- AMD SMI
                 # cannot tell the used power -- so both route to ROCm SMI.
@@ -523,6 +495,67 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum:
         return ret
 
 
+def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum:
+    """
+    Get a device's memory health from its uncorrected ECC error count.
+
+    Both queries produce it, mirroring the operator, which reports `Unhealthy`
+    from `DetectAccelerator` and `MonitorAccelerator` alike.
+
+    Args:
+        dev:
+            The AMD SMI device handle.
+        dev_idx:
+            The device index, for the ROCm SMI fallback.
+
+    Returns:
+        The memory status.
+
+    """
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
+            dev,
+            pyamdsmi.AmdSmiGpuBlock.UMC,
+        )
+    except pyamdsmi.AmdSmiException as e:
+        # Fail closed: a genuine driver error marks the device unhealthy, while
+        # a card without the counter cannot be judged. The stub binding raises
+        # a codeless error when amdsmi is absent, in which case ROCm SMI is
+        # the fallback, as it is for the memory read.
+        err_code = getattr(e, "err_code", None)
+        if err_code is not None:
+            if err_code != getattr(pyamdsmi, "AMDSMI_STATUS_NOT_SUPPORTED", None):
+                return DeviceMemoryStatusEnum.UNHEALTHY
+            return DeviceMemoryStatusEnum.HEALTHY
+    else:
+        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(dev_idx)
+        if dev_ecc_count.uncorrectable_err > 0:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyrocmsmi.ROCMSMIError as e:
+        # rsmi_status_t comes from the ROCm-installed rsmiBindings, so it is
+        # looked up late: without ROCm the module has no such attribute.
+        rsmi_not_supported = getattr(
+            getattr(pyrocmsmi, "rsmi_status_t", None),
+            "RSMI_STATUS_NOT_SUPPORTED",
+            None,
+        )
+        if e.value not in (
+            pyrocmsmi.ROCMSMI_ERROR_FUNCTION_NOT_FOUND,
+            rsmi_not_supported,
+        ):
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    return DeviceMemoryStatusEnum.HEALTHY
+
+
 def _get_reading(dev_info: dict, key: str, default: Any = None) -> Any:
     """
     Read one value out of an AMD SMI answer, treating its sentinel as absent.

gpustack_runtime/detector/iluvatar.py

@@ -141,7 +141,6 @@ def detect_info(self) -> Devices | None:
                     dev_cores = pyixml.nvmlDeviceGetNumGpuCores(dev)
 
                 dev_mem = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with contextlib.suppress(pyixml.NVMLError):
                     # Prefer the v2 memory structure, falling back to v1 --
                     # mirrors the operator's GetMemoryInfoV, which drops the
@@ -150,11 +149,7 @@ def detect_info(self) -> Devices | None:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         dev_mem_info.total,
                     )
-                if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                    with contextlib.suppress(pyixml.NVMLError):
-                        dev_health = pyixml.ixmlDeviceGetHealth(dev)
-                        if dev_health != pyixml.IXML_HEALTH_OK:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev)
 
                 dev_power = None
                 with contextlib.suppress(pyixml.NVMLError):
@@ -251,7 +246,6 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None:
 
                 dev_mem = 0
                 dev_mem_used = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with contextlib.suppress(pyixml.NVMLError):
                     # Same v2-then-v1 fallback as detect_info: the operator's
                     # MonitorAccelerator re-reads the memory info rather than
@@ -263,11 +257,7 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None:
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         dev_mem_info.used,
                     )
-                if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                    with contextlib.suppress(pyixml.NVMLError):
-                        dev_health = pyixml.ixmlDeviceGetHealth(dev)
-                        if dev_health != pyixml.IXML_HEALTH_OK:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev)
 
                 dev_cores_util = None
                 with contextlib.suppress(pyixml.NVMLError):
@@ -405,3 +395,34 @@ def _get_memory_info(dev):
         return pyixml.nvmlDeviceGetMemoryInfo(dev, version=pyixml.nvmlMemory_v2)
     except pyixml.NVMLError:
         return pyixml.nvmlDeviceGetMemoryInfo(dev)
+
+
+def _get_memory_status(dev) -> DeviceMemoryStatusEnum:
+    """
+    Get a device's health from its dedicated health query.
+
+    Both queries produce it, mirroring the operator, which reports `Unhealthy`
+    from `DetectAccelerator` and `MonitorAccelerator` alike.
+
+    Args:
+        dev:
+            The device handle.
+
+    Returns:
+        The memory status.
+
+    """
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_health = pyixml.ixmlDeviceGetHealth(dev)
+        if dev_health != pyixml.IXML_HEALTH_OK:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyixml.NVMLError as e:
+        # Fail closed: a query the driver errors on marks the device
+        # unhealthy, while an unsupported query means it cannot be judged.
+        if e.value != pyixml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    return DeviceMemoryStatusEnum.HEALTHY

gpustack_runtime/detector/nvidia.py

@@ -529,7 +529,12 @@ def _get_memory_status(
     memory_location: int,
 ) -> DeviceMemoryStatusEnum:
     """
-    Get a device's memory health from its uncorrected ECC error counter.
+    Get a device's memory health.
+
+    The verdict is the uncorrected ECC error counter plus the driver's
+    recovery state: a GSP failure (Xid 119/154) leaves the ECC counters
+    readable at zero, so the recovery action and reset status fields are
+    probed as well.
 
     Both queries produce it, mirroring the operator, which reports `Unhealthy`
     from `DetectAccelerator` and `MonitorAccelerator` alike.
@@ -549,7 +554,7 @@ def _get_memory_status(
     if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
         return DeviceMemoryStatusEnum.HEALTHY
 
-    with contextlib.suppress(pynvml.NVMLError):
+    try:
         dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter(
             dev,
             pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED,
@@ -558,6 +563,29 @@ def _get_memory_status(
         )
         if dev_mem_ecc_errors > 0:
             return DeviceMemoryStatusEnum.UNHEALTHY
+    except pynvml.NVMLError as e:
+        # Fail closed: a query the driver errors on (a wedged GSP answers
+        # NVML_ERROR_UNKNOWN after its RPC timeout) marks the card unhealthy,
+        # while an unsupported counter means the card cannot be judged.
+        if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    try:
+        dev_fields = pynvml.nvmlDeviceGetFieldValues(
+            dev,
+            fieldIds=[
+                pynvml.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION,
+                pynvml.NVML_FI_DEV_RESET_STATUS,
+            ],
+        )
+    except pynvml.NVMLError as e:
+        if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+        return DeviceMemoryStatusEnum.HEALTHY
+    for dev_field in dev_fields:
+        # A per-field error extracts to None: cannot judge, keep the verdict.
+        if _extract_field_value(dev_field):
+            return DeviceMemoryStatusEnum.UNHEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY
 

gpustack_runtime/detector/thead.py

@@ -508,7 +508,7 @@ def _get_memory_status(
     if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
         return DeviceMemoryStatusEnum.HEALTHY
 
-    with contextlib.suppress(pyhgml.HGMLError):
+    try:
         dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter(
             dev,
             pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED,
@@ -517,6 +517,11 @@ def _get_memory_status(
         )
         if dev_mem_ecc_errors > 0:
             return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyhgml.HGMLError as e:
+        # Fail closed: a query the driver errors on marks the device
+        # unhealthy, while an unsupported counter means it cannot be judged.
+        if e.value != pyhgml.HGML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY
 

gpustack_runtime/envs.py

@@ -46,8 +46,15 @@
     """
     GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: bool = True
     """
-    Set true to disable ECC errors check during detection,
+    Set true to disable the health check during detection,
     which is used to determine the health status of the device.
+
+    When enabled (set false), the check reads the uncorrected ECC error
+    counter and, on NVIDIA, the driver's recovery state (GPU reset required,
+    e.g. after a GSP failure); a query the driver errors on reports the device
+    unhealthy. Each query is a driver call per device, and against a wedged
+    device it can block until the driver's RPC timeout (up to 45s with GSP
+    firmware).
     """
     GPUSTACK_RUNTIME_DETECT_BACKEND_MAP_RESOURCE_KEY: dict[str, str] | None = None
     """

specs/2026-08-14-detector-alignment-and-workload-exit-status.md

@@ -128,7 +128,7 @@ covered by F3). Known gaps, each to be closed or documented as a deliberate dive
 | Cambricon | Not a binding at all: a `cnmon info -e -m -u -j` shell-out parsing `cnmon_info.json`, with a `TODO(thxCode)` where the sample output should be. No driver version, no Neuware version, no cores, no power, no BDF, no NUMA, no health/ECC, no PCIe bus id. | New hand-written `pycndev` ctypes binding (following `pydcmi`/`pymxsml`) and a rewritten `cambricon.py` calling `GetDeviceCount`, `GetDeviceHandleByIndex`, `GetUUID`, `GetPCIeInfoV`, `GetCardName`, `GetMemoryInfoV`, `GetCardHealthStateV`, `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, `GetPowerInfo`, plus the Neuware version from `/usr/local/neuware/version.txt`. `cndev.h` in the operator's `binding/cndev` is the header of record. Four decisions taken while building `T3`, all recorded rather than assumed: **cores and the power limit stay unreported** — `pycndev` binds no core-count entry point and the operator reads neither for this vendor, so closing those two needs a binding addition of its own; **`driver_version` carries `major.minor.build`** where the operator formats only `major.minor`, since it is free precision from the same query and every other vendor here reports the driver's full string; **health comes from `cndevGetCardHealthStateV2().health` alone, with no ECC read**, because that is literally what the operator does (`memoryUnhealthy = healthInfo.Health == 0`); and **memory needs no unit conversion** — the header says MB but the operator, `cnmon` and this repo's Ascend detector all treat the value as MiB, so converting would under-report ~4.8 % against the operator on the same host, which is the very discrepancy Story 1 exists to remove. `C2` confirms the memory figure against `cnmon`. |
 | Cambricon | Per-device failure policy diverges from every sibling **on purpose**: `T3` follows the operator and `continue`s past a card whose required reads fail, where the other eight detectors let the error propagate. | A single faulty card therefore costs one device on Cambricon and **all** devices on the other eight. Graceful degradation is very likely the right behaviour everywhere and the other eight should move toward it, but that is a repo-wide behaviour change deserving its own task — not something to slip in per vendor. |
 | THead | Enumerates MIG-style GPU/compute instances, which the operator does not. | Keep — this is the appendix mechanism F2 preserves. |
-| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. |
+| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. With the flag off, the NVIDIA check additionally probes the driver's recovery state (`NVML_FI_DEV_GET_GPU_RECOVERY_ACTION` / `NVML_FI_DEV_RESET_STATUS`, i.e. the Xid 154 GPU-reset-required state) and treats an errored query as `unhealthy` rather than swallowing it; against a wedged card such a query can block until the driver's GSP RPC timeout (up to 45s). |
 
 Acceptance criteria:
 - ~~A written parity table lives in the repo (module docstring or `docs/`) listing, per vendor, the