diff --git a/gpustack_runtime/detector/amd.py b/gpustack_runtime/detector/amd.py index 08551e4..7cc28d9 100644 --- a/gpustack_runtime/detector/amd.py +++ b/gpustack_runtime/detector/amd.py @@ -186,28 +186,14 @@ def detect_info(self) -> Devices | None: dev_asic_family_id = dev_gpudev_info.family_id dev_mem = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY try: dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) dev_mem = dev_gpu_vram_usage.get("vram_total") - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( - dev, - pyamdsmi.AmdSmiGpuBlock.UMC, - ) - if dev_ecc_count.get("uncorrectable_count", 0) > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY except pyamdsmi.AmdSmiException: dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pyrocmsmi.ROCMSMIError): - dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( - dev_idx, - ) - if dev_ecc_count.uncorrectable_err > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + dev_mem_status = _get_memory_status(dev, dev_idx) # The power limit is inventory, so it stays here, while the used # power the same call carries belongs to the usage query. @@ -331,20 +317,10 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: dev_mem = 0 dev_mem_used = 0 - # Health is reported by both queries, as the operator reports it - # from DetectAccelerator and MonitorAccelerator alike. - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY try: dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev) dev_mem = dev_gpu_vram_usage.get("vram_total") dev_mem_used = dev_gpu_vram_usage.get("vram_used") - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( - dev, - pyamdsmi.AmdSmiGpuBlock.UMC, - ) - if dev_ecc_count.get("uncorrectable_count", 0) > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY except pyamdsmi.AmdSmiException: dev_mem = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_total_get(dev_idx), @@ -352,13 +328,9 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: dev_mem_used = byte_to_mebibyte( # byte to MiB pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx), ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pyrocmsmi.ROCMSMIError): - dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get( - dev_idx, - ) - if dev_ecc_count.uncorrectable_err > 0: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + # Health is reported by both queries, as the operator reports it + # from DetectAccelerator and MonitorAccelerator alike. + dev_mem_status = _get_memory_status(dev, dev_idx) # A sentinel reading means the same as a failed call -- AMD SMI # cannot tell the used power -- so both route to ROCm SMI. @@ -523,6 +495,67 @@ def distance_pci_devices(bdf_a: str, bdf_b: str) -> TopologyDistanceEnum: return ret +def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum: + """ + Get a device's memory health from its uncorrected ECC error count. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. + + Args: + dev: + The AMD SMI device handle. + dev_idx: + The device index, for the ROCm SMI fallback. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + try: + dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count( + dev, + pyamdsmi.AmdSmiGpuBlock.UMC, + ) + except pyamdsmi.AmdSmiException as e: + # Fail closed: a genuine driver error marks the device unhealthy, while + # a card without the counter cannot be judged. The stub binding raises + # a codeless error when amdsmi is absent, in which case ROCm SMI is + # the fallback, as it is for the memory read. + err_code = getattr(e, "err_code", None) + if err_code is not None: + if err_code != getattr(pyamdsmi, "AMDSMI_STATUS_NOT_SUPPORTED", None): + return DeviceMemoryStatusEnum.UNHEALTHY + return DeviceMemoryStatusEnum.HEALTHY + else: + if dev_ecc_count.get("uncorrectable_count", 0) > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + return DeviceMemoryStatusEnum.HEALTHY + + try: + dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(dev_idx) + if dev_ecc_count.uncorrectable_err > 0: + return DeviceMemoryStatusEnum.UNHEALTHY + except pyrocmsmi.ROCMSMIError as e: + # rsmi_status_t comes from the ROCm-installed rsmiBindings, so it is + # looked up late: without ROCm the module has no such attribute. + rsmi_not_supported = getattr( + getattr(pyrocmsmi, "rsmi_status_t", None), + "RSMI_STATUS_NOT_SUPPORTED", + None, + ) + if e.value not in ( + pyrocmsmi.ROCMSMI_ERROR_FUNCTION_NOT_FOUND, + rsmi_not_supported, + ): + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY + + def _get_reading(dev_info: dict, key: str, default: Any = None) -> Any: """ Read one value out of an AMD SMI answer, treating its sentinel as absent. diff --git a/gpustack_runtime/detector/iluvatar.py b/gpustack_runtime/detector/iluvatar.py index 5689519..34e93ac 100644 --- a/gpustack_runtime/detector/iluvatar.py +++ b/gpustack_runtime/detector/iluvatar.py @@ -141,7 +141,6 @@ def detect_info(self) -> Devices | None: dev_cores = pyixml.nvmlDeviceGetNumGpuCores(dev) dev_mem = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY with contextlib.suppress(pyixml.NVMLError): # Prefer the v2 memory structure, falling back to v1 -- # mirrors the operator's GetMemoryInfoV, which drops the @@ -150,11 +149,7 @@ def detect_info(self) -> Devices | None: dev_mem = byte_to_mebibyte( # byte to MiB dev_mem_info.total, ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pyixml.NVMLError): - dev_health = pyixml.ixmlDeviceGetHealth(dev) - if dev_health != pyixml.IXML_HEALTH_OK: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + dev_mem_status = _get_memory_status(dev) dev_power = None with contextlib.suppress(pyixml.NVMLError): @@ -251,7 +246,6 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: dev_mem = 0 dev_mem_used = 0 - dev_mem_status = DeviceMemoryStatusEnum.HEALTHY with contextlib.suppress(pyixml.NVMLError): # Same v2-then-v1 fallback as detect_info: the operator's # MonitorAccelerator re-reads the memory info rather than @@ -263,11 +257,7 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: dev_mem_used = byte_to_mebibyte( # byte to MiB dev_mem_info.used, ) - if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: - with contextlib.suppress(pyixml.NVMLError): - dev_health = pyixml.ixmlDeviceGetHealth(dev) - if dev_health != pyixml.IXML_HEALTH_OK: - dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY + dev_mem_status = _get_memory_status(dev) dev_cores_util = None with contextlib.suppress(pyixml.NVMLError): @@ -405,3 +395,34 @@ def _get_memory_info(dev): return pyixml.nvmlDeviceGetMemoryInfo(dev, version=pyixml.nvmlMemory_v2) except pyixml.NVMLError: return pyixml.nvmlDeviceGetMemoryInfo(dev) + + +def _get_memory_status(dev) -> DeviceMemoryStatusEnum: + """ + Get a device's health from its dedicated health query. + + Both queries produce it, mirroring the operator, which reports `Unhealthy` + from `DetectAccelerator` and `MonitorAccelerator` alike. + + Args: + dev: + The device handle. + + Returns: + The memory status. + + """ + if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: + return DeviceMemoryStatusEnum.HEALTHY + + try: + dev_health = pyixml.ixmlDeviceGetHealth(dev) + if dev_health != pyixml.IXML_HEALTH_OK: + return DeviceMemoryStatusEnum.UNHEALTHY + except pyixml.NVMLError as e: + # Fail closed: a query the driver errors on marks the device + # unhealthy, while an unsupported query means it cannot be judged. + if e.value != pyixml.NVML_ERROR_NOT_SUPPORTED: + return DeviceMemoryStatusEnum.UNHEALTHY + + return DeviceMemoryStatusEnum.HEALTHY diff --git a/gpustack_runtime/detector/nvidia.py b/gpustack_runtime/detector/nvidia.py index 8ec4a51..704fa39 100644 --- a/gpustack_runtime/detector/nvidia.py +++ b/gpustack_runtime/detector/nvidia.py @@ -529,7 +529,12 @@ def _get_memory_status( memory_location: int, ) -> DeviceMemoryStatusEnum: """ - Get a device's memory health from its uncorrected ECC error counter. + Get a device's memory health. + + The verdict is the uncorrected ECC error counter plus the driver's + recovery state: a GSP failure (Xid 119/154) leaves the ECC counters + readable at zero, so the recovery action and reset status fields are + probed as well. Both queries produce it, mirroring the operator, which reports `Unhealthy` from `DetectAccelerator` and `MonitorAccelerator` alike. @@ -549,7 +554,7 @@ def _get_memory_status( if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: return DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pynvml.NVMLError): + try: dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter( dev, pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED, @@ -558,6 +563,29 @@ def _get_memory_status( ) if dev_mem_ecc_errors > 0: return DeviceMemoryStatusEnum.UNHEALTHY + except pynvml.NVMLError as e: + # Fail closed: a query the driver errors on (a wedged GSP answers + # NVML_ERROR_UNKNOWN after its RPC timeout) marks the card unhealthy, + # while an unsupported counter means the card cannot be judged. + if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED: + return DeviceMemoryStatusEnum.UNHEALTHY + + try: + dev_fields = pynvml.nvmlDeviceGetFieldValues( + dev, + fieldIds=[ + pynvml.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION, + pynvml.NVML_FI_DEV_RESET_STATUS, + ], + ) + except pynvml.NVMLError as e: + if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED: + return DeviceMemoryStatusEnum.UNHEALTHY + return DeviceMemoryStatusEnum.HEALTHY + for dev_field in dev_fields: + # A per-field error extracts to None: cannot judge, keep the verdict. + if _extract_field_value(dev_field): + return DeviceMemoryStatusEnum.UNHEALTHY return DeviceMemoryStatusEnum.HEALTHY diff --git a/gpustack_runtime/detector/thead.py b/gpustack_runtime/detector/thead.py index 483b100..211c427 100644 --- a/gpustack_runtime/detector/thead.py +++ b/gpustack_runtime/detector/thead.py @@ -508,7 +508,7 @@ def _get_memory_status( if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: return DeviceMemoryStatusEnum.HEALTHY - with contextlib.suppress(pyhgml.HGMLError): + try: dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter( dev, pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED, @@ -517,6 +517,11 @@ def _get_memory_status( ) if dev_mem_ecc_errors > 0: return DeviceMemoryStatusEnum.UNHEALTHY + except pyhgml.HGMLError as e: + # Fail closed: a query the driver errors on marks the device + # unhealthy, while an unsupported counter means it cannot be judged. + if e.value != pyhgml.HGML_ERROR_NOT_SUPPORTED: + return DeviceMemoryStatusEnum.UNHEALTHY return DeviceMemoryStatusEnum.HEALTHY diff --git a/gpustack_runtime/envs.py b/gpustack_runtime/envs.py index 3cfa266..f327411 100644 --- a/gpustack_runtime/envs.py +++ b/gpustack_runtime/envs.py @@ -46,8 +46,15 @@ """ GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: bool = True """ - Set true to disable ECC errors check during detection, + Set true to disable the health check during detection, which is used to determine the health status of the device. + + When enabled (set false), the check reads the uncorrected ECC error + counter and, on NVIDIA, the driver's recovery state (GPU reset required, + e.g. after a GSP failure); a query the driver errors on reports the device + unhealthy. Each query is a driver call per device, and against a wedged + device it can block until the driver's RPC timeout (up to 45s with GSP + firmware). """ GPUSTACK_RUNTIME_DETECT_BACKEND_MAP_RESOURCE_KEY: dict[str, str] | None = None """ diff --git a/specs/2026-08-14-detector-alignment-and-workload-exit-status.md b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md index 027bec4..b2ea05d 100644 --- a/specs/2026-08-14-detector-alignment-and-workload-exit-status.md +++ b/specs/2026-08-14-detector-alignment-and-workload-exit-status.md @@ -128,7 +128,7 @@ covered by F3). Known gaps, each to be closed or documented as a deliberate dive | Cambricon | Not a binding at all: a `cnmon info -e -m -u -j` shell-out parsing `cnmon_info.json`, with a `TODO(thxCode)` where the sample output should be. No driver version, no Neuware version, no cores, no power, no BDF, no NUMA, no health/ECC, no PCIe bus id. | New hand-written `pycndev` ctypes binding (following `pydcmi`/`pymxsml`) and a rewritten `cambricon.py` calling `GetDeviceCount`, `GetDeviceHandleByIndex`, `GetUUID`, `GetPCIeInfoV`, `GetCardName`, `GetMemoryInfoV`, `GetCardHealthStateV`, `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, `GetPowerInfo`, plus the Neuware version from `/usr/local/neuware/version.txt`. `cndev.h` in the operator's `binding/cndev` is the header of record. Four decisions taken while building `T3`, all recorded rather than assumed: **cores and the power limit stay unreported** — `pycndev` binds no core-count entry point and the operator reads neither for this vendor, so closing those two needs a binding addition of its own; **`driver_version` carries `major.minor.build`** where the operator formats only `major.minor`, since it is free precision from the same query and every other vendor here reports the driver's full string; **health comes from `cndevGetCardHealthStateV2().health` alone, with no ECC read**, because that is literally what the operator does (`memoryUnhealthy = healthInfo.Health == 0`); and **memory needs no unit conversion** — the header says MB but the operator, `cnmon` and this repo's Ascend detector all treat the value as MiB, so converting would under-report ~4.8 % against the operator on the same host, which is the very discrepancy Story 1 exists to remove. `C2` confirms the memory figure against `cnmon`. | | Cambricon | Per-device failure policy diverges from every sibling **on purpose**: `T3` follows the operator and `continue`s past a card whose required reads fail, where the other eight detectors let the error propagate. | A single faulty card therefore costs one device on Cambricon and **all** devices on the other eight. Graceful degradation is very likely the right behaviour everywhere and the other eight should move toward it, but that is a repo-wide behaviour change deserving its own task — not something to slip in per vendor. | | THead | Enumerates MIG-style GPU/compute instances, which the operator does not. | Keep — this is the appendix mechanism F2 preserves. | -| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. | +| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. With the flag off, the NVIDIA check additionally probes the driver's recovery state (`NVML_FI_DEV_GET_GPU_RECOVERY_ACTION` / `NVML_FI_DEV_RESET_STATUS`, i.e. the Xid 154 GPU-reset-required state) and treats an errored query as `unhealthy` rather than swallowing it; against a wedged card such a query can block until the driver's GSP RPC timeout (up to 45s). | Acceptance criteria: - ~~A written parity table lives in the repo (module docstring or `docs/`) listing, per vendor, the diff --git a/tests/gpustack_runtime/detector/test_amd.py b/tests/gpustack_runtime/detector/test_amd.py index 28b2c53..2b5db9e 100644 --- a/tests/gpustack_runtime/detector/test_amd.py +++ b/tests/gpustack_runtime/detector/test_amd.py @@ -56,12 +56,20 @@ class _FakeAmdSmiError(Exception): package's, which takes a status code and is absent here anyway. """ + def __init__(self, msg: str, err_code: int | None = None): + super().__init__(msg) + self.err_code = err_code + class _FakeRocmSmiError(Exception): """ The fake ROCm SMI's own error type. """ + def __init__(self, value): + super().__init__(value) + self.value = value + class _FakeAmdSmi: """ @@ -70,6 +78,7 @@ class _FakeAmdSmi: """ AmdSmiException = _FakeAmdSmiError + AMDSMI_STATUS_NOT_SUPPORTED = 2 class AmdSmiGpuBlock: UMC = 1 @@ -99,10 +108,20 @@ def amdsmi_get_gpu_driver_info(self, dev: dict) -> dict: def amdsmi_get_gpu_vram_usage(self, dev: dict) -> dict: self.calls.append("amdsmi_get_gpu_vram_usage") + if dev.get("vram_error") is not None: + msg = "VRAM usage unreadable" + raise _FakeAmdSmiError(msg, dev["vram_error"]) return dev["vram"] def amdsmi_get_gpu_ecc_count(self, dev: dict, _block: int) -> dict: self.calls.append("amdsmi_get_gpu_ecc_count") + if dev.get("ecc_error_codeless"): + # The stub binding's codeless error: amdsmi itself is absent. + msg = "amdsmi module is not installed" + raise _FakeAmdSmiError(msg) + if dev.get("ecc_error") is not None: + msg = "ECC count unreadable" + raise _FakeAmdSmiError(msg, dev["ecc_error"]) return dev["ecc"] def amdsmi_get_power_info(self, dev: dict) -> dict: @@ -130,6 +149,8 @@ class _FakeRocmSmi: """ ROCMSMIError = _FakeRocmSmiError + ROCMSMI_ERROR_FUNCTION_NOT_FOUND = -99998 + rsmi_status_t = SimpleNamespace(RSMI_STATUS_NOT_SUPPORTED=2) def __init__(self, calls: list[str], cards: list[dict]): self.calls = calls @@ -138,6 +159,24 @@ def __init__(self, calls: list[str], cards: list[dict]): def rsmi_init(self, *_args): self.calls.append("rsmi_init") + def rsmi_dev_memory_total_get(self, dev_idx: int, *_args) -> int: + self.calls.append("rsmi_dev_memory_total_get") + # ROCm SMI reports bytes where AMD SMI reports MiB. + return self.cards[dev_idx]["vram"]["vram_total"] * 1024**2 + + def rsmi_dev_memory_usage_get(self, dev_idx: int, *_args) -> int: + self.calls.append("rsmi_dev_memory_usage_get") + return self.cards[dev_idx]["vram"]["vram_used"] * 1024**2 + + def rsmi_dev_ecc_count_get(self, dev_idx: int, *_args): + self.calls.append("rsmi_dev_ecc_count_get") + error = self.cards[dev_idx].get("ecc_error_rocm") + if error is not None: + raise _FakeRocmSmiError(error) + return SimpleNamespace( + uncorrectable_err=self.cards[dev_idx]["ecc"]["uncorrectable_count"], + ) + def rsmi_dev_busy_percent_get(self, dev_idx: int) -> int: self.calls.append("rsmi_dev_busy_percent_get") return self.cards[dev_idx]["busy_percent"] @@ -282,6 +321,22 @@ def _agent(bdf: str) -> pyhsa.Agent: ) +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the device health check on: it is opt-in, as the queries cost a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK + defaults to true. A real module attribute is set because the env lookup + is cached. + """ + monkeypatch.setattr( + amd.envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + @pytest.fixture def amd_bindings(monkeypatch, tmp_path): """ @@ -453,6 +508,98 @@ def test_detect_info_issues_no_usage_call(amd_bindings): assert dev.power_used is None +# --------------------------------------------------------------------------- # +# The health check fails closed: a card the driver cannot answer is unhealthy. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.usefixtures("health_check") +def test_detect_reports_uncorrectable_ecc_errors(amd_bindings): + card = _card("0000:05:00.0", "0x00a1b2c3d4e5f600") + card["ecc"] = {"uncorrectable_count": 1} + amd_bindings([card], agents=[_agent("0000:05:00.0")]) + + dev = AMDDetector().detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_fails_closed_on_an_ecc_query_error(amd_bindings): + # A genuine driver error used to be swallowed and reported healthy. + card = _card("0000:05:00.0", "0x00a1b2c3d4e5f600") + card["ecc_error"] = 1 # AMDSMI_STATUS_INVAL: any genuine driver error + amd_bindings([card], agents=[_agent("0000:05:00.0")]) + + dev = AMDDetector().detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_tolerates_an_unsupported_ecc_query(amd_bindings): + # A card without the counter cannot be judged, not condemned. + card = _card("0000:05:00.0", "0x00a1b2c3d4e5f600") + card["ecc_error"] = _FakeAmdSmi.AMDSMI_STATUS_NOT_SUPPORTED + amd_bindings([card], agents=[_agent("0000:05:00.0")]) + + dev = AMDDetector().detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_fails_closed_on_the_rocm_smi_fallback(amd_bindings): + # With AMD SMI absent (its stub raises a codeless error), health comes + # from ROCm SMI, which follows the same policy: a genuine error marks the + # card unhealthy. + card = _card("0000:05:00.0", "0x00a1b2c3d4e5f600") + card["vram_error"] = 1 + card["ecc_error_codeless"] = True + card["ecc_error_rocm"] = 5 # a genuine ROCm SMI error + amd_bindings([card], agents=[_agent("0000:05:00.0")]) + + dev = AMDDetector().detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +@pytest.mark.usefixtures("health_check") +@pytest.mark.parametrize( + "ecc_error_rocm", + [ + 2, # RSMI_STATUS_NOT_SUPPORTED: the card has no ECC counter. + _FakeRocmSmi.ROCMSMI_ERROR_FUNCTION_NOT_FOUND, + ], +) +def test_detect_tolerates_an_unreadable_rocm_smi_ecc_query( + amd_bindings, + ecc_error_rocm, +): + card = _card("0000:05:00.0", "0x00a1b2c3d4e5f600") + card["vram_error"] = 1 + card["ecc_error_codeless"] = True + card["ecc_error_rocm"] = ecc_error_rocm + amd_bindings([card], agents=[_agent("0000:05:00.0")]) + + dev = AMDDetector().detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_issues_no_health_check_call_by_default(amd_bindings): + # The check is opt-in: at the default, detection issues no ECC call. + calls = amd_bindings( + [_card("0000:05:00.0", "0x00a1b2c3d4e5f600")], + agents=[_agent("0000:05:00.0")], + ) + + AMDDetector().detect() + + assert "amdsmi_get_gpu_ecc_count" not in calls + assert "rsmi_dev_ecc_count_get" not in calls + + # --------------------------------------------------------------------------- # # detect_usage. # # --------------------------------------------------------------------------- # diff --git a/tests/gpustack_runtime/detector/test_iluvatar.py b/tests/gpustack_runtime/detector/test_iluvatar.py index 999141a..197deb7 100644 --- a/tests/gpustack_runtime/detector/test_iluvatar.py +++ b/tests/gpustack_runtime/detector/test_iluvatar.py @@ -6,6 +6,7 @@ from gpustack_runtime.deployer.cdi import iluvatar as cdi_iluvatar from gpustack_runtime.deployer.cdi.iluvatar import IluvatarGenerator from gpustack_runtime.detector import Device, ManufacturerEnum, iluvatar +from gpustack_runtime.detector.__types__ import DeviceMemoryStatusEnum from gpustack_runtime.detector.__utils__ import byte_to_mebibyte from gpustack_runtime.detector.iluvatar import IluvatarDetector @@ -42,6 +43,10 @@ class _FakeNVMLError(Exception): Stand-in for pyixml.NVMLError. """ + def __init__(self, msg: str, value: int | None = None): + super().__init__(msg) + self.value = value + class _FakeHandle: def __init__(self, index: int): @@ -72,11 +77,20 @@ class FakePyixml: NVMLError = _FakeNVMLError IXML_HEALTH_OK = 0 + NVML_ERROR_NOT_SUPPORTED = 3 + NVML_ERROR_UNKNOWN = 999 NVML_TEMPERATURE_GPU = 0 NVML_AFFINITY_SCOPE_NODE = 0 nvmlMemory_v2 = 0x02000028 # noqa: N815 - def __init__(self, *, device_count: int = 2, v2_memory: bool = True): + def __init__( + self, + *, + device_count: int = 2, + v2_memory: bool = True, + health: int = 0, + health_error_code: int | None = None, + ): self.calls: list[str] = [] self.device_count = device_count self.v2_memory = v2_memory @@ -86,6 +100,8 @@ def __init__(self, *, device_count: int = 2, v2_memory: bool = True): self.v2_memory_used = 8 * 1024**3 self.v1_memory_total = 16 * 1024**3 self.v1_memory_used = 4 * 1024**3 + self.health = health + self.health_error_code = health_error_code # The method names below mirror pyixml's real (camelCase) API one-for-one, # so a test can monkeypatch this in as a drop-in for the module. @@ -136,7 +152,10 @@ def nvmlDeviceGetMemoryInfo(self, dev, version=None): # noqa: N802 def ixmlDeviceGetHealth(self, dev): # noqa: N802 self.calls.append("ixmlDeviceGetHealth") - return self.IXML_HEALTH_OK + if self.health_error_code is not None: + msg = "health unreadable" + raise self.NVMLError(msg, self.health_error_code) + return self.health def nvmlDeviceGetPowerManagementDefaultLimit(self, dev): # noqa: N802 self.calls.append("nvmlDeviceGetPowerManagementDefaultLimit") @@ -191,6 +210,22 @@ def _install(**kwargs) -> FakePyixml: return _install +@pytest.fixture +def health_check(monkeypatch): + """ + Turn the device health check on: it is opt-in, as the query costs a + driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK + defaults to true. A real module attribute is set because the env lookup + is cached. + """ + monkeypatch.setattr( + envs, + "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", + False, + raising=False, + ) + + # --------------------------------------------------------------------------- # # detect_info: memory V2->V1 fallback, no vgpu, no usage calls. # # --------------------------------------------------------------------------- # @@ -291,6 +326,55 @@ def test_detect_composes_info_and_usage_by_default(fake_pyixml): assert "vgpu" not in devices[0].appendix +# --------------------------------------------------------------------------- # +# The health check fails closed: a device the driver cannot answer is # +# unhealthy. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_reports_an_unhealthy_device(fake_pyixml): + fake_pyixml(health=1) + + devices = IluvatarDetector().detect_info() + + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.UNHEALTHY, + ] * 2 + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_fails_closed_on_a_health_query_error(fake_pyixml): + fake_pyixml(health_error_code=FakePyixml.NVML_ERROR_UNKNOWN) + + devices = IluvatarDetector().detect_info() + + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.UNHEALTHY, + ] * 2 + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_tolerates_an_unsupported_health_query(fake_pyixml): + # A device without the health query cannot be judged, not condemned. + fake_pyixml(health_error_code=FakePyixml.NVML_ERROR_NOT_SUPPORTED) + + devices = IluvatarDetector().detect_info() + + assert [dev.memory_status for dev in devices] == [ + DeviceMemoryStatusEnum.HEALTHY, + ] * 2 + + +def test_detect_issues_no_health_check_call_by_default(fake_pyixml): + # The check is opt-in: at the default, detection issues no health call. + fake = fake_pyixml() + + IluvatarDetector().detect() + + assert "ixmlDeviceGetHealth" not in fake.calls + + # --------------------------------------------------------------------------- # # CDI: /dev/iluvatar{N} still reads the appendix minor number. # # --------------------------------------------------------------------------- # diff --git a/tests/gpustack_runtime/detector/test_nvidia.py b/tests/gpustack_runtime/detector/test_nvidia.py index 1562e34..208e80a 100644 --- a/tests/gpustack_runtime/detector/test_nvidia.py +++ b/tests/gpustack_runtime/detector/test_nvidia.py @@ -38,6 +38,10 @@ class _NVMLError(Exception): The fake binding's error type, standing in for pynvml.NVMLError. """ + def __init__(self, msg: str, value: int | None = None): + super().__init__(msg) + self.value = value + class _FakeFabricInfo(ctypes.Structure): """ @@ -97,6 +101,19 @@ class _FakeDevice: memory_bus_width: int | None = 192 ecc_mode: int = 1 # NVML_FEATURE_ENABLED ecc_errors: int = 0 + ecc_counter_error_code: int | None = None + """ + The error code the ECC counter query raises, or None for a readable one. + """ + recovery_action: int | None = 0 + reset_status: int | None = 0 + """ + The recovery field values, or None for a field the driver will not answer. + """ + field_values_error_code: int | None = None + """ + The error code the field-values query raises, or None for an answered one. + """ temperature: int = 47 power_limit: int = 72_000 # mW power_used: int = 30_000 # mW @@ -120,6 +137,16 @@ class _FakeNVML: NVMLError = _NVMLError NVML_SUCCESS = 0 NVML_ERROR_NOT_SUPPORTED = 3 + NVML_ERROR_UNKNOWN = 999 + NVML_VALUE_TYPE_DOUBLE = 0 + NVML_VALUE_TYPE_UNSIGNED_INT = 1 + NVML_VALUE_TYPE_UNSIGNED_LONG = 2 + NVML_VALUE_TYPE_UNSIGNED_LONG_LONG = 3 + NVML_VALUE_TYPE_SIGNED_LONG_LONG = 4 + NVML_VALUE_TYPE_SIGNED_INT = 5 + NVML_VALUE_TYPE_UNSIGNED_SHORT = 6 + NVML_FI_DEV_RESET_STATUS = 226 + NVML_FI_DEV_GET_GPU_RECOVERY_ACTION = 230 NVML_FEATURE_DISABLED = 0 NVML_FEATURE_ENABLED = 1 NVML_TEMPERATURE_GPU = 0 @@ -261,8 +288,41 @@ def nvmlDeviceGetEccMode(self, handle): # noqa: N802 def nvmlDeviceGetMemoryErrorCounter(self, handle, error_type, scope, location): # noqa: N802 self.calls.append("nvmlDeviceGetMemoryErrorCounter") + error_code = getattr(handle, "ecc_counter_error_code", None) + if error_code is not None: + msg = "ECC counter unreadable" + raise _NVMLError(msg, error_code) return handle.ecc_errors + def nvmlDeviceGetFieldValues(self, handle, fieldIds): # noqa: N802, N803 + self.calls.append("nvmlDeviceGetFieldValues") + error_code = getattr(handle, "field_values_error_code", None) + if error_code is not None: + msg = "field values unreadable" + raise _NVMLError(msg, error_code) + values = { + self.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION: getattr( + handle, + "recovery_action", + None, + ), + self.NVML_FI_DEV_RESET_STATUS: getattr(handle, "reset_status", None), + } + return [ + self._field_value(field_id, values.get(field_id)) for field_id in fieldIds + ] + + @staticmethod + def _field_value(field_id, value): + if value is None: + return SimpleNamespace(nvmlReturn=_FakeNVML.NVML_ERROR_NOT_SUPPORTED) + return SimpleNamespace( + fieldId=field_id, + nvmlReturn=_FakeNVML.NVML_SUCCESS, + valueType=_FakeNVML.NVML_VALUE_TYPE_UNSIGNED_INT, + value=SimpleNamespace(uiVal=value), + ) + # Usage. def nvmlDeviceGetUtilizationRates(self, handle): # noqa: N802 @@ -383,7 +443,7 @@ def nvmlGpuInstanceGetComputeInstanceProfileInfo( # noqa: N802 @pytest.fixture def health_check(monkeypatch): """ - Turn the ECC error check on: it is opt-in, as reading the counters costs a + Turn the device health check on: it is opt-in, as the queries cost a driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults to true. A real module attribute is set because the env lookup is cached. """ @@ -505,6 +565,83 @@ def test_detect_info_reports_the_memory_health(fake_nvml): assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY +# --------------------------------------------------------------------------- # +# The health check fails closed: a card the driver cannot answer is unhealthy. # +# --------------------------------------------------------------------------- # + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_fails_closed_on_an_ecc_query_error(fake_nvml): + # A wedged GSP answers the ECC query with an error after its RPC timeout + # (Xid 119); swallowing that error reported the card healthy. + fake_nvml([_FakeDevice(ecc_counter_error_code=_FakeNVML.NVML_ERROR_UNKNOWN)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_tolerates_an_unsupported_ecc_query(fake_nvml): + # A card without the counter cannot be judged, not condemned. + fake_nvml([_FakeDevice(ecc_counter_error_code=_FakeNVML.NVML_ERROR_NOT_SUPPORTED)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +@pytest.mark.usefixtures("health_check") +@pytest.mark.parametrize( + "knobs", + [ + {"recovery_action": 1}, # Xid 154: GPU Reset Required. + {"reset_status": 1}, # A pending/past reset. + ], +) +def test_detect_info_reports_a_card_awaiting_reset(fake_nvml, knobs): + # The recovery state survives a GSP failure that leaves the ECC counters + # readable at zero, so the check probes it directly. + fake_nvml([_FakeDevice(**knobs)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_falls_back_to_ecc_when_the_recovery_fields_are_unreadable( + fake_nvml, +): + # An old driver or card answers the fields with an error apiece: cannot + # judge, so the ECC verdict stands. + fake_nvml([_FakeDevice(recovery_action=None, reset_status=None)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.HEALTHY + + +@pytest.mark.usefixtures("health_check") +def test_detect_info_fails_closed_on_a_field_values_error(fake_nvml): + fake_nvml([_FakeDevice(field_values_error_code=_FakeNVML.NVML_ERROR_UNKNOWN)]) + + devices = NVIDIADetector().detect_info() + + assert devices[0].memory_status == DeviceMemoryStatusEnum.UNHEALTHY + + +def test_detect_issues_no_health_check_call_by_default(fake_nvml): + # The check is opt-in: at the default, detection issues no ECC or + # field-values call. + fake = fake_nvml([_FakeDevice()]) + + NVIDIADetector().detect() + + assert "nvmlDeviceGetMemoryErrorCounter" not in fake.calls + assert "nvmlDeviceGetFieldValues" not in fake.calls + + # --------------------------------------------------------------------------- # # Memory is what the card can allocate, not its ECC-restored capacity. # # --------------------------------------------------------------------------- # diff --git a/tests/gpustack_runtime/detector/test_thead.py b/tests/gpustack_runtime/detector/test_thead.py index 0a27761..f94aaf8 100644 --- a/tests/gpustack_runtime/detector/test_thead.py +++ b/tests/gpustack_runtime/detector/test_thead.py @@ -59,6 +59,10 @@ class _Instance: memory_total: int = 8192 * _MIB memory_used: int = 512 * _MIB ecc_errors: int = 0 + ecc_counter_error_code: int | None = None + """ + The error code the ECC counter query raises, or None for a readable one. + """ sm_util: float | None = 60.0 """ The SM utilization GPM samples for the instance, or None when unreadable. @@ -85,6 +89,10 @@ class _Card: memory_total: int = 65536 * _MIB memory_used: int = 1024 * _MIB ecc_errors: int = 0 + ecc_counter_error_code: int | None = None + """ + The error code the ECC counter query raises, or None for a readable one. + """ cores_utilization: int = 33 temperature: int = 55 power_limit: int = 350_000 # mW @@ -254,6 +262,8 @@ def _device_get_memory_error_counter( counter_type: int, location_type: int, ) -> int: + if handle.ecc_counter_error_code is not None: + raise pyhgml.HGMLError(handle.ecc_counter_error_code) return handle.ecc_errors # Usage. @@ -406,7 +416,7 @@ def _gi_get_ci_profile_info( @pytest.fixture def health_check(monkeypatch): """ - Turn the ECC error check on: it is opt-in, as reading the counters costs a + Turn the device health check on: it is opt-in, as the queries cost a driver call per device, so GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK defaults to true. A real module attribute is set because the env lookup is cached. """ @@ -735,6 +745,56 @@ def test_detect_reports_an_uncorrectable_ecc_error(detector): ) +@pytest.mark.usefixtures("health_check") +def test_detect_fails_closed_on_an_ecc_query_error(detector): + # A card the driver refuses to answer is unhealthy, not healthy: the query + # error used to be swallowed. + det, _ = detector( + _Card( + uuid="PPU-0", + ecc_counter_error_code=pyhgml.HGML_ERROR_UNKNOWN, + instances=[ + _Instance( + uuid="PPU-0-MIG-0", + ecc_counter_error_code=pyhgml.HGML_ERROR_UNKNOWN, + ), + ], + ), + ) + + dev = det.detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.UNHEALTHY + assert ( + dev.appendix["mig_devices"][0]["memory_status"] + == DeviceMemoryStatusEnum.UNHEALTHY + ) + + +@pytest.mark.usefixtures("health_check") +def test_detect_tolerates_an_unsupported_ecc_query(detector): + # A card without the counter cannot be judged, not condemned. + det, _ = detector( + _Card( + uuid="PPU-0", + ecc_counter_error_code=pyhgml.HGML_ERROR_NOT_SUPPORTED, + ), + ) + + dev = det.detect()[0] + + assert dev.memory_status == DeviceMemoryStatusEnum.HEALTHY + + +def test_detect_issues_no_health_check_call_by_default(detector): + # The check is opt-in: at the default, detection issues no ECC call. + det, fake = detector(_Card(uuid="PPU-0")) + + det.detect() + + assert "hgmlDeviceGetMemoryErrorCounter" not in fake.calls + + # --------------------------------------------------------------------------- # # CDI: /dev/alixpu_ppu{N} is named after the enumeration index. # # --------------------------------------------------------------------------- #