diff --git a/gpustack_runtime/detector/hygon.py b/gpustack_runtime/detector/hygon.py index fd704b5..68eadb5 100644 --- a/gpustack_runtime/detector/hygon.py +++ b/gpustack_runtime/detector/hygon.py @@ -7,7 +7,7 @@ from .. import envs from ..logging import debug_log_exception, debug_log_warning -from . import Topology, pyamdgpu, pyhsa, pyrocmsmi +from . import Topology, pyamdgpu, pydmi, pyhsa, pyrocmsmi from .__types__ import ( Detector, Device, @@ -15,6 +15,7 @@ Devices, ManufacturerEnum, TopologyDistanceEnum, + index_mig_devices, merge_devices_usage, ) from .__utils__ import ( @@ -31,6 +32,22 @@ logger = logging.getLogger(__name__) +_DMI_MIG_CONFIG_DIR = Path("/etc/dmi_mig_config") +""" +The vendor's registry of live MIG instances: the driver writes one +devgici.conf per compute instance under ci/ the moment it is +created, and the conf's mig_uuid is the identity a workload container binds +to (DMI_MIG_VISIBLE_DEVICE=MIG-). A variable so tests can point at a +fixture. +""" + +_MIG_UUID_PREFIX = "MIG-" +""" +What the vendor's own tooling prefixes an instance UUID with, in its listing +and in the DMI_MIG_VISIBLE_DEVICE value; reported identities carry it for the +same reason. +""" + class HygonDetector(Detector): """ @@ -102,6 +119,24 @@ def detect_info(self) -> Devices | None: pyrocmsmi.rsmi_init() + # The MIG mode is a property of the node, not of a card, so it is + # read once here; a missing library or an unreadable mode means + # physical-only detection, exactly as without this branch. + sys_mig_enabled = False + try: + pydmi.dmiInit() + sys_mig_current, _ = pydmi.dmiGetSystemMigMode() + sys_mig_enabled = sys_mig_current == pydmi.DMI_DEVICE_MIG_ENABLE + except pydmi.DMIError: + debug_log_exception(logger, "Failed to query the node MIG mode") + + # MIG devices of every MIG-enabled card, keyed by the card's + # enumeration index, and the largest number of MIG devices a card + # can host: both are needed to number them once every card is + # detected, see index_mig_devices. + devs_mig_devices: dict[int, list[dict]] = {} + devs_mig_slots = 0 + sys_driver_ver = None for path in [ Path("/sys/module/hycu/version"), @@ -202,6 +237,44 @@ def detect_info(self) -> Devices | None: if dev_renderd_id is not None: dev_appendix["renderd_id"] = dev_renderd_id + if sys_mig_enabled: + try: + dev_dmi = pydmi.dmiDeviceGetHandleByPciBusId(dev_bdf) + dev_dmi_index = pydmi.dmiDeviceGetIndex(dev_dmi) + except pydmi.DMIError: + debug_log_exception( + logger, + "Failed to reach device %s through the DMI library", + dev_bdf, + ) + else: + dev_appendix["mig"] = True + dev_mig_slots = 0 + with contextlib.suppress(pydmi.DMIError): + dev_mig_slots = pydmi.dmiDeviceGetMaxMigDeviceCount( + dev_dmi, + ) + devs_mig_slots = max(devs_mig_slots, dev_mig_slots) + # With MIG enabled HSA exposes a partition's view of + # the card, so dev_cores can be one slice's count or + # nothing; the profiles always reach the whole card. + dev_cores = _get_mig_physical_cores(dev_dmi) or dev_cores + dev_mig_devices = _get_mig_devices( + dev_dmi, + dev_dmi_index, + dev_mig_slots, + sys_driver_ver, + sys_runtime_ver, + sys_runtime_ver_original, + dev_cc, + dev_mem_status, + dev_power, + dev_bdf, + dev_numa, + ) + dev_appendix["mig_devices"] = dev_mig_devices + devs_mig_devices[dev_index] = dev_mig_devices + ret.append( Device( manufacturer=self.manufacturer, @@ -219,6 +292,8 @@ def detect_info(self) -> Devices | None: appendix=dev_appendix, ), ) + + index_mig_devices(ret, devs_mig_devices, devs_mig_slots) except pyrocmsmi.ROCMSMIError: debug_log_exception(logger, "Failed to fetch devices") raise @@ -234,7 +309,8 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: Args: devices: - The devices to refresh, matched by UUID. + The devices to refresh, matched by UUID, MIG entries in + ``appendix["mig_devices"]`` included. If None, detects the devices' information first. Returns: @@ -258,6 +334,17 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: try: pyrocmsmi.rsmi_init() + # The MIG mode is a property of the node, not of a card, so it is + # read once here; a missing library or an unreadable mode means + # physical-only usage, exactly as without this branch. + sys_mig_enabled = False + try: + pydmi.dmiInit() + sys_mig_current, _ = pydmi.dmiGetSystemMigMode() + sys_mig_enabled = sys_mig_current == pydmi.DMI_DEVICE_MIG_ENABLE + except pydmi.DMIError: + debug_log_exception(logger, "Failed to query the node MIG mode") + devs_count = pyrocmsmi.rsmi_num_monitor_devices() for dev_idx in range(devs_count): dev_uuid = f"GPU-{pyrocmsmi.rsmi_dev_unique_id_get(dev_idx)[2:]}" @@ -302,6 +389,34 @@ def detect_usage(self, devices: Devices | None = None) -> Devices | None: power_used=dev_power_used, ), ) + + if sys_mig_enabled: + dev_bdf = pyrocmsmi.rsmi_dev_pci_id_get(dev_idx) + try: + dev_dmi = pydmi.dmiDeviceGetHandleByPciBusId(dev_bdf) + dev_dmi_index = pydmi.dmiDeviceGetIndex(dev_dmi) + except pydmi.DMIError: + debug_log_exception( + logger, + "Failed to reach device %s through the DMI library", + dev_bdf, + ) + else: + dev_mig_slots = 0 + with contextlib.suppress(pydmi.DMIError): + dev_mig_slots = pydmi.dmiDeviceGetMaxMigDeviceCount( + dev_dmi, + ) + usages.extend( + _get_mig_usages( + dev_dmi, + dev_dmi_index, + dev_mig_slots, + dev_bdf, + dev_temp, + dev_power_used, + ), + ) except pyrocmsmi.ROCMSMIError: debug_log_exception(logger, "Failed to fetch devices usage") raise @@ -446,3 +561,303 @@ def _get_card_and_renderd_id(dev_bdf: str) -> tuple[int | None, int | None]: break return card_id, renderd_id + + +def _get_mig_physical_cores(dev_dmi) -> int | None: + """ + Derive the card's full core count from its GPU-instance profiles. + + A profile's CU count times its instance capacity always spans the whole + card (e.g. 20x4, 40x2 and 80x1 all reach the C-3000's 80), which HSA's + partition view does not. None when no profile answers. + """ + ret = None + for profile in range(pydmi.DMI_GPU_INSTANCE_PROFILE_COUNT): + with contextlib.suppress(pydmi.DMIError): + gi_prf = pydmi.dmiDeviceGetGpuInstanceProfileInfo(dev_dmi, profile) + cores = gi_prf.cu_count * gi_prf.gi_count_max + ret = max(ret, cores) if ret is not None else cores + return ret + + +def _iter_mig_device_handles(dev_dmi, dev_mig_slots: int) -> list: + """ + Sweep the node-global MIG device index space for this card's MIG devices. + + The index is not per-card, despite the query taking a card: it numbers + every MIG device on the node, and an index belonging to another card + answers NOT_FOUND. The sweep is bounded by the node's own capacity -- + device count times per-card maximum -- and stops early once the card's own + maximum has been found. A gap or an unreadable index is skipped per index, + never aborting the sweep. + + Args: + dev_dmi: + The DMI handle of the card. + dev_mig_slots: + The number of MIG devices the card can host. + + Returns: + The card's MIG device handles, in index order. + + """ + if not dev_mig_slots: + return [] + + ret = [] + try: + cards = pydmi.dmiDeviceGetCount() + except pydmi.DMIError: + debug_log_exception(logger, "Failed to get the DMI device count") + return ret + + for mdev_gidx in range(dev_mig_slots * cards): + if len(ret) >= dev_mig_slots: + break + try: + mdev = pydmi.dmiDeviceGetMigDeviceHandleByIndex(dev_dmi, mdev_gidx) + except pydmi.DMIError as e: + if e.value not in ( + pydmi.DMI_ERROR_NOT_SUPPORTED, + pydmi.DMI_ERROR_NOT_FOUND, + pydmi.DMI_ERROR_INVALID_ARGUMENT, + ): + debug_log_exception( + logger, + "Failed to get the MIG device at index %d", + mdev_gidx, + ) + continue + ret.append(mdev) + return ret + + +def _get_mig_devices( + dev_dmi, + dev_dmi_index: int, + dev_mig_slots: int, + sys_driver_ver, + sys_runtime_ver, + sys_runtime_ver_original, + dev_cc, + dev_mem_status, + dev_power, + dev_bdf: str, + dev_numa, +) -> list[dict]: + """ + Enumerate the card's current MIG devices with the same inventory detail a + plain device carries, returned as appendix entries of the physical card + rather than standalone devices. Empty when MIG is enabled but no instances + exist yet. + + An entry keeps a Device's shape, so the fields the usage query owns are + present at a Device's defaults: `_get_mig_usages` fills them. + + Each entry's `index` is the per-card discovery ordinal: index_mig_devices + turns it into the device index once every card is detected. + """ + ret: list[dict] = [] + for mdev_ordinal, mdev in enumerate( + _iter_mig_device_handles(dev_dmi, dev_mig_slots), + ): + # Suppressed per instance, not per card: one instance refusing a read + # must not vanish every later instance from the inventory. + with contextlib.suppress(pydmi.DMIError): + mdev_gi_id = pydmi.dmiDeviceGetGpuInstanceId(mdev) + mdev_ci_id = pydmi.dmiDeviceGetComputeInstanceId(mdev) + + mdev_gi = pydmi.dmiDeviceGetGpuInstanceById(dev_dmi, mdev_gi_id) + mdev_gi_info = pydmi.dmiGpuInstanceGetInfo(mdev_gi) + if mdev_gi_info.device != dev_dmi.value: + # The index space is node-global; attribute strictly by the GI + # info's parent device, never by index ranges. + continue + + # The library offers no UUID getter, so the identity comes from + # the vendor's registry, as the operator's does. + mdev_uuid = _get_mig_device_uuid( + dev_dmi_index, + mdev_gi_id, + mdev_ci_id, + dev_bdf, + ) + + # The profile carries the name, the memory and the CU count; find + # it by sweeping the fixed slice-count space for the profile id + # the instance reports. + mdev_name = "" + mdev_mem = None + mdev_cores = None + for width in range(1, pydmi.DMI_GPU_INSTANCE_PROFILE_COUNT + 1): + with contextlib.suppress(pydmi.DMIError): + gi_prf = pydmi.dmiDeviceGetGpuInstanceProfileInfo( + dev_dmi, + width - 1, + ) + if gi_prf.id != mdev_gi_info.profile_id: + continue + # memory_size_MB carries MiB despite the name. + mdev_mem = gi_prf.memory_size_MB + mdev_cores = gi_prf.cu_count + mdev_name = gi_prf.name.decode(errors="replace").removeprefix( + "MIG ", + ) + break + + mdev_appendix = { + "sliced": True, + "mig": True, + "bdf": dev_bdf, + "gpu_instance_id": mdev_gi_id, + "compute_instance_id": mdev_ci_id, + # The GPU instance's placement, in GPU-slice units; the + # compute instance's own placement sits inside it. + "placement": { + "start": mdev_gi_info.placement.start, + "length": mdev_gi_info.placement.size, + }, + } + if dev_numa: + mdev_appendix["numa"] = dev_numa + + ret.append( + { + # The discovery ordinal, independent of skipped reads: + # a failed instance ahead must not renumber this one. + "index": mdev_ordinal, + "name": mdev_name, + "uuid": mdev_uuid, + "driver_version": sys_driver_ver, + "runtime_version": sys_runtime_ver, + "runtime_version_original": sys_runtime_ver_original, + "compute_capability": dev_cc, + "cores": mdev_cores, + "cores_utilization": 0, + "memory": mdev_mem, + "memory_used": 0, + "memory_utilization": 0, + # A partition shares its card's memory-health verdict. + "memory_status": dev_mem_status, + "temperature": None, + "power": dev_power, + "power_used": None, + "appendix": mdev_appendix, + }, + ) + return ret + + +def _get_mig_usages( + dev_dmi, + dev_dmi_index: int, + dev_mig_slots: int, + dev_bdf: str, + dev_temp, + dev_power_used, +) -> Devices: + """ + Fetch the usage of the card's current MIG devices, one UUID-keyed entry per + instance, to merge into the card's `appendix["mig_devices"]`. + + Memory and utilization are read through each instance's own MIG device + handle, which reports the partition's figures rather than the card's. + + Args: + dev_dmi: + The DMI handle of the card hosting them. + dev_dmi_index: + The card's DMI enumeration index, used to resolve identities from + the vendor's instance registry. + dev_mig_slots: + The number of MIG devices the card can host. + dev_bdf: + The card's BDF, for the synthetic identity fallback. + dev_temp: + The card's temperature. + dev_power_used: + The card's used power. + + Returns: + The MIG devices' usage, keyed by UUID. + + """ + ret: Devices = [] + for mdev in _iter_mig_device_handles(dev_dmi, dev_mig_slots): + # Suppressed per instance, not per card: one instance refusing its + # reads keeps the inventory's defaults while its siblings refresh. + with contextlib.suppress(pydmi.DMIError): + mdev_gi_id = pydmi.dmiDeviceGetGpuInstanceId(mdev) + mdev_ci_id = pydmi.dmiDeviceGetComputeInstanceId(mdev) + mdev_uuid = _get_mig_device_uuid( + dev_dmi_index, + mdev_gi_id, + mdev_ci_id, + dev_bdf, + ) + + mdev_mem = pydmi.dmiDeviceGetMemoryInfo(mdev) + mdev_util = pydmi.dmiDeviceGetUtilizationRates(mdev) + + mdev_mem_total = byte_to_mebibyte(mdev_mem.total) # byte to MiB + mdev_mem_used = byte_to_mebibyte(mdev_mem.used) # byte to MiB + + ret.append( + Device( + uuid=mdev_uuid, + cores_utilization=mdev_util.gpu, + memory_used=mdev_mem_used, + memory_utilization=get_utilization(mdev_mem_used, mdev_mem_total), + # A MIG device reports neither temperature nor power, so it + # carries the card's. + temperature=dev_temp, + power_used=dev_power_used, + ), + ) + return ret + + +def _get_mig_device_uuid( + dev_dmi_index: int, + gi_id: int, + ci_id: int, + dev_bdf: str, +) -> str: + """ + Resolve a MIG device's identity from the vendor's instance registry, the + mig_uuid line of devgici.conf, reported as MIG-. + + The registry is a driver convenience, not part of the library's API, so it + can be absent or stale (e.g. instances created outside the operator); then + the identity falls back to a synthetic MIG--gi-ci, which is + still unique on the node, and the gap is logged at debug. + + Args: + dev_dmi_index: + The card's DMI enumeration index, the conf name's N. + gi_id: + The GPU instance ID, the conf name's G. + ci_id: + The compute instance ID, the conf name's C. + dev_bdf: + The parent card's BDF, for the synthetic fallback. + + Returns: + The instance's identity, prefixed the way the vendor's tooling spells it. + + """ + conf = _DMI_MIG_CONFIG_DIR / "ci" / f"dev{dev_dmi_index}gi{gi_id}ci{ci_id}.conf" + with contextlib.suppress(OSError): + for line in conf.read_text(errors="replace").splitlines(): + key, sep, value = line.partition(":") + if sep and key.strip() == "mig_uuid" and value.strip(): + uuid = value.strip() + if uuid.startswith(_MIG_UUID_PREFIX): + return uuid + return f"{_MIG_UUID_PREFIX}{uuid}" + + logger.debug( + "Failed to read %s, falling back to a synthetic MIG identity", + conf, + ) + return f"{_MIG_UUID_PREFIX}{dev_bdf}-gi{gi_id}-ci{ci_id}" diff --git a/gpustack_runtime/detector/pydmi/__init__.py b/gpustack_runtime/detector/pydmi/__init__.py new file mode 100644 index 0000000..e1f3010 --- /dev/null +++ b/gpustack_runtime/detector/pydmi/__init__.py @@ -0,0 +1,662 @@ +## +# Python bindings for the Hygon DMI Multi-Instance library (libhydmi_mig.so). +# +# The vendor exports this API under NVML's symbol names while implementing +# something else, so every public wrapper here is dmi-prefixed and the library +# is loaded RTLD_LOCAL: loading it globally would let these names collide with +# libnvidia-ml.so's in a process that has both. Struct layouts follow the +# vendor's dmi_mig.h (v1.3.1), NOT pynvml, despite the shared names. +## +import string +import sys +import threading +from ctypes import * + +## C Type mappings ## +_dmiReturn_t = c_int +DMI_SUCCESS = 0 +DMI_ERROR_UNINITIALIZED = 1 +DMI_ERROR_INVALID_ARGUMENT = 2 +DMI_ERROR_NOT_SUPPORTED = 3 +DMI_ERROR_NO_PERMISSION = 4 +DMI_ERROR_ALREADY_INITIALIZED = 5 +DMI_ERROR_NOT_FOUND = 6 +DMI_ERROR_INSUFFICIENT_SIZE = 7 +DMI_ERROR_INSUFFICIENT_POWER = 8 +DMI_ERROR_DRIVER_NOT_LOADED = 9 +DMI_ERROR_TIMEOUT = 10 +DMI_ERROR_IRQ_ISSUE = 11 +DMI_ERROR_LIBRARY_NOT_FOUND = 12 +DMI_ERROR_FUNCTION_NOT_FOUND = 13 +DMI_ERROR_CORRUPTED_INFOROM = 14 +DMI_ERROR_GPU_IS_LOST = 15 +DMI_ERROR_RESET_REQUIRED = 16 +DMI_ERROR_OPERATING_SYSTEM = 17 +DMI_ERROR_LIB_RM_VERSION_MISMATCH = 18 +DMI_ERROR_IN_USE = 19 +DMI_ERROR_MEMORY = 20 +DMI_ERROR_NO_DATA = 21 +DMI_ERROR_VGPU_ECC_NOT_SUPPORTED = 22 +DMI_ERROR_INSUFFICIENT_RESOURCES = 23 +DMI_ERROR_FREQ_NOT_SUPPORTED = 24 +DMI_ERROR_ARGUMENT_VERSION_MISMATCH = 25 +DMI_ERROR_DEPRECATED = 26 +DMI_ERROR_NOT_READY = 27 +DMI_ERROR_UNKNOWN = 999 + +DMI_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32 +DMI_DEVICE_SERIAL_BUFFER_SIZE = 32 +DMI_DEVICE_NAME_BUFFER_SIZE = 256 +DMI_DEVICE_UUID_BUFFER_SIZE = 256 + +# Disable Multi Instance GPU mode. +DMI_DEVICE_MIG_DISABLE = 0x0 +# Enable Multi Instance GPU mode. +DMI_DEVICE_MIG_ENABLE = 0x1 + +# GPU instance profiles. +# These are the values accepted by dmiDeviceGetGpuInstanceProfileInfo's +# `profile` argument: an index into a fixed slice-count enumeration, NOT a +# profile id -- 0 asks for the one-slice profile, 3 for the four-slice one. +DMI_GPU_INSTANCE_PROFILE_1_SLICE = 0x0 +DMI_GPU_INSTANCE_PROFILE_2_SLICE = 0x1 +DMI_GPU_INSTANCE_PROFILE_3_SLICE = 0x2 +DMI_GPU_INSTANCE_PROFILE_4_SLICE = 0x3 +DMI_GPU_INSTANCE_PROFILE_COUNT = 0x4 + +# Compute instance profiles, indexed the same way. +DMI_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0 +DMI_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1 +DMI_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2 +DMI_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3 +DMI_COMPUTE_INSTANCE_PROFILE_COUNT = 0x4 + +# Compute instance engine profiles. +DMI_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0 +DMI_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1 + +## Opaque handles ## +# The vendor defines these as pointers to private structs; their only +# transferable property is identity, which is what attributes a MIG device to +# its card. Wrappers return c_void_p handles; struct fields of these types +# read back as plain ints, so compare a handle's `.value` against them. +_dmiDevice_t = c_void_p +_dmiGpuInstance_t = c_void_p +_dmiComputeInstance_t = c_void_p + + +## Structs ## +# Field layouts copied from dmi_mig.h v1.3.1. The vendor names fields in +# snake_case, which is kept. +class _dmiGpuInstanceProfileInfo_t(Structure): + _fields_ = [ + ("id", c_uint), + ("gi_count_max", c_uint), + ("cu_count", c_uint), + ("gpu_slice_count", c_uint), + # Carries MiB despite the name. + ("memory_size_MB", c_ulonglong), + ("name", c_char * DMI_DEVICE_NAME_BUFFER_SIZE), + ] + + +class _dmiComputeInstanceProfileInfo_t(Structure): + _fields_ = [ + ("id", c_uint), + ("ci_count_max", c_uint), + ("cu_count", c_uint), + ("gpu_slice_count", c_uint), + ("name", c_char * DMI_DEVICE_NAME_BUFFER_SIZE), + ] + + +class _dmiGpuInstancePlacement_t(Structure): + _fields_ = [ + # Index of first occupied memory slice. + ("start", c_uint), + # Number of memory slices occupied. + ("size", c_uint), + ] + + +class _dmiComputeInstancePlacement_t(Structure): + _fields_ = [ + # Index of first occupied compute slice. + ("start", c_uint), + # Number of compute slices occupied. + ("size", c_uint), + ] + + +class _dmiGpuInstanceInfo_t(Structure): + _fields_ = [ + # Parent device handle. + ("device", _dmiDevice_t), + # Unique instance ID within the device. + ("id", c_uint), + # Unique profile ID within the device. + ("profile_id", c_uint), + ("placement", _dmiGpuInstancePlacement_t), + ] + + +class _dmiComputeInstanceInfo_t(Structure): + _fields_ = [ + # Parent device handle. + ("device", _dmiDevice_t), + # Parent GPU instance handle. + ("gpu_instance", _dmiGpuInstance_t), + # Unique instance ID within the GPU instance. + ("id", c_uint), + # Unique profile ID within the GPU instance. + ("profile_id", c_uint), + # Placement within the GPU instance's compute slice range. + ("placement", _dmiComputeInstancePlacement_t), + ] + + +class _dmiMemory_t(Structure): + _fields_ = [ + ("total", c_ulonglong), + ("free", c_ulonglong), + ("used", c_ulonglong), + ] + + +class _dmiUtilization_t(Structure): + _fields_ = [ + # Compute core usage percent. + ("gpu", c_uint), + # Memory usage percent. + ("memory", c_uint), + ] + + +## Error Checking ## +class DMIError(Exception): + _valClassMapping = {} + # List of currently known error codes. + # The library exports no error-string function, so the map is static. + _errcode_to_string = { + DMI_ERROR_UNINITIALIZED: "Uninitialized", + DMI_ERROR_INVALID_ARGUMENT: "Invalid Argument", + DMI_ERROR_NOT_SUPPORTED: "Not Supported", + DMI_ERROR_NO_PERMISSION: "Insufficient Permissions", + DMI_ERROR_ALREADY_INITIALIZED: "Already Initialized", + DMI_ERROR_NOT_FOUND: "Not Found", + DMI_ERROR_INSUFFICIENT_SIZE: "Insufficient Size", + DMI_ERROR_INSUFFICIENT_POWER: "Insufficient External Power", + DMI_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded", + DMI_ERROR_TIMEOUT: "Timeout", + DMI_ERROR_IRQ_ISSUE: "Interrupt Request Issue", + DMI_ERROR_LIBRARY_NOT_FOUND: "DMI Shared Library Not Found", + DMI_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", + DMI_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM", + DMI_ERROR_GPU_IS_LOST: "GPU is lost", + DMI_ERROR_RESET_REQUIRED: "GPU requires restart", + DMI_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.", + DMI_ERROR_LIB_RM_VERSION_MISMATCH: "Driver/library version mismatch.", + DMI_ERROR_IN_USE: "In Use", + DMI_ERROR_MEMORY: "Insufficient Memory", + DMI_ERROR_NO_DATA: "No Data", + DMI_ERROR_VGPU_ECC_NOT_SUPPORTED: "VGPU ECC Not Supported", + DMI_ERROR_INSUFFICIENT_RESOURCES: "Insufficient Resources", + DMI_ERROR_FREQ_NOT_SUPPORTED: "Frequency Not Supported", + DMI_ERROR_ARGUMENT_VERSION_MISMATCH: "Argument Version Mismatch", + DMI_ERROR_DEPRECATED: "Deprecated", + DMI_ERROR_NOT_READY: "Not Ready", + DMI_ERROR_UNKNOWN: "Unknown Error", + } + + def __new__(typ, value): + """ + Maps value to a proper subclass of DMIError. + See _extractDMIErrorsAsClasses function for more details. + """ + if typ == DMIError: + typ = DMIError._valClassMapping.get(value, typ) + obj = Exception.__new__(typ) + obj.value = value + return obj + + def __str__(self): + return DMIError._errcode_to_string.get( + self.value, + "DMI Error with code %d" % self.value, + ) + + def __eq__(self, other): + return self.value == other.value + + +def dmiExceptionClass(dmiErrorCode): + if dmiErrorCode not in DMIError._valClassMapping: + msg = f"dmiErrorCode {dmiErrorCode} is not valid" + raise ValueError(msg) + return DMIError._valClassMapping[dmiErrorCode] + + +def _extractDMIErrorsAsClasses(): + """ + Generates a hierarchy of classes on top of DMIError class. + + Each DMI Error gets a new DMIError subclass. This way try,except blocks can + filter appropriate exceptions more easily. + """ + this_module = sys.modules[__name__] + dmiErrorsNames = [x for x in dir(this_module) if x.startswith("DMI_ERROR_")] + for err_name in dmiErrorsNames: + # e.g. Turn DMI_ERROR_ALREADY_INITIALIZED into DMIError_AlreadyInitialized + class_name = "DMIError_" + string.capwords( + err_name.replace("DMI_ERROR_", ""), "_" + ).replace("_", "") + err_val = getattr(this_module, err_name) + + def gen_new(val): + def new(typ, *args): + obj = DMIError.__new__(typ, val) + return obj + + return new + + new_error_class = type(class_name, (DMIError,), {"__new__": gen_new(err_val)}) + new_error_class.__module__ = __name__ + setattr(this_module, class_name, new_error_class) + DMIError._valClassMapping[err_val] = new_error_class + + +_extractDMIErrorsAsClasses() + + +def _dmiCheckReturn(ret): + if ret != DMI_SUCCESS: + raise DMIError(ret) + return ret + + +## Function access ## +libLoadLock = threading.Lock() +_dmiGetFunctionPointer_cache = {} # function pointers are cached to prevent unnecessary libLoadLock locking +dmiLib = None + + +def _dmiGetFunctionPointer(name): + global dmiLib + + if name in _dmiGetFunctionPointer_cache: + return _dmiGetFunctionPointer_cache[name] + + libLoadLock.acquire() + try: + # ensure library was loaded + if dmiLib is None: + raise DMIError(DMI_ERROR_UNINITIALIZED) + try: + _dmiGetFunctionPointer_cache[name] = getattr(dmiLib, name) + return _dmiGetFunctionPointer_cache[name] + except AttributeError: + raise DMIError(DMI_ERROR_FUNCTION_NOT_FOUND) + finally: + # lock is always freed + libLoadLock.release() + + +def _LoadDmiLibrary(): + """ + Load the library if it isn't loaded already. + + The library ships in the hyhal tree, which is not on the dynamic linker's + search path, so the absolute locations are tried as well. The versioned + soname comes first because a host can carry it without the bare one. + """ + global dmiLib + + if dmiLib is None: + # lock to ensure only one caller loads the library + libLoadLock.acquire() + try: + # ensure the library still isn't loaded + if dmiLib is None: + if not sys.platform.startswith("linux"): + # Do not support other platforms yet. + raise DMIError(DMI_ERROR_LIBRARY_NOT_FOUND) + locs = [ + "libhydmi_mig.so.1", + "libhydmi_mig.so", + ] + for loc_dir in ["/opt/hyhal/lib", "/opt/dtk/lib"]: + locs.append(loc_dir + "/libhydmi_mig.so.1") + locs.append(loc_dir + "/libhydmi_mig.so") + for loc in locs: + try: + # RTLD_LOCAL, never GLOBAL: the vendor exports NVML's + # symbol names, and global linkage would collide with + # libnvidia-ml.so in a process that has both. + dmiLib = CDLL(loc, mode=RTLD_LOCAL) + break + except OSError: + pass + if dmiLib is None: + raise DMIError(DMI_ERROR_LIBRARY_NOT_FOUND) + finally: + # lock is always freed + libLoadLock.release() + + +def dmiInit(): + """ + Load and resolve the vendor library. + + The vendor API has no initialize/shutdown calls -- despite the NVML symbol + names, there is no nvmlInit to forward to -- so "init" here means loading + the library, after which every wrapper works. + """ + _LoadDmiLibrary() + + +## System functions ## +def dmiGetSystemMigMode(): + """ + Report the node's Multi-Instance mode, current and pending. + + The mode is a property of the NODE, not of a card: this call takes no + device, and every card of a host answers alike. + """ + fn = _dmiGetFunctionPointer("nvmlGetSystemMigMode") + c_current = c_uint() + c_pending = c_uint() + ret = fn(byref(c_current), byref(c_pending)) + _dmiCheckReturn(ret) + return c_current.value, c_pending.value + + +## Device functions ## +def dmiDeviceGetCount(): + fn = _dmiGetFunctionPointer("nvmlDeviceGetCount") + c_count = c_uint() + ret = fn(byref(c_count)) + _dmiCheckReturn(ret) + return c_count.value + + +def dmiDeviceGetHandleByIndex(index): + fn = _dmiGetFunctionPointer("nvmlDeviceGetHandleByIndex") + c_device = _dmiDevice_t() + ret = fn(c_uint(index), byref(c_device)) + _dmiCheckReturn(ret) + return c_device + + +def dmiDeviceGetHandleByPciBusId(pci_bus_id): + """ + Return a handle for the physical DCU at a PCI address. + + This is the bridge from an identity another library (RSMI) enumerates by. + The address must be domain-qualified, "0000:09:00.0" rather than "09:00.0"; + an address no card answers for returns DMI_ERROR_NOT_FOUND. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetHandleByPciBusId") + c_device = _dmiDevice_t() + ret = fn(c_char_p(pci_bus_id.encode()), byref(c_device)) + _dmiCheckReturn(ret) + return c_device + + +def dmiDeviceGetIndex(device): + fn = _dmiGetFunctionPointer("nvmlDeviceGetIndex") + c_index = c_uint() + ret = fn(device, byref(c_index)) + _dmiCheckReturn(ret) + return c_index.value + + +def dmiDeviceGetMigMode(device): + """ + Report the device's current and pending Multi-Instance mode. + + The mode is set for the whole node, so every card of a host answers alike; + see dmiGetSystemMigMode. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetMigMode") + c_current = c_uint() + c_pending = c_uint() + ret = fn(device, byref(c_current), byref(c_pending)) + _dmiCheckReturn(ret) + return c_current.value, c_pending.value + + +def dmiDeviceGetMemoryInfo(device): + """ + Report total, free and used memory in bytes. + + Asked of a MIG device handle it reports that instance's own memory rather + than its card's, which is what makes it usable as a per-instance figure. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetMemoryInfo") + c_memory = _dmiMemory_t() + ret = fn(device, byref(c_memory)) + _dmiCheckReturn(ret) + return c_memory + + +def dmiDeviceGetUtilizationRates(device): + """ + Report compute and memory utilization as percentages. + + The vendor header does not declare this entry point, though the shared + object exports it. Asked of a MIG device handle it reports that instance's + own utilization; nothing else on this API measures per-instance compute. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetUtilizationRates") + c_utilization = _dmiUtilization_t() + ret = fn(device, byref(c_utilization)) + _dmiCheckReturn(ret) + return c_utilization + + +## MIG functions ## +_DMI_MAX_INSTANCES_PER_QUERY = 32 +""" +Bounds the buffers handed to the array-filling queries. A card carries four +GPU slices, so neither its GPU instances of one profile nor the compute +instances inside one of them can exceed that; the headroom is there so a +future card with a finer split does not silently truncate. +""" + + +def dmiDeviceGetGpuInstanceProfileInfo(device, profile): + """ + Return the GPU-instance profile at `profile`, an index into the fixed + slice-count enumeration -- 0 asks for the one-slice profile, 3 for the + four-slice one -- NOT a profile id, which comes back inside the answer and + bears no relation to the index. + + A card that offers no profile at this index answers DMI_ERROR_NOT_SUPPORTED, + DMI_ERROR_NOT_FOUND or DMI_ERROR_INVALID_ARGUMENT; that is routine, not a + fault. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstanceProfileInfo") + c_info = _dmiGpuInstanceProfileInfo_t() + ret = fn(device, c_uint(profile), byref(c_info)) + _dmiCheckReturn(ret) + return c_info + + +def dmiDeviceGetGpuInstancePossiblePlacements(device, profile_id): + """ + Return every placement the profile may legally occupy on an empty card, + as _dmiGpuInstancePlacement_t entries. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements") + c_placements = (_dmiGpuInstancePlacement_t * _DMI_MAX_INSTANCES_PER_QUERY)() + c_count = c_uint(_DMI_MAX_INSTANCES_PER_QUERY) + ret = fn(device, c_uint(profile_id), c_placements, byref(c_count)) + _dmiCheckReturn(ret) + if c_count.value > _DMI_MAX_INSTANCES_PER_QUERY: + raise DMIError(DMI_ERROR_INSUFFICIENT_SIZE) + return [c_placements[i] for i in range(c_count.value)] + + +def dmiDeviceGetGpuInstanceRemainingCapacity(device, profile_id): + """ + Report how many more instances of the profile the card can still hold, + accounting for what other profiles already occupy. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity") + c_count = c_uint() + ret = fn(device, c_uint(profile_id), byref(c_count)) + _dmiCheckReturn(ret) + return c_count.value + + +def dmiDeviceGetGpuInstances(device, profile_id): + """ + Return the GPU instances of ONE profile that currently exist on the card. + + The query filters by profile id, so enumerating every instance on a card + means asking once per profile the card offers. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstances") + c_instances = (_dmiGpuInstance_t * _DMI_MAX_INSTANCES_PER_QUERY)() + c_count = c_uint(_DMI_MAX_INSTANCES_PER_QUERY) + ret = fn(device, c_uint(profile_id), c_instances, byref(c_count)) + _dmiCheckReturn(ret) + if c_count.value > _DMI_MAX_INSTANCES_PER_QUERY: + raise DMIError(DMI_ERROR_INSUFFICIENT_SIZE) + # Array elements read back as plain ints; re-wrap so a handle passed into a + # later call keeps its full pointer width instead of ctypes' default c_int + # conversion truncating it. + return [_dmiGpuInstance_t(c_instances[i]) for i in range(c_count.value)] + + +def dmiDeviceGetGpuInstanceById(device, gpu_instance_id): + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstanceById") + c_gpu_instance = _dmiGpuInstance_t() + ret = fn(device, c_uint(gpu_instance_id), byref(c_gpu_instance)) + _dmiCheckReturn(ret) + return c_gpu_instance + + +def dmiGpuInstanceGetInfo(gpu_instance): + """ + Report the instance's id, its profile id, its parent device and where it + sits on the card. + """ + fn = _dmiGetFunctionPointer("nvmlGpuInstanceGetInfo") + c_info = _dmiGpuInstanceInfo_t() + ret = fn(gpu_instance, byref(c_info)) + _dmiCheckReturn(ret) + return c_info + + +def dmiGpuInstanceGetComputeInstanceProfileInfo(gpu_instance, profile, eng_profile): + """ + Return the compute-instance profile at `profile`, the same slice-count + indexing as its GPU-instance counterpart. `eng_profile` selects the engine + profile, of which the vendor defines exactly one, SHARED. + """ + fn = _dmiGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo") + c_info = _dmiComputeInstanceProfileInfo_t() + ret = fn(gpu_instance, c_uint(profile), c_uint(eng_profile), byref(c_info)) + _dmiCheckReturn(ret) + return c_info + + +def dmiGpuInstanceGetComputeInstances(gpu_instance, profile_id): + """ + Return the compute instances of ONE profile inside this GPU instance. + Like its GPU-instance counterpart it filters by profile id and must be + asked once per profile. + """ + fn = _dmiGetFunctionPointer("nvmlGpuInstanceGetComputeInstances") + c_instances = (_dmiComputeInstance_t * _DMI_MAX_INSTANCES_PER_QUERY)() + c_count = c_uint(_DMI_MAX_INSTANCES_PER_QUERY) + ret = fn(gpu_instance, c_uint(profile_id), c_instances, byref(c_count)) + _dmiCheckReturn(ret) + if c_count.value > _DMI_MAX_INSTANCES_PER_QUERY: + raise DMIError(DMI_ERROR_INSUFFICIENT_SIZE) + # Array elements read back as plain ints; re-wrap so a handle passed into a + # later call keeps its full pointer width instead of ctypes' default c_int + # conversion truncating it. + return [_dmiComputeInstance_t(c_instances[i]) for i in range(c_count.value)] + + +def dmiGpuInstanceGetComputeInstanceById(gpu_instance, compute_instance_id): + fn = _dmiGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById") + c_compute_instance = _dmiComputeInstance_t() + ret = fn(gpu_instance, c_uint(compute_instance_id), byref(c_compute_instance)) + _dmiCheckReturn(ret) + return c_compute_instance + + +def dmiComputeInstanceGetInfo(compute_instance): + """ + Report the compute instance's id, its profile id, its parent device and + GPU instance, and its placement within the GPU instance's slice range. + """ + fn = _dmiGetFunctionPointer("nvmlComputeInstanceGetInfo") + c_info = _dmiComputeInstanceInfo_t() + ret = fn(compute_instance, byref(c_info)) + _dmiCheckReturn(ret) + return c_info + + +def dmiDeviceIsMigDeviceHandle(device): + """ + Report whether this handle names a MIG instance rather than a physical card. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceIsMigDeviceHandle") + c_is_mig = c_uint() + ret = fn(device, byref(c_is_mig)) + _dmiCheckReturn(ret) + return c_is_mig.value != 0 + + +def dmiDeviceGetGpuInstanceId(device): + """ + Report which GPU instance a MIG device handle belongs to. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetGpuInstanceId") + c_id = c_uint() + ret = fn(device, byref(c_id)) + _dmiCheckReturn(ret) + return c_id.value + + +def dmiDeviceGetComputeInstanceId(device): + """ + Report which compute instance a MIG device handle is. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetComputeInstanceId") + c_id = c_uint() + ret = fn(device, byref(c_id)) + _dmiCheckReturn(ret) + return c_id.value + + +def dmiDeviceGetMaxMigDeviceCount(device): + """ + Report how many MIG devices this card can hold at once. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount") + c_count = c_uint() + ret = fn(device, byref(c_count)) + _dmiCheckReturn(ret) + return c_count.value + + +def dmiDeviceGetMigDeviceHandleByIndex(device, index): + """ + Return the MIG device at a GLOBAL index, if it belongs to this card. + + The index is not per-card, despite the call taking a card: it numbers + every MIG device on the NODE, and an index belonging to another card + answers DMI_ERROR_NOT_FOUND. A caller must bound and filter the sweep + itself, attributing each handle to its card by the GI info's parent + device, never by index ranges. + """ + fn = _dmiGetFunctionPointer("nvmlDeviceGetMigDeviceHandleByIndex") + c_mig_device = _dmiDevice_t() + ret = fn(device, c_uint(index), byref(c_mig_device)) + _dmiCheckReturn(ret) + return c_mig_device diff --git a/ruff.toml b/ruff.toml index 80ced51..bbf12a1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -125,6 +125,7 @@ parametrize-names-type = "csv" "gpustack_runtime/detector/pyamdgpu/*.py" = ["ALL"] "gpustack_runtime/detector/pycndev/*.py" = ["ALL"] "gpustack_runtime/detector/pydcmi/*.py" = ["ALL"] +"gpustack_runtime/detector/pydmi/*.py" = ["ALL"] "gpustack_runtime/detector/pyhgml/*.py" = ["ALL"] "gpustack_runtime/detector/pyhsa/*.py" = ["ALL"] "gpustack_runtime/detector/pyixml/*.py" = ["ALL"] @@ -133,6 +134,11 @@ parametrize-names-type = "csv" "gpustack_runtime/detector/pynvml/*.py" = ["ALL"] "gpustack_runtime/detector/pyrocmsmi/*.py" = ["ALL"] "tests/*.py" = ["D", "S101"] +# The fakes name methods after the vendor's NVML-style symbols and reset the +# binding's private module state. +"tests/gpustack_runtime/detector/test_pydmi.py" = ["D", "S101", "N802", "SLF001"] +# The fake DMI binding names methods after pydmi's dmi* wrappers. +"tests/gpustack_runtime/detector/test_hygon.py" = ["D", "S101", "N802"] [lint.mccabe] max-complexity = 50 diff --git a/tests/gpustack_runtime/detector/test_detector_cli.py b/tests/gpustack_runtime/detector/test_detector_cli.py index c910ac4..5d5122b 100644 --- a/tests/gpustack_runtime/detector/test_detector_cli.py +++ b/tests/gpustack_runtime/detector/test_detector_cli.py @@ -160,3 +160,57 @@ def test_no_detector_emits_a_vgpu_appendix_key(): # own `*_VGPU_*` constants are upstream SDK symbols, not appendix keys, # and do not match. assert offenders == [] + + +# --------------------------------------------------------------------------- # +# Hygon MIG inventory serialization. # +# --------------------------------------------------------------------------- # +def test_json_keeps_a_hygon_mig_devices_entry(): + dev = Device( + manufacturer=ManufacturerEnum.HYGON, + index=0, + name="K100_AI", + uuid="GPU-9f8e7d6c5b4a3921", + appendix={ + "mig": True, + "bdf": "0000:0b:00.0", + "numa": "0", + "mig_devices": [ + { + "index": 2, + "name": "1g.16gb", + "uuid": "MIG-aaaaaaaa-0000-0000-0000-000000000000", + "cores": 26, + "cores_utilization": 0, + "memory": 16380, + "memory_used": 0, + "memory_utilization": 0, + "temperature": None, + "power": 350, + "power_used": None, + "appendix": { + "mig": True, + "sliced": True, + "bdf": "0000:0b:00.0", + "numa": "0", + "gpu_instance_id": 5, + "compute_instance_id": 0, + "placement": {"start": 0, "length": 1}, + }, + }, + ], + }, + ) + + payload = json.loads(cmds_detector.format_devices_json([dev])) + + assert payload[0]["appendix"]["mig"] is True + mig = payload[0]["appendix"]["mig_devices"][0] + assert mig["uuid"] == "MIG-aaaaaaaa-0000-0000-0000-000000000000" + assert mig["index"] == 2 + assert mig["name"] == "1g.16gb" + assert mig["cores"] == 26 + assert mig["memory"] == 16380 + assert mig["appendix"]["gpu_instance_id"] == 5 + assert mig["appendix"]["compute_instance_id"] == 0 + assert mig["appendix"]["placement"] == {"start": 0, "length": 1} diff --git a/tests/gpustack_runtime/detector/test_hygon.py b/tests/gpustack_runtime/detector/test_hygon.py index db37bc0..f855750 100644 --- a/tests/gpustack_runtime/detector/test_hygon.py +++ b/tests/gpustack_runtime/detector/test_hygon.py @@ -1,6 +1,7 @@ from __future__ import annotations import contextlib +import ctypes from types import SimpleNamespace import pytest @@ -15,12 +16,144 @@ amd, hygon, pyamdgpu, + pydmi, pyhsa, ) from gpustack_runtime.detector.__utils__ import _load_pci_device_names from gpustack_runtime.detector.hygon import HygonDetector +class _FakeDmi: + """ + A pydmi stand-in answering out of fixture cards keyed by BDF. + + Handles are opaque c_void_p tokens the fixture hands out and decodes back, + so a mix-up between a card, GPU instance, compute instance or MIG device + handle reads as a wrong answer, not a passing one. Everything not faked + here (error type, constants) comes from the real binding. + """ + + def __getattr__(self, name): + return getattr(pydmi, name) + + def __init__(self, cards: list[dict], mig_enabled: bool = False): + self.mig_enabled = mig_enabled + self.cards = {card["bdf"]: card for card in cards} + # The MIG device index space is node-global: every card's instances + # share it, and an index belonging to another card answers NOT_FOUND. + self.mig_by_index = {} + for card in cards: + for inst in card.get("mig_instances", []): + self.mig_by_index[inst["mig_index"]] = (card, inst) + + @staticmethod + def _card_handle(card: dict) -> ctypes.c_void_p: + return ctypes.c_void_p(0x1000 + card["dmi_index"]) + + def _card_of_handle(self, handle) -> dict: + for card in self.cards.values(): + if self._card_handle(card).value == handle.value: + return card + raise pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND) + + def dmiInit(self): + pass + + def dmiGetSystemMigMode(self) -> tuple[int, int]: + if self.mig_enabled: + return pydmi.DMI_DEVICE_MIG_ENABLE, 0 + return pydmi.DMI_DEVICE_MIG_DISABLE, 0 + + def dmiDeviceGetCount(self) -> int: + return len(self.cards) + + def dmiDeviceGetHandleByPciBusId(self, bdf: str): + card = self.cards.get(bdf) + if card is None or card.get("dmi_unreachable"): + raise pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND) + return self._card_handle(card) + + def dmiDeviceGetIndex(self, handle) -> int: + return self._card_of_handle(handle)["dmi_index"] + + def dmiDeviceGetMaxMigDeviceCount(self, handle) -> int: + return self._card_of_handle(handle).get("max_mig", 4) + + def dmiDeviceGetMigDeviceHandleByIndex(self, handle, index: int): + card, _inst = self.mig_by_index.get(index) or (None, None) + if card is None or card is not self._card_of_handle(handle): + raise pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND) + return ctypes.c_void_p(0x2000 + index) + + def _mig_instance_of(self, mdev_handle) -> tuple[dict, dict]: + card, inst = self.mig_by_index.get(mdev_handle.value - 0x2000) or (None, None) + if inst is None: + raise pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND) + return card, inst + + def dmiDeviceGetGpuInstanceId(self, mdev_handle) -> int: + _, inst = self._mig_instance_of(mdev_handle) + if inst.get("fail_gi_id"): + raise pydmi.DMIError(pydmi.DMI_ERROR_UNKNOWN) + return inst["gi_id"] + + def dmiDeviceGetComputeInstanceId(self, mdev_handle) -> int: + _, inst = self._mig_instance_of(mdev_handle) + return inst["ci_id"] + + def dmiDeviceGetGpuInstanceById(self, handle, gi_id: int): + card = self._card_of_handle(handle) + return ctypes.c_void_p(0x3000 + card["dmi_index"] * 100 + gi_id) + + def dmiGpuInstanceGetInfo(self, gi_handle): + gi_id = gi_handle.value - 0x3000 + dmi_index, gi_id = divmod(gi_id, 100) + card = next(c for c in self.cards.values() if c["dmi_index"] == dmi_index) + for inst in card.get("mig_instances", []): + if inst["gi_id"] == gi_id: + return SimpleNamespace( + device=self._card_handle(card).value, + id=gi_id, + profile_id=inst["profile_id"], + placement=SimpleNamespace(start=inst["start"], size=inst["size"]), + ) + raise pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND) + + def dmiGpuInstanceGetComputeInstanceById(self, gi_handle, ci_id: int): + return ctypes.c_void_p(0x4000 + gi_handle.value - 0x3000 + ci_id) + + def dmiComputeInstanceGetInfo(self, ci_handle): + return SimpleNamespace( + device=0, + gpu_instance=0, + id=0, + profile_id=0, + placement=SimpleNamespace(start=0, size=1), + ) + + def dmiDeviceGetGpuInstanceProfileInfo(self, handle, profile: int): + card = self._card_of_handle(handle) + prf = card.get("profiles", {}).get(profile) + if prf is None: + # A width the card offers no profile for is a routine gap. + raise pydmi.DMIError(pydmi.DMI_ERROR_INVALID_ARGUMENT) + return SimpleNamespace(**prf) + + def dmiDeviceGetMemoryInfo(self, mdev_handle): + _, inst = self._mig_instance_of(mdev_handle) + return SimpleNamespace( + total=inst["memory_bytes"], + free=inst["memory_bytes"] - inst["memory_used_bytes"], + used=inst["memory_used_bytes"], + ) + + def dmiDeviceGetUtilizationRates(self, mdev_handle): + _, inst = self._mig_instance_of(mdev_handle) + if inst.get("fail_utilization"): + raise pydmi.DMIError(pydmi.DMI_ERROR_UNKNOWN) + return SimpleNamespace(gpu=inst["gpu_util"], memory=0) + + @pytest.mark.skipif( not HygonDetector.is_supported(), reason="Hygon GPU not detected", @@ -223,7 +356,7 @@ def _card( } -def _agent(bdf: str) -> pyhsa.Agent: +def _agent(bdf: str, compute_units: int = 104) -> pyhsa.Agent: return pyhsa.Agent( device_type=1, device_id="0x6210", @@ -231,7 +364,7 @@ def _agent(bdf: str) -> pyhsa.Agent: uuid="", name=_HSA_NAME, compute_capability="gfx936", - compute_units=104, + compute_units=compute_units, ) @@ -246,12 +379,33 @@ def _setup( cards: list[dict], agents: list | None = None, pci_ids: str | None = _PCI_IDS, + dmi_mig_enabled: bool = False, + dmi_mig_confs: dict[str, str] | None = None, ) -> list[str]: calls: list[str] = [] monkeypatch.setattr(HygonDetector, "is_supported", staticmethod(lambda: True)) monkeypatch.setattr(hygon, "pyrocmsmi", _FakeRocmSmi(calls, cards)) monkeypatch.setattr(hygon, "pyhsa", _FakeHSA(list(agents or []))) + monkeypatch.setattr( + hygon, + "pydmi", + _FakeDmi(cards, mig_enabled=dmi_mig_enabled), + ) + + # The vendor's instance registry: only the confs the test names exist. + mig_config_dir = tmp_path / "dmi_mig_config" + if dmi_mig_confs: + ci_dir = mig_config_dir / "ci" + ci_dir.mkdir(parents=True) + for name, uuid in dmi_mig_confs.items(): + (ci_dir / name).write_text( + "cu_count: 20\n" + "memory_size_MB: 16380\n" + f"mig_uuid: {uuid}\n", + encoding="utf-8", + ) + monkeypatch.setattr(hygon, "_DMI_MIG_CONFIG_DIR", mig_config_dir) pci_devices_path = tmp_path / "pci_devices" for card in cards: @@ -519,3 +673,423 @@ def test_cdi_spec_numbers_the_device_nodes_from_the_appendix(monkeypatch): "/dev/kfd", "/dev/mkfd", ] + + +# --------------------------------------------------------------------------- # +# MIG: the node-wide mode gates per-card marking and instance enumeration. # +# --------------------------------------------------------------------------- # + +_MIG_PROFILE_1G = { + "id": 3, + "gi_count_max": 4, + "cu_count": 26, + "gpu_slice_count": 1, + "memory_size_MB": 16380, + "name": b"MIG 1g.16gb", +} + + +def _mig_card( + bdf: str, + unique_id: str, + dmi_index: int, + *, + mig_instances: list[dict] | None = None, + profiles: dict | None = None, + max_mig: int = 4, + dmi_unreachable: bool = False, + **kwargs, +) -> dict: + """ + One fake Hygon card plus what the fake DMI library serves for it. + """ + card = _card(bdf, unique_id, **kwargs) + card["dmi_index"] = dmi_index + card["max_mig"] = max_mig + card["profiles"] = profiles or {} + card["mig_instances"] = mig_instances or [] + card["dmi_unreachable"] = dmi_unreachable + return card + + +def _mig_instance( + mig_index: int, + gi_id: int, + ci_id: int, + *, + profile_id: int = 3, + start: int = 0, + size: int = 1, + memory_bytes: int = 17179869184, # 16384 MiB + memory_used_bytes: int = 4294967296, # 4096 MiB + gpu_util: int = 0, + **kwargs, +) -> dict: + return { + "mig_index": mig_index, + "gi_id": gi_id, + "ci_id": ci_id, + "profile_id": profile_id, + "start": start, + "size": size, + "memory_bytes": memory_bytes, + "memory_used_bytes": memory_used_bytes, + "gpu_util": gpu_util, + **kwargs, + } + + +def test_detect_info_mig_mode_off_reports_physical_only(hygon_bindings): + # The library answers and the conf registry could even hold stale entries: + # with the node-wide mode off the detector behaves exactly as without MIG. + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0)], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=False, + dmi_mig_confs={"dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000"}, + ) + + dev = HygonDetector().detect_info()[0] + + assert "mig" not in dev.appendix + assert "mig_devices" not in dev.appendix + + +def test_detect_info_mig_library_absent_reports_physical_only( + hygon_bindings, + monkeypatch, +): + class _AbsentDmi(_FakeDmi): + def dmiInit(self): + raise pydmi.DMIError(pydmi.DMI_ERROR_LIBRARY_NOT_FOUND) + + hygon_bindings( + [_mig_card("0000:0b:00.0", "0x9f8e7d6c5b4a3921", 0)], + agents=[_agent("0000:0b:00.0")], + ) + monkeypatch.setattr(hygon, "pydmi", _AbsentDmi([], mig_enabled=True)) + + dev = HygonDetector().detect_info()[0] + + assert "mig" not in dev.appendix + assert "mig_devices" not in dev.appendix + + +def test_detect_info_marks_mig_cards_and_enumerates_instances( + hygon_bindings, + monkeypatch, +): + # The ECC read is opt-in, as the health check is disabled by default. + monkeypatch.setattr(envs, "GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK", False) + # Two cards, each holding instances; the node-global MIG index space is + # interleaved (card 1's instance sits between card 0's two), so the sweep + # proves attribution by the card's own handle rather than by index ranges. + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[ + _mig_instance(0, 5, 0, start=0), + _mig_instance(2, 6, 0, start=1), + ], + profiles={0: _MIG_PROFILE_1G}, + ), + _mig_card( + "0000:0c:00.0", + "0x9f8e7d6c5b4a3922", + 1, + mig_instances=[_mig_instance(1, 1, 0, start=0)], + profiles={0: _MIG_PROFILE_1G}, + uncorrectable_err=1, + ), + ], + agents=[_agent("0000:0b:00.0"), _agent("0000:0c:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={ + "dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000", + "dev0gi6ci0.conf": "bbbbbbbb-0000-0000-0000-000000000000", + "dev1gi1ci0.conf": "cccccccc-0000-0000-0000-000000000000", + }, + ) + + devices = HygonDetector().detect_info() + + assert [dev.appendix["mig"] for dev in devices] == [True, True] + + card0_migs = devices[0].appendix["mig_devices"] + assert [m["uuid"] for m in card0_migs] == [ + "MIG-aaaaaaaa-0000-0000-0000-000000000000", + "MIG-bbbbbbbb-0000-0000-0000-000000000000", + ] + assert [m["name"] for m in card0_migs] == ["1g.16gb", "1g.16gb"] + assert [m["memory"] for m in card0_migs] == [16380, 16380] + assert [m["cores"] for m in card0_migs] == [26, 26] + assert [m["appendix"]["gpu_instance_id"] for m in card0_migs] == [5, 6] + assert [m["appendix"]["compute_instance_id"] for m in card0_migs] == [0, 0] + assert [m["appendix"]["placement"] for m in card0_migs] == [ + {"start": 0, "length": 1}, + {"start": 1, "length": 1}, + ] + assert [m["appendix"]["bdf"] for m in card0_migs] == ["0000:0b:00.0"] * 2 + assert [m["appendix"]["numa"] for m in card0_migs] == ["0", "0"] + assert all(m["appendix"]["mig"] and m["appendix"]["sliced"] for m in card0_migs) + # A partition carries its card's memory-health verdict. + assert [m["memory_status"] for m in card0_migs] == [ + DeviceMemoryStatusEnum.HEALTHY, + DeviceMemoryStatusEnum.HEALTHY, + ] + + card1_migs = devices[1].appendix["mig_devices"] + assert [m["uuid"] for m in card1_migs] == [ + "MIG-cccccccc-0000-0000-0000-000000000000", + ] + assert [m["memory_status"] for m in card1_migs] == [ + DeviceMemoryStatusEnum.UNHEALTHY, + ] + + # Synthetic indexes: blocks of the card's slot count above the largest + # physical index (2), per index_mig_devices -- partitioning one card never + # renumbers another's instances. + assert [m["index"] for m in card0_migs] == [2, 3] + assert [m["index"] for m in card1_migs] == [6] + + +def test_detect_info_mig_conf_missing_falls_back_to_a_synthetic_uuid(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0)], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs=None, # no registry at all + ) + + migs = HygonDetector().detect_info()[0].appendix["mig_devices"] + + assert [m["uuid"] for m in migs] == ["MIG-0000:0b:00.0-gi5-ci0"] + + +def test_detect_info_mig_card_unreachable_via_dmi_degrades_to_physical(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0)], + profiles={0: _MIG_PROFILE_1G}, + dmi_unreachable=True, + ), + _mig_card( + "0000:0c:00.0", + "0x9f8e7d6c5b4a3922", + 1, + mig_instances=[_mig_instance(1, 1, 0)], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0"), _agent("0000:0c:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={"dev1gi1ci0.conf": "cccccccc-0000-0000-0000-000000000000"}, + ) + + devices = HygonDetector().detect_info() + + assert "mig" not in devices[0].appendix + assert devices[1].appendix["mig"] is True + assert [m["uuid"] for m in devices[1].appendix["mig_devices"]] == [ + "MIG-cccccccc-0000-0000-0000-000000000000", + ] + + +def test_detect_info_mig_one_unreadable_instance_never_aborts_the_sweep(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[ + _mig_instance(0, 5, 0, fail_gi_id=True), + _mig_instance(1, 6, 0, start=1), + ], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={"dev0gi6ci0.conf": "bbbbbbbb-0000-0000-0000-000000000000"}, + ) + + migs = HygonDetector().detect_info()[0].appendix["mig_devices"] + + assert [m["uuid"] for m in migs] == ["MIG-bbbbbbbb-0000-0000-0000-000000000000"] + # The surviving instance keeps its per-card discovery ordinal (1) under the + # block offset (base 1): the failed read ahead of it must not renumber it. + assert [m["index"] for m in migs] == [2] + + +def test_detect_info_mig_derives_physical_cores_from_profiles(hygon_bindings): + # With MIG enabled HSA exposes a partition's view of the card -- here one + # slice's 26 CUs -- so the card's own count comes from its profiles + # instead: 26 per instance times 4 instances of capacity. + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0)], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0", compute_units=26)], + dmi_mig_enabled=True, + dmi_mig_confs={"dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000"}, + ) + + dev = HygonDetector().detect_info()[0] + + assert dev.cores == 104 + + +def test_detect_info_mig_tolerates_garbled_vendor_strings(hygon_bindings, tmp_path): + # A non-UTF-8 profile name or conf file is a vendor defect to report + # around, never a reason to lose the whole detection. + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0)], + profiles={0: {**_MIG_PROFILE_1G, "name": b"MIG \xff1g.16gb"}}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={"dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000"}, + ) + conf = tmp_path / "dmi_mig_config" / "ci" / "dev0gi5ci0.conf" + conf.write_bytes(b"\xffmig_uuid: garbled\n") + + migs = HygonDetector().detect_info()[0].appendix["mig_devices"] + + # The garbled byte decodes to a replacement char, not a raised error. + assert [m["name"] for m in migs] == ["\N{REPLACEMENT CHARACTER}1g.16gb"] + # The garbled conf holds no readable mig_uuid line, so the identity falls + # back to the synthetic one. + assert [m["uuid"] for m in migs] == ["MIG-0000:0b:00.0-gi5-ci0"] + + +# --------------------------------------------------------------------------- # +# MIG usage: refreshed per instance through its own handle, merged by UUID. # +# --------------------------------------------------------------------------- # + + +def test_detect_usage_merges_mig_usage_by_uuid(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[ + _mig_instance(0, 5, 0, gpu_util=95), + _mig_instance(1, 6, 0, start=1), + ], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={ + "dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000", + "dev0gi6ci0.conf": "bbbbbbbb-0000-0000-0000-000000000000", + }, + ) + det = HygonDetector() + devices = det.detect_info() + + det.detect_usage(devices) + + migs = devices[0].appendix["mig_devices"] + assert [m["cores_utilization"] for m in migs] == [95, 0] + assert [m["memory_used"] for m in migs] == [4096, 4096] + assert [m["memory_utilization"] for m in migs] == [25.0, 25.0] + # A MIG device reports neither temperature nor power, so it carries the + # card's. + assert [m["temperature"] for m in migs] == [51, 51] + assert [m["power_used"] for m in migs] == [217, 217] + # The inventory fields survive the merge. + assert [m["memory"] for m in migs] == [16380, 16380] + assert [m["name"] for m in migs] == ["1g.16gb", "1g.16gb"] + + +def test_detect_usage_mig_suppresses_a_single_unreadable_instance(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[ + _mig_instance(0, 5, 0, fail_utilization=True), + _mig_instance(1, 6, 0, start=1, gpu_util=42), + ], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=True, + dmi_mig_confs={ + "dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000", + "dev0gi6ci0.conf": "bbbbbbbb-0000-0000-0000-000000000000", + }, + ) + det = HygonDetector() + devices = det.detect_info() + + det.detect_usage(devices) + + migs = devices[0].appendix["mig_devices"] + # The unreadable instance keeps the inventory's defaults; its sibling is + # refreshed -- one bad read never blinds the sweep. + assert [m["cores_utilization"] for m in migs] == [0, 42] + + +def test_detect_usage_mig_mode_off_makes_no_dmi_usage_call(hygon_bindings): + hygon_bindings( + [ + _mig_card( + "0000:0b:00.0", + "0x9f8e7d6c5b4a3921", + 0, + mig_instances=[_mig_instance(0, 5, 0, gpu_util=95)], + profiles={0: _MIG_PROFILE_1G}, + ), + ], + agents=[_agent("0000:0b:00.0")], + dmi_mig_enabled=False, + dmi_mig_confs={"dev0gi5ci0.conf": "aaaaaaaa-0000-0000-0000-000000000000"}, + ) + + devices = HygonDetector().detect_usage() + + assert "mig_devices" not in devices[0].appendix diff --git a/tests/gpustack_runtime/detector/test_pydmi.py b/tests/gpustack_runtime/detector/test_pydmi.py new file mode 100644 index 0000000..66badf8 --- /dev/null +++ b/tests/gpustack_runtime/detector/test_pydmi.py @@ -0,0 +1,366 @@ +import ctypes +import sys +from ctypes import RTLD_LOCAL + +import pytest + +from gpustack_runtime.detector import pydmi + + +@pytest.fixture(autouse=True) +def _reset_pydmi(monkeypatch): + """ + Reset pydmi's module-level library state around every test, + so each test controls its own fake CDLL. + """ + monkeypatch.setattr(pydmi, "dmiLib", None) + pydmi._dmiGetFunctionPointer_cache.clear() + yield + pydmi._dmiGetFunctionPointer_cache.clear() + + +class _FakeCDLL: + """ + A CDLL stand-in recording every attempted load (path and mode), + succeeding only for the paths in `accepted` with the given library object. + """ + + def __init__(self, lib, accepted: set[str]): + self.lib = lib + self.accepted = accepted + self.calls: list[tuple[str, int | None]] = [] + + def __call__(self, path, mode=None): + self.calls.append((path, mode)) + if path not in self.accepted: + msg = f"cannot open {path}" + raise OSError(msg) + return self.lib + + +def test_load_searches_default_and_vendor_paths_with_rtld_local(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + lib = object() + fake = _FakeCDLL(lib, {"/opt/dtk/lib/libhydmi_mig.so"}) + monkeypatch.setattr(pydmi, "CDLL", fake) + + pydmi.dmiInit() + + tried = [path for path, _ in fake.calls] + assert tried == [ + "libhydmi_mig.so.1", + "libhydmi_mig.so", + "/opt/hyhal/lib/libhydmi_mig.so.1", + "/opt/hyhal/lib/libhydmi_mig.so", + "/opt/dtk/lib/libhydmi_mig.so.1", + "/opt/dtk/lib/libhydmi_mig.so", + ] + # The vendor exports NVML's symbol names, so the library must never be + # loaded globally: every attempt carries RTLD_LOCAL. + assert all(mode == RTLD_LOCAL for _, mode in fake.calls) + assert pydmi.dmiLib is lib + + +def test_load_failure_raises_dmi_error_never_oserror(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(pydmi, "CDLL", _FakeCDLL(object(), set())) + + with pytest.raises(pydmi.DMIError_LibraryNotFound) as excinfo: + pydmi.dmiInit() + + assert excinfo.value.value == pydmi.DMI_ERROR_LIBRARY_NOT_FOUND + assert not isinstance(excinfo.value, OSError) + + +def test_error_codes_map_to_subclasses(): + assert isinstance( + pydmi.DMIError(pydmi.DMI_ERROR_NOT_FOUND), + pydmi.DMIError_NotFound, + ) + assert isinstance( + pydmi.DMIError(pydmi.DMI_ERROR_NOT_SUPPORTED), + pydmi.DMIError_NotSupported, + ) + assert isinstance( + pydmi.DMIError(pydmi.DMI_ERROR_INVALID_ARGUMENT), + pydmi.DMIError_InvalidArgument, + ) + # An unknown code stays the base class and still stringifies. + assert type(pydmi.DMIError(12345)) is pydmi.DMIError + assert "12345" in str(pydmi.DMIError(12345)) + + +def test_call_without_init_raises_uninitialized(): + with pytest.raises(pydmi.DMIError_Uninitialized): + pydmi.dmiGetSystemMigMode() + + +def test_missing_symbol_raises_function_not_found(monkeypatch): + monkeypatch.setattr(pydmi, "dmiLib", object()) + + with pytest.raises(pydmi.DMIError_FunctionNotFound): + pydmi.dmiGetSystemMigMode() + + +def test_wrapper_raises_the_mapped_error_on_non_success(monkeypatch): + class Lib: + def nvmlGetSystemMigMode(self, current, pending): + return pydmi.DMI_ERROR_NOT_SUPPORTED + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + with pytest.raises(pydmi.DMIError_NotSupported): + pydmi.dmiGetSystemMigMode() + + +def test_system_mig_mode_returns_current_and_pending(monkeypatch): + class Lib: + def nvmlGetSystemMigMode(self, current, pending): + ctypes.cast(current, ctypes.POINTER(ctypes.c_uint)).contents.value = 1 + ctypes.cast(pending, ctypes.POINTER(ctypes.c_uint)).contents.value = 0 + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + assert pydmi.dmiGetSystemMigMode() == (1, 0) + + +def test_handle_by_pci_bus_id_encodes_and_returns_handle(monkeypatch): + seen = {} + + class Lib: + def nvmlDeviceGetHandleByPciBusId(self, bus_id, device): + seen["bus_id"] = bus_id.value + ctypes.cast(device, ctypes.POINTER(ctypes.c_void_p)).contents.value = 0xDEAD + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + handle = pydmi.dmiDeviceGetHandleByPciBusId("0000:09:00.0") + + assert seen["bus_id"] == b"0000:09:00.0" + assert handle.value == 0xDEAD + + +def test_gpu_instance_profile_info_marshals_the_vendor_struct(monkeypatch): + class Lib: + def nvmlDeviceGetGpuInstanceProfileInfo(self, device, profile, info): + assert profile.value == 2 # raw slice-count-minus-one argument + out = ctypes.cast( + info, + ctypes.POINTER(pydmi._dmiGpuInstanceProfileInfo_t), + ).contents + out.id = 7 + out.gi_count_max = 2 + out.cu_count = 30 + out.gpu_slice_count = 3 + out.memory_size_MB = 49152 + out.name = b"MIG 3g.48gb" + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + info = pydmi.dmiDeviceGetGpuInstanceProfileInfo(ctypes.c_void_p(1), 2) + + assert info.id == 7 + assert info.gi_count_max == 2 + assert info.cu_count == 30 + assert info.gpu_slice_count == 3 + assert info.memory_size_MB == 49152 + assert info.name == b"MIG 3g.48gb" + + +def test_gpu_instances_returns_only_the_reported_count(monkeypatch): + # Handle values beyond 32 bits: array elements must survive the round trip + # into later calls at full pointer width. + hi, lo = 0x1_0000_0111, 0x1_0000_0222 + seen = {} + + class Lib: + def nvmlDeviceGetGpuInstances(self, device, profile_id, instances, count): + out = ctypes.cast(instances, ctypes.POINTER(ctypes.c_void_p)) + out[0] = hi + out[1] = lo + ctypes.cast(count, ctypes.POINTER(ctypes.c_uint)).contents.value = 2 + return pydmi.DMI_SUCCESS + + def nvmlGpuInstanceGetInfo(self, gpu_instance, info): + seen["handle"] = gpu_instance + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + handles = pydmi.dmiDeviceGetGpuInstances(ctypes.c_void_p(1), 3) + + assert [h.value for h in handles] == [hi, lo] + # The round trip: an enumerated handle passed back in keeps its full width. + pydmi.dmiGpuInstanceGetInfo(handles[0]) + got = seen["handle"] + assert (got.value if isinstance(got, ctypes.c_void_p) else got) == hi + + +def test_gpu_instances_overflow_reports_insufficient_size(monkeypatch): + class Lib: + def nvmlDeviceGetGpuInstances(self, device, profile_id, instances, count): + ctypes.cast(count, ctypes.POINTER(ctypes.c_uint)).contents.value = ( + pydmi._DMI_MAX_INSTANCES_PER_QUERY + 1 + ) + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + with pytest.raises(pydmi.DMIError_InsufficientSize): + pydmi.dmiDeviceGetGpuInstances(ctypes.c_void_p(1), 3) + + +def test_gpu_instance_info_carries_parent_device_and_placement(monkeypatch): + class Lib: + def nvmlGpuInstanceGetInfo(self, gpu_instance, info): + out = ctypes.cast( + info, + ctypes.POINTER(pydmi._dmiGpuInstanceInfo_t), + ).contents + out.device = 0x999 + out.id = 4 + out.profile_id = 1 + out.placement.start = 2 + out.placement.size = 2 + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + info = pydmi.dmiGpuInstanceGetInfo(ctypes.c_void_p(0xABC)) + + # The parent device is how a MIG device is attributed to its card, + # so it must arrive as a comparable value. + assert info.device == 0x999 + assert info.id == 4 + assert info.profile_id == 1 + assert (info.placement.start, info.placement.size) == (2, 2) + + +def test_compute_instance_info_marshals_the_vendor_struct(monkeypatch): + class Lib: + def nvmlComputeInstanceGetInfo(self, compute_instance, info): + out = ctypes.cast( + info, + ctypes.POINTER(pydmi._dmiComputeInstanceInfo_t), + ).contents + out.device = 0x999 + out.gpu_instance = 0xABC + out.id = 0 + out.profile_id = 0 + out.placement.start = 0 + out.placement.size = 1 + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + info = pydmi.dmiComputeInstanceGetInfo(ctypes.c_void_p(0xDEF)) + + assert info.device == 0x999 + assert info.gpu_instance == 0xABC + assert info.id == 0 + assert (info.placement.start, info.placement.size) == (0, 1) + + +def test_mig_device_queries(monkeypatch): + class Lib: + def nvmlDeviceGetMaxMigDeviceCount(self, device, count): + ctypes.cast(count, ctypes.POINTER(ctypes.c_uint)).contents.value = 4 + return pydmi.DMI_SUCCESS + + def nvmlDeviceGetMigDeviceHandleByIndex(self, device, index, mig_device): + if index.value == 3: + return pydmi.DMI_ERROR_NOT_FOUND + ctypes.cast( + mig_device, + ctypes.POINTER(ctypes.c_void_p), + ).contents.value = 0x1000 + index.value + return pydmi.DMI_SUCCESS + + def nvmlDeviceIsMigDeviceHandle(self, device, is_mig): + ctypes.cast(is_mig, ctypes.POINTER(ctypes.c_uint)).contents.value = 1 + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + dev = ctypes.c_void_p(0x999) + assert pydmi.dmiDeviceGetMaxMigDeviceCount(dev) == 4 + assert pydmi.dmiDeviceGetMigDeviceHandleByIndex(dev, 0).value == 0x1000 + with pytest.raises(pydmi.DMIError_NotFound): + pydmi.dmiDeviceGetMigDeviceHandleByIndex(dev, 3) + assert pydmi.dmiDeviceIsMigDeviceHandle(dev) is True + + +def test_struct_layouts_match_the_vendor_header(): + """ + Independent layout assertions (sizes/offsets computed by hand from + dmi_mig.h v1.3.1, LP64), so a wrong field width or order cannot hide + behind fakes that cast with the very declarations under test. + """ + gi_prf = pydmi._dmiGpuInstanceProfileInfo_t + assert ctypes.sizeof(gi_prf) == 280 + assert gi_prf.id.offset == 0 + assert gi_prf.memory_size_MB.offset == 16 + assert gi_prf.name.offset == 24 + + ci_prf = pydmi._dmiComputeInstanceProfileInfo_t + assert ctypes.sizeof(ci_prf) == 272 + assert ci_prf.name.offset == 16 + + assert ctypes.sizeof(pydmi._dmiGpuInstancePlacement_t) == 8 + assert ctypes.sizeof(pydmi._dmiComputeInstancePlacement_t) == 8 + + gi_info = pydmi._dmiGpuInstanceInfo_t + assert ctypes.sizeof(gi_info) == 24 + assert gi_info.device.offset == 0 + assert gi_info.id.offset == 8 + assert gi_info.profile_id.offset == 12 + assert gi_info.placement.offset == 16 + + ci_info = pydmi._dmiComputeInstanceInfo_t + assert ctypes.sizeof(ci_info) == 32 + assert ci_info.device.offset == 0 + assert ci_info.gpu_instance.offset == 8 + assert ci_info.id.offset == 16 + assert ci_info.profile_id.offset == 20 + assert ci_info.placement.offset == 24 + + assert ctypes.sizeof(pydmi._dmiMemory_t) == 24 + assert ctypes.sizeof(pydmi._dmiUtilization_t) == 8 + + +def test_memory_and_utilization_read_through_a_mig_handle(monkeypatch): + class Lib: + def nvmlDeviceGetMemoryInfo(self, device, memory): + out = ctypes.cast( + memory, + ctypes.POINTER(pydmi._dmiMemory_t), + ).contents + out.total = 16 * 1024**3 + out.free = 12 * 1024**3 + out.used = 4 * 1024**3 + return pydmi.DMI_SUCCESS + + def nvmlDeviceGetUtilizationRates(self, device, utilization): + out = ctypes.cast( + utilization, + ctypes.POINTER(pydmi._dmiUtilization_t), + ).contents + out.gpu = 95 + out.memory = 12 + return pydmi.DMI_SUCCESS + + monkeypatch.setattr(pydmi, "dmiLib", Lib()) + + mig = ctypes.c_void_p(0x1000) + mem = pydmi.dmiDeviceGetMemoryInfo(mig) + util = pydmi.dmiDeviceGetUtilizationRates(mig) + + assert (mem.total, mem.free, mem.used) == ( + 16 * 1024**3, + 12 * 1024**3, + 4 * 1024**3, + ) + assert (util.gpu, util.memory) == (95, 12)