From ad06dab8d48f42549551f2c4064b5650ba0175c9 Mon Sep 17 00:00:00 2001 From: yxf Date: Thu, 3 Sep 2026 23:53:31 +0800 Subject: [PATCH] feat(ascend): add Device injection policy and fix A5 driver/UB mounts On Docker, the default Env injection policy sets the visible-devices env (ASCEND_VISIBLE_DEVICES), which makes the Ascend Docker Runtime apply device isolation. On A5 (950) that isolation hides the UB fabric, so HCCL rootInfo detection fails. The CDI mount profile, modeled on the A2/A3 operator profile, also omits A5's UB driver components (ube_mgmt, under the driver tree) and blanket-mounts a host /etc/hccl_rootinfo.json whose stale content fails rootInfo detection. - Add a "Device" resource injection policy: inject device nodes and mounts as plain Docker devices/binds (reusing the CDI generator), with no visible-devices env and without requiring CDI-capable Docker. - Drop mirrored visible-devices envs under non-Env policies so a mirrored ASCEND_VISIBLE_DEVICES cannot re-trigger the isolation. - For A5, mount the whole /usr/local/Ascend/driver (brings ube_mgmt) and stop mounting /etc/hccl_rootinfo.json. - Keep injected mounts when appending container mounts, and fix a device mirroring typo exposed by the new plain-device path. --- gpustack_runtime/deployer/cdi/__init__.py | 18 +++ gpustack_runtime/deployer/cdi/ascend.py | 51 ++++--- gpustack_runtime/deployer/docker.py | 168 ++++++++++++++++++---- gpustack_runtime/envs.py | 5 +- 4 files changed, 199 insertions(+), 43 deletions(-) diff --git a/gpustack_runtime/deployer/cdi/__init__.py b/gpustack_runtime/deployer/cdi/__init__.py index a6eeace..eda9160 100644 --- a/gpustack_runtime/deployer/cdi/__init__.py +++ b/gpustack_runtime/deployer/cdi/__init__.py @@ -110,6 +110,23 @@ def generate_config( return cfg +def generate_config_by_manufacturer( + manufacturer: ManufacturerEnum, +) -> Config | None: + """ + Generate the CDI configuration for all devices of a manufacturer. + + Returns the Config object, or None if not supported. Unlike ``dump_config``, + the object is returned as-is so a caller can translate its device nodes and + mounts into plain container options. + """ + gen = _GENERATORS_MAP.get(manufacturer) + if not gen: + return None + + return gen.generate() + + def available_manufacturers() -> list[ManufacturerEnum]: """ Get a list of available manufacturers, @@ -156,6 +173,7 @@ def available_backends() -> list[str]: "available_manufacturers", "dump_config", "generate_config", + "generate_config_by_manufacturer", "manufacturer_to_cdi_kind", "manufacturer_to_runtime_env", "supported_manufacturers", diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index b171387..26c4a99 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -124,30 +124,47 @@ def generate( if not common_device_nodes: return None + # Device.appendix defaults to None and callers pass their own devices + # in, so every read goes through `or {}`. + is_a5 = any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ) + + if is_a5: + # A5's UB fabric management lives under the driver tree (ube_mgmt, + # device), siblings of lib64 that the per-subdir profile omits, so + # HCCL cannot init UB -- mount the whole driver. hccl_rootinfo.json + # is dropped: nothing here generates it, and a stale host copy makes + # rootInfo detection fail. + mount_paths = [ + "/usr/local/Ascend/driver", + "/usr/local/dcmi", + "/usr/local/bin/npu-smi", + "/var/queue_schedule", + ] + else: + mount_paths = [ + "/etc/hccl_rootinfo.json", + "/usr/local/Ascend/driver/topo", + "/usr/local/Ascend/driver/lib64", + "/usr/local/Ascend/driver/include", + "/usr/local/dcmi", + "/usr/local/bin/npu-smi", + "/var/queue_schedule", + ] + common_mounts = [] - for p in [ - "/etc/hccl_rootinfo.json", - "/usr/local/Ascend/driver/topo", - "/usr/local/Ascend/driver/lib64", - "/usr/local/Ascend/driver/include", - "/usr/local/dcmi", - "/usr/local/bin/npu-smi", - "/var/queue_schedule", - ]: + for p in mount_paths: cm = path_to_cdi_mount( path=p, ) if cm: common_mounts.append(cm) - # Device.appendix defaults to None and callers pass their own devices - # in, so every read goes through `or {}`. - if any( - get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) - == _A5_CANN_VARIANT - for dev in devices - if dev - ): + if is_a5: for pattern in _A5_UB_MOUNT_PATTERNS: ub_mounts = glob_to_cdi_mounts(pattern=pattern) if not ub_mounts: diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index e37b21d..a736b48 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -58,6 +58,7 @@ sensitive_env_var, ) from .cdi import dump_config as cdi_dump_config +from .cdi import generate_config_by_manufacturer as cdi_generate_config if TYPE_CHECKING: from collections.abc import Callable, Generator @@ -255,7 +256,7 @@ def parse_state( elif not _has_restart_policy(ci): d_init_state = WorkloadStatusStateEnum.INITIALIZING - return d_init_state if d_init_state else d_run_state + return d_init_state or d_run_state def __init__( self, @@ -776,7 +777,112 @@ def _append_container_mounts( mount_binding.append(binding) if mount_binding: - create_options["mounts"] = mount_binding + create_options["mounts"] = ( + create_options.get("mounts") or [] + ) + mount_binding + + @staticmethod + def _apply_compute_resource( + create_options: dict[str, Any], + r_k: str, + r_v: Any, + ) -> bool: + """Apply a cpu/memory resource request; return True if handled.""" + if r_k == "cpu": + if isinstance(r_v, int | float): + create_options["cpu_shares"] = ceil(r_v * 1024) + elif isinstance(r_v, str) and r_v.isdigit(): + create_options["cpu_shares"] = ceil(float(r_v) * 1024) + return True + if r_k == "memory": + if isinstance(r_v, int): + create_options["mem_limit"] = r_v + create_options["mem_reservation"] = r_v + create_options["memswap_limit"] = r_v + elif isinstance(r_v, str): + v = r_v.lower().removesuffix("i") + create_options["mem_limit"] = v + create_options["mem_reservation"] = v + create_options["memswap_limit"] = v + return True + return False + + def _inject_plain_devices( + self, + create_options: dict[str, Any], + runtime_envs: list[str], + resource_values: list[str], + r_v: str, + privileged: bool, + ): + """ + Plain device passthrough for one resource request: inject each mapped + manufacturer's device nodes and mounts, skipping the visible-devices env. + """ + if r_v == "all": + create_options["privileged"] = True + want_all = r_v == "all" or privileged + for ren in runtime_envs: + self._inject_devices_plain( + create_options, + self.get_manufacturer(ren), + resource_values, + want_all, + ) + + @staticmethod + def _inject_devices_plain( + create_options: dict[str, Any], + manufacturer: Any, + resource_values: list[str], + want_all: bool, + ): + """ + Inject a manufacturer's device nodes and mounts as plain Docker devices + and bind mounts, reusing the CDI generator's knowledge. No visible-devices + env is set, so the vendor runtime does not apply device isolation. + """ + cfg = cdi_generate_config(manufacturer) + if not cfg: + return + + edits = cfg.container_edits + device_nodes = list(edits.device_nodes or []) if edits else [] + wanted = set(resource_values) + for dev in cfg.devices: + if (want_all and dev.name == "all") or ( + not want_all and dev.name in wanted + ): + device_nodes.extend(dev.container_edits.device_nodes or []) + + devices = create_options.get("devices") or [] + seen = {d.split(":")[1] if ":" in d else d for d in devices} + for dn in device_nodes: + if dn.path in seen: + continue + seen.add(dn.path) + devices.append( + f"{dn.host_path or dn.path}:{dn.path}:{dn.permissions or 'rwm'}", + ) + if devices: + create_options["devices"] = devices + + mounts = create_options.get("mounts") or [] + targets = {m.get("Target") for m in mounts} + for m in (edits.mounts if edits else None) or []: + if m.container_path in targets: + continue + targets.add(m.container_path) + mounts.append( + docker.types.Mount( + type="bind", + source=m.host_path, + target=m.container_path, + read_only=bool(m.options) and "ro" in m.options, + ), + ) + if mounts: + create_options["mounts"] = mounts @staticmethod def _parameterize_healthcheck( @@ -994,29 +1100,15 @@ def _create_containers( # Parameterize resources. if c.resources: - cdi = ( - envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY.lower() - == "cdi" - ) - fmt = "plain" if not cdi else "cdi" + policy = ( + envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or "" + ).lower() + cdi = policy == "cdi" + plain_device = policy == "device" + fmt = "cdi" if cdi else "plain" for r_k, r_v in c.resources.items(): - if r_k == "cpu": - if isinstance(r_v, int | float): - create_options["cpu_shares"] = ceil(r_v * 1024) - elif isinstance(r_v, str) and r_v.isdigit(): - create_options["cpu_shares"] = ceil(float(r_v) * 1024) - continue - if r_k == "memory": - if isinstance(r_v, int): - create_options["mem_limit"] = r_v - create_options["mem_reservation"] = r_v - create_options["memswap_limit"] = r_v - elif isinstance(r_v, str): - v = r_v.lower().removesuffix("i") - create_options["mem_limit"] = v - create_options["mem_reservation"] = v - create_options["memswap_limit"] = v + if self._apply_compute_resource(create_options, r_k, r_v): continue if ( @@ -1038,6 +1130,20 @@ def _create_containers( privileged = create_options.get("privileged", False) resource_values = [x.strip() for x in r_v.split(",")] + # Plain device passthrough: inject device nodes and mounts + # directly, skipping the visible-devices env. The env drives + # the vendor runtime's device isolation, which breaks the + # Ascend A5 UB fabric; a bare device passthrough does not. + if plain_device: + self._inject_plain_devices( + create_options, + runtime_envs, + resource_values, + r_v, + privileged, + ) + continue + # Generate CDI config if not yet. if cdi and envs.GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE: for ren in runtime_envs: @@ -1313,6 +1419,20 @@ def _prepare_mirrored_deployment(self): for k, v in mirrored_envs.items() if k not in igs } + # Non-Env policies inject devices without the visible-devices env, so a + # mirrored one would defeat that -- and re-trigger the vendor runtime's + # device isolation (breaks e.g. Ascend A5 UB). Drop those env names. + if ( + envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or "Env" + ).lower() != "env": + visible_envs = set( + envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES.values(), + ) + for names in envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_BACKEND_VISIBLE_DEVICES.values(): + visible_envs.update(names if isinstance(names, list) else [names]) + mirrored_envs = { + k: v for k, v in mirrored_envs.items() if k not in visible_envs + } ## - Container customized mounts mirrored_mounts: list[dict[str, Any]] = [ # Always filter out Docker Socket mount. @@ -1402,7 +1522,7 @@ def mutate_create_options(create_options: dict[str, Any]) -> dict[str, Any]: c_devices: list[dict[str, Any]] = [] for c_device in create_options.get("devices") or []: sp = c_device.split(":") - c_device.append( + c_devices.append( { "PathOnHost": sp[0], "PathInContainer": sp[1] if len(sp) > 1 else sp[0], diff --git a/gpustack_runtime/envs.py b/gpustack_runtime/envs.py index 3cfa266..e063286 100644 --- a/gpustack_runtime/envs.py +++ b/gpustack_runtime/envs.py @@ -247,9 +247,10 @@ """ GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY: str | None = None """ - Resource injection policy for the Docker deployer (e.g., Env, CDI). + Resource injection policy for the Docker deployer (e.g., Env, CDI, Device). `Env`: Injects resources using standard environment variable, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES`. `CDI`: Injects resources using CDI, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_CDI`. + `Device`: Injects device nodes and mounts directly, without the visible-devices env or a CDI-capable Docker. Suits hosts where the visible-devices env breaks the accelerator (e.g. Ascend A5 UB fabric). """ GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE: bool = True """ @@ -647,7 +648,7 @@ getenv( "GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY", ), - options=["Env", "CDI"], + options=["Env", "CDI", "Device"], default="Env", ), "GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE": lambda: ternary(