diff --git a/gpustack_runtime/deployer/cdi/__init__.py b/gpustack_runtime/deployer/cdi/__init__.py index a6eeace..eda9160 100644 --- a/gpustack_runtime/deployer/cdi/__init__.py +++ b/gpustack_runtime/deployer/cdi/__init__.py @@ -110,6 +110,23 @@ def generate_config( return cfg +def generate_config_by_manufacturer( + manufacturer: ManufacturerEnum, +) -> Config | None: + """ + Generate the CDI configuration for all devices of a manufacturer. + + Returns the Config object, or None if not supported. Unlike ``dump_config``, + the object is returned as-is so a caller can translate its device nodes and + mounts into plain container options. + """ + gen = _GENERATORS_MAP.get(manufacturer) + if not gen: + return None + + return gen.generate() + + def available_manufacturers() -> list[ManufacturerEnum]: """ Get a list of available manufacturers, @@ -156,6 +173,7 @@ def available_backends() -> list[str]: "available_manufacturers", "dump_config", "generate_config", + "generate_config_by_manufacturer", "manufacturer_to_cdi_kind", "manufacturer_to_runtime_env", "supported_manufacturers", diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index b171387..26c4a99 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -124,30 +124,47 @@ def generate( if not common_device_nodes: return None + # Device.appendix defaults to None and callers pass their own devices + # in, so every read goes through `or {}`. + is_a5 = any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ) + + if is_a5: + # A5's UB fabric management lives under the driver tree (ube_mgmt, + # device), siblings of lib64 that the per-subdir profile omits, so + # HCCL cannot init UB -- mount the whole driver. hccl_rootinfo.json + # is dropped: nothing here generates it, and a stale host copy makes + # rootInfo detection fail. + mount_paths = [ + "/usr/local/Ascend/driver", + "/usr/local/dcmi", + "/usr/local/bin/npu-smi", + "/var/queue_schedule", + ] + else: + mount_paths = [ + "/etc/hccl_rootinfo.json", + "/usr/local/Ascend/driver/topo", + "/usr/local/Ascend/driver/lib64", + "/usr/local/Ascend/driver/include", + "/usr/local/dcmi", + "/usr/local/bin/npu-smi", + "/var/queue_schedule", + ] + common_mounts = [] - for p in [ - "/etc/hccl_rootinfo.json", - "/usr/local/Ascend/driver/topo", - "/usr/local/Ascend/driver/lib64", - "/usr/local/Ascend/driver/include", - "/usr/local/dcmi", - "/usr/local/bin/npu-smi", - "/var/queue_schedule", - ]: + for p in mount_paths: cm = path_to_cdi_mount( path=p, ) if cm: common_mounts.append(cm) - # Device.appendix defaults to None and callers pass their own devices - # in, so every read goes through `or {}`. - if any( - get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) - == _A5_CANN_VARIANT - for dev in devices - if dev - ): + if is_a5: for pattern in _A5_UB_MOUNT_PATTERNS: ub_mounts = glob_to_cdi_mounts(pattern=pattern) if not ub_mounts: diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index e37b21d..a736b48 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -58,6 +58,7 @@ sensitive_env_var, ) from .cdi import dump_config as cdi_dump_config +from .cdi import generate_config_by_manufacturer as cdi_generate_config if TYPE_CHECKING: from collections.abc import Callable, Generator @@ -255,7 +256,7 @@ def parse_state( elif not _has_restart_policy(ci): d_init_state = WorkloadStatusStateEnum.INITIALIZING - return d_init_state if d_init_state else d_run_state + return d_init_state or d_run_state def __init__( self, @@ -776,7 +777,112 @@ def _append_container_mounts( mount_binding.append(binding) if mount_binding: - create_options["mounts"] = mount_binding + create_options["mounts"] = ( + create_options.get("mounts") or [] + ) + mount_binding + + @staticmethod + def _apply_compute_resource( + create_options: dict[str, Any], + r_k: str, + r_v: Any, + ) -> bool: + """Apply a cpu/memory resource request; return True if handled.""" + if r_k == "cpu": + if isinstance(r_v, int | float): + create_options["cpu_shares"] = ceil(r_v * 1024) + elif isinstance(r_v, str) and r_v.isdigit(): + create_options["cpu_shares"] = ceil(float(r_v) * 1024) + return True + if r_k == "memory": + if isinstance(r_v, int): + create_options["mem_limit"] = r_v + create_options["mem_reservation"] = r_v + create_options["memswap_limit"] = r_v + elif isinstance(r_v, str): + v = r_v.lower().removesuffix("i") + create_options["mem_limit"] = v + create_options["mem_reservation"] = v + create_options["memswap_limit"] = v + return True + return False + + def _inject_plain_devices( + self, + create_options: dict[str, Any], + runtime_envs: list[str], + resource_values: list[str], + r_v: str, + privileged: bool, + ): + """ + Plain device passthrough for one resource request: inject each mapped + manufacturer's device nodes and mounts, skipping the visible-devices env. + """ + if r_v == "all": + create_options["privileged"] = True + want_all = r_v == "all" or privileged + for ren in runtime_envs: + self._inject_devices_plain( + create_options, + self.get_manufacturer(ren), + resource_values, + want_all, + ) + + @staticmethod + def _inject_devices_plain( + create_options: dict[str, Any], + manufacturer: Any, + resource_values: list[str], + want_all: bool, + ): + """ + Inject a manufacturer's device nodes and mounts as plain Docker devices + and bind mounts, reusing the CDI generator's knowledge. No visible-devices + env is set, so the vendor runtime does not apply device isolation. + """ + cfg = cdi_generate_config(manufacturer) + if not cfg: + return + + edits = cfg.container_edits + device_nodes = list(edits.device_nodes or []) if edits else [] + wanted = set(resource_values) + for dev in cfg.devices: + if (want_all and dev.name == "all") or ( + not want_all and dev.name in wanted + ): + device_nodes.extend(dev.container_edits.device_nodes or []) + + devices = create_options.get("devices") or [] + seen = {d.split(":")[1] if ":" in d else d for d in devices} + for dn in device_nodes: + if dn.path in seen: + continue + seen.add(dn.path) + devices.append( + f"{dn.host_path or dn.path}:{dn.path}:{dn.permissions or 'rwm'}", + ) + if devices: + create_options["devices"] = devices + + mounts = create_options.get("mounts") or [] + targets = {m.get("Target") for m in mounts} + for m in (edits.mounts if edits else None) or []: + if m.container_path in targets: + continue + targets.add(m.container_path) + mounts.append( + docker.types.Mount( + type="bind", + source=m.host_path, + target=m.container_path, + read_only=bool(m.options) and "ro" in m.options, + ), + ) + if mounts: + create_options["mounts"] = mounts @staticmethod def _parameterize_healthcheck( @@ -994,29 +1100,15 @@ def _create_containers( # Parameterize resources. if c.resources: - cdi = ( - envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY.lower() - == "cdi" - ) - fmt = "plain" if not cdi else "cdi" + policy = ( + envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or "" + ).lower() + cdi = policy == "cdi" + plain_device = policy == "device" + fmt = "cdi" if cdi else "plain" for r_k, r_v in c.resources.items(): - if r_k == "cpu": - if isinstance(r_v, int | float): - create_options["cpu_shares"] = ceil(r_v * 1024) - elif isinstance(r_v, str) and r_v.isdigit(): - create_options["cpu_shares"] = ceil(float(r_v) * 1024) - continue - if r_k == "memory": - if isinstance(r_v, int): - create_options["mem_limit"] = r_v - create_options["mem_reservation"] = r_v - create_options["memswap_limit"] = r_v - elif isinstance(r_v, str): - v = r_v.lower().removesuffix("i") - create_options["mem_limit"] = v - create_options["mem_reservation"] = v - create_options["memswap_limit"] = v + if self._apply_compute_resource(create_options, r_k, r_v): continue if ( @@ -1038,6 +1130,20 @@ def _create_containers( privileged = create_options.get("privileged", False) resource_values = [x.strip() for x in r_v.split(",")] + # Plain device passthrough: inject device nodes and mounts + # directly, skipping the visible-devices env. The env drives + # the vendor runtime's device isolation, which breaks the + # Ascend A5 UB fabric; a bare device passthrough does not. + if plain_device: + self._inject_plain_devices( + create_options, + runtime_envs, + resource_values, + r_v, + privileged, + ) + continue + # Generate CDI config if not yet. if cdi and envs.GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE: for ren in runtime_envs: @@ -1313,6 +1419,20 @@ def _prepare_mirrored_deployment(self): for k, v in mirrored_envs.items() if k not in igs } + # Non-Env policies inject devices without the visible-devices env, so a + # mirrored one would defeat that -- and re-trigger the vendor runtime's + # device isolation (breaks e.g. Ascend A5 UB). Drop those env names. + if ( + envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or "Env" + ).lower() != "env": + visible_envs = set( + envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES.values(), + ) + for names in envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_BACKEND_VISIBLE_DEVICES.values(): + visible_envs.update(names if isinstance(names, list) else [names]) + mirrored_envs = { + k: v for k, v in mirrored_envs.items() if k not in visible_envs + } ## - Container customized mounts mirrored_mounts: list[dict[str, Any]] = [ # Always filter out Docker Socket mount. @@ -1402,7 +1522,7 @@ def mutate_create_options(create_options: dict[str, Any]) -> dict[str, Any]: c_devices: list[dict[str, Any]] = [] for c_device in create_options.get("devices") or []: sp = c_device.split(":") - c_device.append( + c_devices.append( { "PathOnHost": sp[0], "PathInContainer": sp[1] if len(sp) > 1 else sp[0], diff --git a/gpustack_runtime/envs.py b/gpustack_runtime/envs.py index 3cfa266..e063286 100644 --- a/gpustack_runtime/envs.py +++ b/gpustack_runtime/envs.py @@ -247,9 +247,10 @@ """ GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY: str | None = None """ - Resource injection policy for the Docker deployer (e.g., Env, CDI). + Resource injection policy for the Docker deployer (e.g., Env, CDI, Device). `Env`: Injects resources using standard environment variable, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES`. `CDI`: Injects resources using CDI, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_CDI`. + `Device`: Injects device nodes and mounts directly, without the visible-devices env or a CDI-capable Docker. Suits hosts where the visible-devices env breaks the accelerator (e.g. Ascend A5 UB fabric). """ GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE: bool = True """ @@ -647,7 +648,7 @@ getenv( "GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY", ), - options=["Env", "CDI"], + options=["Env", "CDI", "Device"], default="Env", ), "GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE": lambda: ternary(