feat(ascend): add Device injection policy and fix A5 driver/UB mounts

gpustack_runtime/deployer/cdi/__init__.py

@@ -110,6 +110,23 @@ def generate_config(
     return cfg
 
 
+def generate_config_by_manufacturer(
+    manufacturer: ManufacturerEnum,
+) -> Config | None:
+    """
+    Generate the CDI configuration for all devices of a manufacturer.
+
+    Returns the Config object, or None if not supported. Unlike ``dump_config``,
+    the object is returned as-is so a caller can translate its device nodes and
+    mounts into plain container options.
+    """
+    gen = _GENERATORS_MAP.get(manufacturer)
+    if not gen:
+        return None
+
+    return gen.generate()
+
+
 def available_manufacturers() -> list[ManufacturerEnum]:
     """
     Get a list of available manufacturers,
@@ -156,6 +173,7 @@ def available_backends() -> list[str]:
     "available_manufacturers",
     "dump_config",
     "generate_config",
+    "generate_config_by_manufacturer",
     "manufacturer_to_cdi_kind",
     "manufacturer_to_runtime_env",
     "supported_manufacturers",

gpustack_runtime/deployer/cdi/ascend.py

@@ -124,30 +124,47 @@ def generate(
         if not common_device_nodes:
             return None
 
+        # Device.appendix defaults to None and callers pass their own devices
+        # in, so every read goes through `or {}`.
+        is_a5 = any(
+            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+            == _A5_CANN_VARIANT
+            for dev in devices
+            if dev
+        )
+
+        if is_a5:
+            # A5's UB fabric management lives under the driver tree (ube_mgmt,
+            # device), siblings of lib64 that the per-subdir profile omits, so
+            # HCCL cannot init UB -- mount the whole driver. hccl_rootinfo.json
+            # is dropped: nothing here generates it, and a stale host copy makes
+            # rootInfo detection fail.
+            mount_paths = [
+                "/usr/local/Ascend/driver",
+                "/usr/local/dcmi",
+                "/usr/local/bin/npu-smi",
+                "/var/queue_schedule",
+            ]
+        else:
+            mount_paths = [
+                "/etc/hccl_rootinfo.json",
+                "/usr/local/Ascend/driver/topo",
+                "/usr/local/Ascend/driver/lib64",
+                "/usr/local/Ascend/driver/include",
+                "/usr/local/dcmi",
+                "/usr/local/bin/npu-smi",
+                "/var/queue_schedule",
+            ]
+
         common_mounts = []
-        for p in [
-            "/etc/hccl_rootinfo.json",
-            "/usr/local/Ascend/driver/topo",
-            "/usr/local/Ascend/driver/lib64",
-            "/usr/local/Ascend/driver/include",
-            "/usr/local/dcmi",
-            "/usr/local/bin/npu-smi",
-            "/var/queue_schedule",
-        ]:
+        for p in mount_paths:
             cm = path_to_cdi_mount(
                 path=p,
             )
             if cm:
                 common_mounts.append(cm)
 
-        # Device.appendix defaults to None and callers pass their own devices
-        # in, so every read goes through `or {}`.
-        if any(
-            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
-            == _A5_CANN_VARIANT
-            for dev in devices
-            if dev
-        ):
+        if is_a5:
             for pattern in _A5_UB_MOUNT_PATTERNS:
                 ub_mounts = glob_to_cdi_mounts(pattern=pattern)
                 if not ub_mounts:

gpustack_runtime/deployer/docker.py

@@ -58,6 +58,7 @@
     sensitive_env_var,
 )
 from .cdi import dump_config as cdi_dump_config
+from .cdi import generate_config_by_manufacturer as cdi_generate_config
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator
@@ -255,7 +256,7 @@ def parse_state(
             elif not _has_restart_policy(ci):
                 d_init_state = WorkloadStatusStateEnum.INITIALIZING
 
-        return d_init_state if d_init_state else d_run_state
+        return d_init_state or d_run_state
 
     def __init__(
         self,
@@ -776,7 +777,112 @@ def _append_container_mounts(
                 mount_binding.append(binding)
 
         if mount_binding:
-            create_options["mounts"] = mount_binding
+            create_options["mounts"] = (
+                create_options.get("mounts") or []
+            ) + mount_binding
+
+    @staticmethod
+    def _apply_compute_resource(
+        create_options: dict[str, Any],
+        r_k: str,
+        r_v: Any,
+    ) -> bool:
+        """Apply a cpu/memory resource request; return True if handled."""
+        if r_k == "cpu":
+            if isinstance(r_v, int | float):
+                create_options["cpu_shares"] = ceil(r_v * 1024)
+            elif isinstance(r_v, str) and r_v.isdigit():
+                create_options["cpu_shares"] = ceil(float(r_v) * 1024)
+            return True
+        if r_k == "memory":
+            if isinstance(r_v, int):
+                create_options["mem_limit"] = r_v
+                create_options["mem_reservation"] = r_v
+                create_options["memswap_limit"] = r_v
+            elif isinstance(r_v, str):
+                v = r_v.lower().removesuffix("i")
+                create_options["mem_limit"] = v
+                create_options["mem_reservation"] = v
+                create_options["memswap_limit"] = v
+            return True
+        return False
+
+    def _inject_plain_devices(
+        self,
+        create_options: dict[str, Any],
+        runtime_envs: list[str],
+        resource_values: list[str],
+        r_v: str,
+        privileged: bool,
+    ):
+        """
+        Plain device passthrough for one resource request: inject each mapped
+        manufacturer's device nodes and mounts, skipping the visible-devices env.
+        """
+        if r_v == "all":
+            create_options["privileged"] = True
+        want_all = r_v == "all" or privileged
+        for ren in runtime_envs:
+            self._inject_devices_plain(
+                create_options,
+                self.get_manufacturer(ren),
+                resource_values,
+                want_all,
+            )
+
+    @staticmethod
+    def _inject_devices_plain(
+        create_options: dict[str, Any],
+        manufacturer: Any,
+        resource_values: list[str],
+        want_all: bool,
+    ):
+        """
+        Inject a manufacturer's device nodes and mounts as plain Docker devices
+        and bind mounts, reusing the CDI generator's knowledge. No visible-devices
+        env is set, so the vendor runtime does not apply device isolation.
+        """
+        cfg = cdi_generate_config(manufacturer)
+        if not cfg:
+            return
+
+        edits = cfg.container_edits
+        device_nodes = list(edits.device_nodes or []) if edits else []
+        wanted = set(resource_values)
+        for dev in cfg.devices:
+            if (want_all and dev.name == "all") or (
+                not want_all and dev.name in wanted
+            ):
+                device_nodes.extend(dev.container_edits.device_nodes or [])
+
+        devices = create_options.get("devices") or []
+        seen = {d.split(":")[1] if ":" in d else d for d in devices}
+        for dn in device_nodes:
+            if dn.path in seen:
+                continue
+            seen.add(dn.path)
+            devices.append(
+                f"{dn.host_path or dn.path}:{dn.path}:{dn.permissions or 'rwm'}",
+            )
+        if devices:
+            create_options["devices"] = devices
+
+        mounts = create_options.get("mounts") or []
+        targets = {m.get("Target") for m in mounts}
+        for m in (edits.mounts if edits else None) or []:
+            if m.container_path in targets:
+                continue
+            targets.add(m.container_path)
+            mounts.append(
+                docker.types.Mount(
+                    type="bind",
+                    source=m.host_path,
+                    target=m.container_path,
+                    read_only=bool(m.options) and "ro" in m.options,
+                ),
+            )
+        if mounts:
+            create_options["mounts"] = mounts
 
     @staticmethod
     def _parameterize_healthcheck(
@@ -994,29 +1100,15 @@ def _create_containers(
 
             # Parameterize resources.
             if c.resources:
-                cdi = (
-                    envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY.lower()
-                    == "cdi"
-                )
-                fmt = "plain" if not cdi else "cdi"
+                policy = (
+                    envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or ""
+                ).lower()
+                cdi = policy == "cdi"
+                plain_device = policy == "device"
+                fmt = "cdi" if cdi else "plain"
 
                 for r_k, r_v in c.resources.items():
-                    if r_k == "cpu":
-                        if isinstance(r_v, int | float):
-                            create_options["cpu_shares"] = ceil(r_v * 1024)
-                        elif isinstance(r_v, str) and r_v.isdigit():
-                            create_options["cpu_shares"] = ceil(float(r_v) * 1024)
-                        continue
-                    if r_k == "memory":
-                        if isinstance(r_v, int):
-                            create_options["mem_limit"] = r_v
-                            create_options["mem_reservation"] = r_v
-                            create_options["memswap_limit"] = r_v
-                        elif isinstance(r_v, str):
-                            v = r_v.lower().removesuffix("i")
-                            create_options["mem_limit"] = v
-                            create_options["mem_reservation"] = v
-                            create_options["memswap_limit"] = v
+                    if self._apply_compute_resource(create_options, r_k, r_v):
                         continue
 
                     if (
@@ -1038,6 +1130,20 @@ def _create_containers(
                     privileged = create_options.get("privileged", False)
                     resource_values = [x.strip() for x in r_v.split(",")]
 
+                    # Plain device passthrough: inject device nodes and mounts
+                    # directly, skipping the visible-devices env. The env drives
+                    # the vendor runtime's device isolation, which breaks the
+                    # Ascend A5 UB fabric; a bare device passthrough does not.
+                    if plain_device:
+                        self._inject_plain_devices(
+                            create_options,
+                            runtime_envs,
+                            resource_values,
+                            r_v,
+                            privileged,
+                        )
+                        continue
+
                     # Generate CDI config if not yet.
                     if cdi and envs.GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE:
                         for ren in runtime_envs:
@@ -1313,6 +1419,20 @@ def _prepare_mirrored_deployment(self):
                 for k, v in mirrored_envs.items()
                 if k not in igs
             }
+        # Non-Env policies inject devices without the visible-devices env, so a
+        # mirrored one would defeat that -- and re-trigger the vendor runtime's
+        # device isolation (breaks e.g. Ascend A5 UB). Drop those env names.
+        if (
+            envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY or "Env"
+        ).lower() != "env":
+            visible_envs = set(
+                envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES.values(),
+            )
+            for names in envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_BACKEND_VISIBLE_DEVICES.values():
+                visible_envs.update(names if isinstance(names, list) else [names])
+            mirrored_envs = {
+                k: v for k, v in mirrored_envs.items() if k not in visible_envs
+            }
         ## - Container customized mounts
         mirrored_mounts: list[dict[str, Any]] = [
             # Always filter out Docker Socket mount.
@@ -1402,7 +1522,7 @@ def mutate_create_options(create_options: dict[str, Any]) -> dict[str, Any]:
                 c_devices: list[dict[str, Any]] = []
                 for c_device in create_options.get("devices") or []:
                     sp = c_device.split(":")
-                    c_device.append(
+                    c_devices.append(
                         {
                             "PathOnHost": sp[0],
                             "PathInContainer": sp[1] if len(sp) > 1 else sp[0],

gpustack_runtime/envs.py

@@ -247,9 +247,10 @@
     """
     GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY: str | None = None
     """
-    Resource injection policy for the Docker deployer (e.g., Env, CDI).
+    Resource injection policy for the Docker deployer (e.g., Env, CDI, Device).
     `Env`: Injects resources using standard environment variable, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES`.
     `CDI`: Injects resources using CDI, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_CDI`.
+    `Device`: Injects device nodes and mounts directly, without the visible-devices env or a CDI-capable Docker. Suits hosts where the visible-devices env breaks the accelerator (e.g. Ascend A5 UB fabric).
     """
     GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE: bool = True
     """
@@ -647,7 +648,7 @@
         getenv(
             "GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY",
         ),
-        options=["Env", "CDI"],
+        options=["Env", "CDI", "Device"],
         default="Env",
     ),
     "GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE": lambda: ternary(