fix(ascend): stop A5 from receiving a ranktable its HCCL rejects

gpustack_runtime/deployer/cdi/ascend.py

@@ -1,6 +1,9 @@
 from __future__ import annotations as __future_annotations__
 
+import json
 import logging
+from functools import cache
+from pathlib import Path
 
 from ...detector import (
     Devices,
@@ -32,6 +35,28 @@
 earlier generations reach through the shared memory device.
 """
 
+_HCCL_RANKTABLE_PATH = "/etc/hccl_rootinfo.json"
+"""
+The host ranktable ("rootinfo") file. Optional, user-maintained state: it
+belongs to no driver package, and the vendor documents it as generated by
+mindcluster-tools and mounted only when present, see
+https://gitcode.com/Ascend/mind-cluster/blob/master/docs/en/scheduling/references/appendix.md.
+"""
+
+_A5_RANKTABLE_VERSION = "2.0"
+"""
+The ranktable version the A5 generation requires.
+
+The format is bound to the chip generation -- 1.0 for A2, 1.2 for A3, 2.0 for
+A5, see ascend-operator/pkg/ranktable/common/common.go:37-38 and
+ranktable/v2dot0/ranktable.go:27. A5 loads libhccl_v2.so, which rejects an
+older table outright rather than ignoring it.
+
+Only the A5 row of that mapping has a measured failure behind it, so only A5 is
+acted on. An A3 host carrying a 1.2 table is presumed correct rather than
+warned about, because nothing here established otherwise.
+"""
+
 _A5_UB_MOUNT_PATTERNS = [
     "/usr/lib64/libummu*",
     "/usr/lib64/liburma*",
@@ -124,30 +149,51 @@ def generate(
         if not common_device_nodes:
             return None
 
-        common_mounts = []
-        for p in [
-            "/etc/hccl_rootinfo.json",
+        # Device.appendix defaults to None and callers pass their own devices
+        # in, so every read goes through `or {}`.
+        is_a5 = any(
+            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+            == _A5_CANN_VARIANT
+            for dev in devices
+            if dev
+        )
+
+        mount_paths = [
+            _HCCL_RANKTABLE_PATH,
             "/usr/local/Ascend/driver/topo",
             "/usr/local/Ascend/driver/lib64",
             "/usr/local/Ascend/driver/include",
             "/usr/local/dcmi",
             "/usr/local/bin/npu-smi",
             "/var/queue_schedule",
-        ]:
+        ]
+
+        if is_a5:
+            # A5 loads libhccl_v2.so, which accepts a 2.0 ranktable only. A host
+            # file left over from an A2 fleet is 1.0, and HCCL refuses it with
+            # Config_Error_Ranktable(EI0014) rather than falling back, so every
+            # multi-card init dies. Nothing here writes the file, so the only
+            # lever is not mounting it -- verified by isolation: mounting only
+            # this file into an otherwise-working container reproduces EI0014,
+            # mounting only driver/topo does not.
+            #
+            # This is single-node evidence. A multi-node deployment does need a
+            # ranktable; when that lands it has to be a 2.0 one, not this.
+            #
+            # Only GPUStack's own mount list is covered here. Under the Env
+            # policy ascend-docker-runtime mounts the file itself, which
+            # `warn_incompatible_ranktable` reports and cannot prevent.
+            mount_paths.remove(_HCCL_RANKTABLE_PATH)
+
+        common_mounts = []
+        for p in mount_paths:
             cm = path_to_cdi_mount(
                 path=p,
             )
             if cm:
                 common_mounts.append(cm)
 
-        # Device.appendix defaults to None and callers pass their own devices
-        # in, so every read goes through `or {}`.
-        if any(
-            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
-            == _A5_CANN_VARIANT
-            for dev in devices
-            if dev
-        ):
+        if is_a5:
             for pattern in _A5_UB_MOUNT_PATTERNS:
                 ub_mounts = glob_to_cdi_mounts(pattern=pattern)
                 if not ub_mounts:
@@ -225,3 +271,75 @@ def generate(
                 mounts=common_mounts,
             ),
         )
+
+
+def read_ranktable_version(path: str = _HCCL_RANKTABLE_PATH) -> str | None:
+    """
+    Read the ``version`` of the host ranktable file.
+
+    The path is resolved as this process sees it, exactly as the mount list
+    above resolves the paths it emits. Both therefore agree on whether the file
+    exists.
+
+    Args:
+        path:
+            The ranktable path to read.
+
+    Returns:
+        The declared version, or None if the file is absent, unreadable or not
+        a JSON object carrying a string version.
+
+    """
+    try:
+        content = json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+    if not isinstance(content, dict):
+        return None
+
+    version = content.get("version")
+    return version if isinstance(version, str) else None
+
+
+@cache
+def warn_incompatible_ranktable() -> str | None:
+    """
+    Warn once if this A5 host carries a ranktable A5's HCCL will reject.
+
+    ascend-docker-runtime mounts the host ranktable whenever the file is there
+    -- its `addUBMount` is a bare `os.Stat` with no version check -- so under
+    the default Env policy GPUStack cannot prevent the mount. Reporting is the
+    only available action: the operator has to remove the file or replace it
+    with a 2.0 one.
+
+    Returns:
+        The warning message emitted, or None if there is nothing to warn about.
+
+    """
+    # Ordered so a host without the file pays nothing: the file is absent on
+    # almost every non-Ascend host, and detection is the expensive half.
+    version = read_ranktable_version()
+    if version is None or version == _A5_RANKTABLE_VERSION:
+        return None
+
+    devices = detect_devices(manufacturer=ManufacturerEnum.ASCEND)
+    if not devices or not any(
+        get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+        == _A5_CANN_VARIANT
+        for dev in devices
+        if dev
+    ):
+        # An older generation's own ranktable is correct for it.
+        return None
+
+    msg = (
+        f"Host ranktable {_HCCL_RANKTABLE_PATH} declares version {version!r}, "
+        f"but this A5 (Ascend 950) host needs {_A5_RANKTABLE_VERSION!r}. "
+        f"Under the default Env policy ascend-docker-runtime mounts it on "
+        f"presence and HCCL refuses it with Config_Error_Ranktable(EI0014), "
+        f"failing every multi-card workload. Remove the file or replace it "
+        f"with a {_A5_RANKTABLE_VERSION} table."
+    )
+    logger.warning(msg)
+    return msg

gpustack_runtime/deployer/docker.py

@@ -58,6 +58,9 @@
     sensitive_env_var,
 )
 from .cdi import dump_config as cdi_dump_config
+from .cdi.ascend import (
+    warn_incompatible_ranktable as warn_incompatible_ascend_ranktable,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator
@@ -1568,6 +1571,9 @@ def _create(self, workload: WorkloadPlan):
             raise TypeError(msg)
 
         self._prepare_mirrored_deployment()
+        # ascend-docker-runtime mounts the host ranktable itself, so this can
+        # only report, not prevent. Cached, so it costs one check per process.
+        warn_incompatible_ascend_ranktable()
 
         if isinstance(workload, WorkloadPlan):
             workload = DockerWorkloadPlan(**workload.__dict__)