From 535e7a45bfbcf29e97f5edbc8fcc42c37bea0ba4 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sun, 6 Sep 2026 21:54:56 +0800 Subject: [PATCH 1/2] fix(ascend): stop A5 from receiving a ranktable its HCCL rejects Supersedes #25. yxf0314's diagnosis was right that multi-card A5 needed a change; this takes the part of it that holds up, and drops the rest. A5 loads libhccl_v2.so, which accepts a 2.0 ranktable only. Ranktable format is bound to the chip generation -- 1.0 for A2, 1.2 for A3, 2.0 for A5 -- so a table left over from an older fleet is refused, not ignored: HCCL fails with Config_Error_Ranktable(EI0014) and multi-card init never happens. Isolated on an 8x Ascend950PR host: mounting only /etc/hccl_rootinfo.json into an otherwise working container reproduces EI0014, mounting only driver/topo does not. Two things follow, and nothing else does. GPUStack stops mounting the file on A5. This is its own mount list only, so it covers the CDI path. Older generations keep the file, which is correct for them. Under the default Env policy the mount is ascend-docker-runtime's -- addUBMount is a bare os.Stat with no version check -- so GPUStack cannot prevent it and reporting is the only available action. A one-shot warning names EI0014 and says the file must be absent or 2.0. An absent file is the healthy host and stays silent; so does an older generation with its own table, since only the A5 row of the generation mapping has a measured failure behind it. This is defence, not repair: with the host file moved aside, unmodified gpustack-runtime already runs GLM-5.3-Flash TP=8 under the default policy. What it stops is a node's leftover ranktable silently killing multi-card workloads on it. Deliberately not included, having been measured and found unnecessary: A Device injection policy. It adds a configuration knob that does nothing by default, on the Env path where the vendor runtime does the mounting -- so it would not protect anyone who had not already been told to configure it. Its isolation is also nominal in the shape GPUStack deploys in, since a privileged container sees every device node whatever it was granted. Mounting the whole /usr/local/Ascend/driver for A5. driver/ube_mgmt is a staging directory for upgrade-tool holding one zero-byte lock file; no library under driver/lib64 references it, and the urma symbols are already inside the lib64 mount. The tree would also carry upgrade-tool and hccn_tool into a container holding /dev/davinci_manager, plus the CA store and 2570 kernel sources. --- gpustack_runtime/deployer/cdi/ascend.py | 141 ++++++++++- gpustack_runtime/deployer/docker.py | 6 + .../deployer/test_ascend_ranktable.py | 224 ++++++++++++++++++ 3 files changed, 359 insertions(+), 12 deletions(-) create mode 100644 tests/gpustack_runtime/deployer/test_ascend_ranktable.py diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index b171387..768073d 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -1,6 +1,9 @@ from __future__ import annotations as __future_annotations__ +import json import logging +from functools import cache +from pathlib import Path from ...detector import ( Devices, @@ -32,6 +35,28 @@ earlier generations reach through the shared memory device. """ +_HCCL_RANKTABLE_PATH = "/etc/hccl_rootinfo.json" +""" +The host ranktable ("rootinfo") file. Optional, user-maintained state: it +belongs to no driver package, and the vendor documents it as generated by +mindcluster-tools and mounted only when present, see +https://gitcode.com/Ascend/mind-cluster/blob/master/docs/en/scheduling/references/appendix.md. +""" + +_A5_RANKTABLE_VERSION = "2.0" +""" +The ranktable version the A5 generation requires. + +The format is bound to the chip generation -- 1.0 for A2, 1.2 for A3, 2.0 for +A5, see ascend-operator/pkg/ranktable/common/common.go:37-38 and +ranktable/v2dot0/ranktable.go:27. A5 loads libhccl_v2.so, which rejects an +older table outright rather than ignoring it. + +Only the A5 row of that mapping has a measured failure behind it, so only A5 is +acted on. An A3 host carrying a 1.2 table is presumed correct rather than +warned about, because nothing here established otherwise. +""" + _A5_UB_MOUNT_PATTERNS = [ "/usr/lib64/libummu*", "/usr/lib64/liburma*", @@ -124,30 +149,51 @@ def generate( if not common_device_nodes: return None - common_mounts = [] - for p in [ - "/etc/hccl_rootinfo.json", + # Device.appendix defaults to None and callers pass their own devices + # in, so every read goes through `or {}`. + is_a5 = any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ) + + mount_paths = [ + _HCCL_RANKTABLE_PATH, "/usr/local/Ascend/driver/topo", "/usr/local/Ascend/driver/lib64", "/usr/local/Ascend/driver/include", "/usr/local/dcmi", "/usr/local/bin/npu-smi", "/var/queue_schedule", - ]: + ] + + if is_a5: + # A5 loads libhccl_v2.so, which accepts a 2.0 ranktable only. A host + # file left over from an A2 fleet is 1.0, and HCCL refuses it with + # Config_Error_Ranktable(EI0014) rather than falling back, so every + # multi-card init dies. Nothing here writes the file, so the only + # lever is not mounting it -- verified by isolation: mounting only + # this file into an otherwise-working container reproduces EI0014, + # mounting only driver/topo does not. + # + # This is single-node evidence. A multi-node deployment does need a + # ranktable; when that lands it has to be a 2.0 one, not this. + # + # Only GPUStack's own mount list is covered here. Under the Env + # policy ascend-docker-runtime mounts the file itself, which + # `warn_incompatible_ranktable` reports and cannot prevent. + mount_paths.remove(_HCCL_RANKTABLE_PATH) + + common_mounts = [] + for p in mount_paths: cm = path_to_cdi_mount( path=p, ) if cm: common_mounts.append(cm) - # Device.appendix defaults to None and callers pass their own devices - # in, so every read goes through `or {}`. - if any( - get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) - == _A5_CANN_VARIANT - for dev in devices - if dev - ): + if is_a5: for pattern in _A5_UB_MOUNT_PATTERNS: ub_mounts = glob_to_cdi_mounts(pattern=pattern) if not ub_mounts: @@ -225,3 +271,74 @@ def generate( mounts=common_mounts, ), ) + + +def read_ranktable_version(path: str = _HCCL_RANKTABLE_PATH) -> str | None: + """ + Read the ``version`` of the host ranktable file. + + The path is resolved as this process sees it, exactly as the mount list + above resolves the paths it emits. Both therefore agree on whether the file + exists. + + Args: + path: + The ranktable path to read. + + Returns: + The declared version, or None if the file is absent, unreadable or not + a JSON object carrying a string version. + + """ + try: + content = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + if not isinstance(content, dict): + return None + + version = content.get("version") + return version if isinstance(version, str) else None + + +@cache +def warn_incompatible_ranktable() -> str | None: + """ + Warn once if this A5 host carries a ranktable A5's HCCL will reject. + + ascend-docker-runtime mounts the host ranktable whenever the file is there + -- its `addUBMount` is a bare `os.Stat` with no version check -- so under + the default Env policy GPUStack cannot prevent the mount. Reporting is the + only available action: the operator has to remove the file or replace it + with a 2.0 one. + + Returns: + The warning message emitted, or None if there is nothing to warn about. + + """ + # Ordered so a host without the file pays nothing: the file is absent on + # almost every non-Ascend host, and detection is the expensive half. + version = read_ranktable_version() + if version is None or version == _A5_RANKTABLE_VERSION: + return None + + devices = detect_devices(manufacturer=ManufacturerEnum.ASCEND) + if not devices or not any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ): + # An older generation's own ranktable is correct for it. + return None + + msg = ( + f"Host ranktable {_HCCL_RANKTABLE_PATH} declares version {version!r}, " + f"but this A5 (Ascend 950) host needs {_A5_RANKTABLE_VERSION!r}. " + f"HCCL will refuse it with Config_Error_Ranktable(EI0014) and every " + f"multi-card workload will fail to initialize. Remove the file or " + f"replace it with a {_A5_RANKTABLE_VERSION} table." + ) + logger.warning(msg) + return msg diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index e37b21d..71e43fb 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -58,6 +58,9 @@ sensitive_env_var, ) from .cdi import dump_config as cdi_dump_config +from .cdi.ascend import ( + warn_incompatible_ranktable as warn_incompatible_ascend_ranktable, +) if TYPE_CHECKING: from collections.abc import Callable, Generator @@ -1568,6 +1571,9 @@ def _create(self, workload: WorkloadPlan): raise TypeError(msg) self._prepare_mirrored_deployment() + # ascend-docker-runtime mounts the host ranktable itself, so this can + # only report, not prevent. Cached, so it costs one check per process. + warn_incompatible_ascend_ranktable() if isinstance(workload, WorkloadPlan): workload = DockerWorkloadPlan(**workload.__dict__) diff --git a/tests/gpustack_runtime/deployer/test_ascend_ranktable.py b/tests/gpustack_runtime/deployer/test_ascend_ranktable.py new file mode 100644 index 0000000..abbd7d0 --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_ascend_ranktable.py @@ -0,0 +1,224 @@ +# The ranktable cases below read the module's own private constant, which is +# the point: the warning and the mount list must agree on one path. +# ruff: noqa: SLF001 + +import inspect + +import pytest + +from gpustack_runtime.deployer.cdi import ascend as cdi_ascend +from gpustack_runtime.deployer.cdi.__types__ import ( + Config, + ConfigDeviceNode, + ConfigMount, +) +from gpustack_runtime.detector import Device, ManufacturerEnum + + +def _ascend_device(index: int, soc_name: str) -> Device: + return Device( + manufacturer=ManufacturerEnum.ASCEND, + index=index, + name=soc_name, + appendix={"arch_family": soc_name, "physical_id": index}, + ) + + +# --------------------------------------------------------------------------- +# The A5 mount profile. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def _synthetic_host(monkeypatch): + """ + Make the generator's host probes answer for every path it asks about, so + the mount list reflects the code's intent rather than the test machine. + """ + monkeypatch.setattr( + cdi_ascend, + "device_to_cdi_device_node", + lambda path, container_path=None: ConfigDeviceNode( + path=container_path or path, + host_path=path, + ), + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_device_nodes", + lambda path: [ConfigDeviceNode(path=path)], + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_mount", + lambda path: ConfigMount(host_path=path, options=["ro"]), + ) + monkeypatch.setattr( + cdi_ascend, + "glob_to_cdi_mounts", + lambda pattern: [ConfigMount(host_path=pattern, options=["ro"])], + ) + + +def _mount_targets(cfg: Config) -> set[str]: + return {m.container_path for m in cfg.container_edits.mounts} + + +@pytest.mark.usefixtures("_synthetic_host") +def test_a5_omits_the_host_ranktable(): + """ + A5 loads libhccl_v2, which rejects a 1.0 ranktable with EI0014 rather than + ignoring it, and nothing here writes a 2.0 one -- so GPUStack must not + mount it. See the comment at the mount list for the measurement. + """ + cfg = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend950PR")], + ) + + assert cfg is not None + targets = _mount_targets(cfg) + assert cdi_ascend._HCCL_RANKTABLE_PATH not in targets + # The rest of the vendor's named list is untouched. + assert "/usr/local/Ascend/driver/topo" in targets + assert "/usr/local/Ascend/driver/lib64" in targets + assert "/usr/local/Ascend/driver/include" in targets + assert "/usr/local/dcmi" in targets + assert "/usr/local/bin/npu-smi" in targets + + +@pytest.mark.usefixtures("_synthetic_host") +def test_non_a5_keeps_the_host_ranktable(): + """An older generation's own ranktable is correct for it.""" + cfg = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend910B4")], + ) + + assert cdi_ascend._HCCL_RANKTABLE_PATH in _mount_targets(cfg) + + +@pytest.mark.usefixtures("_synthetic_host") +def test_only_a5_gets_the_ub_user_space_mounts(): + """ + Unchanged behaviour, asserted here because the A5 branch was restructured + around it: libnl and friends are ordinary system libraries, and mounting + them for an older generation would shadow what its image ships with. + """ + a5 = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend950PR")], + ) + a2 = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend910B4")], + ) + + assert "/usr/lib64/liburma*" in _mount_targets(a5) + assert "/usr/lib64/liburma*" not in _mount_targets(a2) + + +# --------------------------------------------------------------------------- +# The warning -- the only thing that reaches the default Env path. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name, version, soc_name, expected", + [ + ("a5 with a stale A2 table", "1.0", "Ascend950PR", True), + ("a5 with an A3 table", "1.2", "Ascend950PR", True), + ("a5 with the right table", "2.0", "Ascend950PR", False), + # An absent file is the healthy host: a node that never ran + # mindcluster-tools has nothing to be wrong. + ("a5 with no table at all", None, "Ascend950PR", False), + # Only the A5 row of the generation mapping has a measured failure. + ("a2 with its own table", "1.0", "Ascend910B4", False), + ], +) +def test_warn_incompatible_ranktable(name, version, soc_name, expected, monkeypatch): + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: version) + monkeypatch.setattr( + cdi_ascend, + "detect_devices", + # Scoped to Ascend: the warning must not pay for a whole-host probe. + lambda manufacturer: ( + [_ascend_device(0, soc_name)] + if manufacturer == ManufacturerEnum.ASCEND + else [] + ), + ) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + msg = cdi_ascend.warn_incompatible_ranktable() + + assert (msg is not None) == expected, f"case {name}" + if expected: + assert "EI0014" in msg, "the warning must name the error the operator sees" + + +def test_warn_incompatible_ranktable_does_not_detect_without_the_file(monkeypatch): + """A host without the file must not pay for device detection.""" + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: None) + + calls = [] + + def _detect(manufacturer): + calls.append(manufacturer) + return [] + + monkeypatch.setattr(cdi_ascend, "detect_devices", _detect) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + assert cdi_ascend.warn_incompatible_ranktable() is None + assert calls == [], "detection must not run when there is no ranktable" + + +def test_warn_incompatible_ranktable_warns_once(monkeypatch): + """It runs per deploy, so it has to stay quiet after the first time.""" + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: "1.0") + monkeypatch.setattr( + cdi_ascend, + "detect_devices", + lambda manufacturer: [_ascend_device(0, "Ascend950PR")], # noqa: ARG005 + ) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + warnings = [] + monkeypatch.setattr(cdi_ascend.logger, "warning", warnings.append) + + for _ in range(3): + cdi_ascend.warn_incompatible_ranktable() + + assert len(warnings) == 1, f"warned {len(warnings)} times, expected once" + + +@pytest.mark.parametrize( + "name, content, expected", + [ + ("well formed", '{"version": "1.0"}', "1.0"), + ("no version key", '{"server_count": "1"}', None), + ("not an object", "[]", None), + ("not json", "{", None), + ("version not a string", '{"version": 1.0}', None), + ], +) +def test_read_ranktable_version(name, content, expected, tmp_path): + path = tmp_path / "hccl_rootinfo.json" + path.write_text(content, encoding="utf-8") + + assert cdi_ascend.read_ranktable_version(str(path)) == expected, f"case {name}" + + +def test_read_ranktable_version_missing_file(tmp_path): + assert cdi_ascend.read_ranktable_version(str(tmp_path / "absent.json")) is None + + +def test_the_warning_reads_the_path_the_mount_list_decides_on(): + """ + Both resolve the ranktable as this process sees it, so they cannot + disagree about whether it exists. A tmp_path test cannot catch a divergence + here -- it has to assert on the path the code itself reaches for. + """ + default = ( + inspect.signature(cdi_ascend.read_ranktable_version).parameters["path"].default + ) + + assert default == cdi_ascend._HCCL_RANKTABLE_PATH + assert cdi_ascend._HCCL_RANKTABLE_PATH == "/etc/hccl_rootinfo.json" From a97a376edc410d55922f3903000ef351f9e21606 Mon Sep 17 00:00:00 2001 From: thxCode Date: Sun, 6 Sep 2026 22:09:25 +0800 Subject: [PATCH 2/2] fixup! fix(ascend): stop A5 from receiving a ranktable its HCCL rejects --- gpustack_runtime/deployer/cdi/ascend.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index 768073d..b00c634 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -336,9 +336,10 @@ def warn_incompatible_ranktable() -> str | None: msg = ( f"Host ranktable {_HCCL_RANKTABLE_PATH} declares version {version!r}, " f"but this A5 (Ascend 950) host needs {_A5_RANKTABLE_VERSION!r}. " - f"HCCL will refuse it with Config_Error_Ranktable(EI0014) and every " - f"multi-card workload will fail to initialize. Remove the file or " - f"replace it with a {_A5_RANKTABLE_VERSION} table." + f"Under the default Env policy ascend-docker-runtime mounts it on " + f"presence and HCCL refuses it with Config_Error_Ranktable(EI0014), " + f"failing every multi-card workload. Remove the file or replace it " + f"with a {_A5_RANKTABLE_VERSION} table." ) logger.warning(msg) return msg