diff --git a/gpustack_runtime/deployer/cdi/ascend.py b/gpustack_runtime/deployer/cdi/ascend.py index b171387..b00c634 100644 --- a/gpustack_runtime/deployer/cdi/ascend.py +++ b/gpustack_runtime/deployer/cdi/ascend.py @@ -1,6 +1,9 @@ from __future__ import annotations as __future_annotations__ +import json import logging +from functools import cache +from pathlib import Path from ...detector import ( Devices, @@ -32,6 +35,28 @@ earlier generations reach through the shared memory device. """ +_HCCL_RANKTABLE_PATH = "/etc/hccl_rootinfo.json" +""" +The host ranktable ("rootinfo") file. Optional, user-maintained state: it +belongs to no driver package, and the vendor documents it as generated by +mindcluster-tools and mounted only when present, see +https://gitcode.com/Ascend/mind-cluster/blob/master/docs/en/scheduling/references/appendix.md. +""" + +_A5_RANKTABLE_VERSION = "2.0" +""" +The ranktable version the A5 generation requires. + +The format is bound to the chip generation -- 1.0 for A2, 1.2 for A3, 2.0 for +A5, see ascend-operator/pkg/ranktable/common/common.go:37-38 and +ranktable/v2dot0/ranktable.go:27. A5 loads libhccl_v2.so, which rejects an +older table outright rather than ignoring it. + +Only the A5 row of that mapping has a measured failure behind it, so only A5 is +acted on. An A3 host carrying a 1.2 table is presumed correct rather than +warned about, because nothing here established otherwise. +""" + _A5_UB_MOUNT_PATTERNS = [ "/usr/lib64/libummu*", "/usr/lib64/liburma*", @@ -124,30 +149,51 @@ def generate( if not common_device_nodes: return None - common_mounts = [] - for p in [ - "/etc/hccl_rootinfo.json", + # Device.appendix defaults to None and callers pass their own devices + # in, so every read goes through `or {}`. + is_a5 = any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ) + + mount_paths = [ + _HCCL_RANKTABLE_PATH, "/usr/local/Ascend/driver/topo", "/usr/local/Ascend/driver/lib64", "/usr/local/Ascend/driver/include", "/usr/local/dcmi", "/usr/local/bin/npu-smi", "/var/queue_schedule", - ]: + ] + + if is_a5: + # A5 loads libhccl_v2.so, which accepts a 2.0 ranktable only. A host + # file left over from an A2 fleet is 1.0, and HCCL refuses it with + # Config_Error_Ranktable(EI0014) rather than falling back, so every + # multi-card init dies. Nothing here writes the file, so the only + # lever is not mounting it -- verified by isolation: mounting only + # this file into an otherwise-working container reproduces EI0014, + # mounting only driver/topo does not. + # + # This is single-node evidence. A multi-node deployment does need a + # ranktable; when that lands it has to be a 2.0 one, not this. + # + # Only GPUStack's own mount list is covered here. Under the Env + # policy ascend-docker-runtime mounts the file itself, which + # `warn_incompatible_ranktable` reports and cannot prevent. + mount_paths.remove(_HCCL_RANKTABLE_PATH) + + common_mounts = [] + for p in mount_paths: cm = path_to_cdi_mount( path=p, ) if cm: common_mounts.append(cm) - # Device.appendix defaults to None and callers pass their own devices - # in, so every read goes through `or {}`. - if any( - get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) - == _A5_CANN_VARIANT - for dev in devices - if dev - ): + if is_a5: for pattern in _A5_UB_MOUNT_PATTERNS: ub_mounts = glob_to_cdi_mounts(pattern=pattern) if not ub_mounts: @@ -225,3 +271,75 @@ def generate( mounts=common_mounts, ), ) + + +def read_ranktable_version(path: str = _HCCL_RANKTABLE_PATH) -> str | None: + """ + Read the ``version`` of the host ranktable file. + + The path is resolved as this process sees it, exactly as the mount list + above resolves the paths it emits. Both therefore agree on whether the file + exists. + + Args: + path: + The ranktable path to read. + + Returns: + The declared version, or None if the file is absent, unreadable or not + a JSON object carrying a string version. + + """ + try: + content = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + + if not isinstance(content, dict): + return None + + version = content.get("version") + return version if isinstance(version, str) else None + + +@cache +def warn_incompatible_ranktable() -> str | None: + """ + Warn once if this A5 host carries a ranktable A5's HCCL will reject. + + ascend-docker-runtime mounts the host ranktable whenever the file is there + -- its `addUBMount` is a bare `os.Stat` with no version check -- so under + the default Env policy GPUStack cannot prevent the mount. Reporting is the + only available action: the operator has to remove the file or replace it + with a 2.0 one. + + Returns: + The warning message emitted, or None if there is nothing to warn about. + + """ + # Ordered so a host without the file pays nothing: the file is absent on + # almost every non-Ascend host, and detection is the expensive half. + version = read_ranktable_version() + if version is None or version == _A5_RANKTABLE_VERSION: + return None + + devices = detect_devices(manufacturer=ManufacturerEnum.ASCEND) + if not devices or not any( + get_ascend_cann_variant((dev.appendix or {}).get("arch_family")) + == _A5_CANN_VARIANT + for dev in devices + if dev + ): + # An older generation's own ranktable is correct for it. + return None + + msg = ( + f"Host ranktable {_HCCL_RANKTABLE_PATH} declares version {version!r}, " + f"but this A5 (Ascend 950) host needs {_A5_RANKTABLE_VERSION!r}. " + f"Under the default Env policy ascend-docker-runtime mounts it on " + f"presence and HCCL refuses it with Config_Error_Ranktable(EI0014), " + f"failing every multi-card workload. Remove the file or replace it " + f"with a {_A5_RANKTABLE_VERSION} table." + ) + logger.warning(msg) + return msg diff --git a/gpustack_runtime/deployer/docker.py b/gpustack_runtime/deployer/docker.py index e37b21d..71e43fb 100644 --- a/gpustack_runtime/deployer/docker.py +++ b/gpustack_runtime/deployer/docker.py @@ -58,6 +58,9 @@ sensitive_env_var, ) from .cdi import dump_config as cdi_dump_config +from .cdi.ascend import ( + warn_incompatible_ranktable as warn_incompatible_ascend_ranktable, +) if TYPE_CHECKING: from collections.abc import Callable, Generator @@ -1568,6 +1571,9 @@ def _create(self, workload: WorkloadPlan): raise TypeError(msg) self._prepare_mirrored_deployment() + # ascend-docker-runtime mounts the host ranktable itself, so this can + # only report, not prevent. Cached, so it costs one check per process. + warn_incompatible_ascend_ranktable() if isinstance(workload, WorkloadPlan): workload = DockerWorkloadPlan(**workload.__dict__) diff --git a/tests/gpustack_runtime/deployer/test_ascend_ranktable.py b/tests/gpustack_runtime/deployer/test_ascend_ranktable.py new file mode 100644 index 0000000..abbd7d0 --- /dev/null +++ b/tests/gpustack_runtime/deployer/test_ascend_ranktable.py @@ -0,0 +1,224 @@ +# The ranktable cases below read the module's own private constant, which is +# the point: the warning and the mount list must agree on one path. +# ruff: noqa: SLF001 + +import inspect + +import pytest + +from gpustack_runtime.deployer.cdi import ascend as cdi_ascend +from gpustack_runtime.deployer.cdi.__types__ import ( + Config, + ConfigDeviceNode, + ConfigMount, +) +from gpustack_runtime.detector import Device, ManufacturerEnum + + +def _ascend_device(index: int, soc_name: str) -> Device: + return Device( + manufacturer=ManufacturerEnum.ASCEND, + index=index, + name=soc_name, + appendix={"arch_family": soc_name, "physical_id": index}, + ) + + +# --------------------------------------------------------------------------- +# The A5 mount profile. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def _synthetic_host(monkeypatch): + """ + Make the generator's host probes answer for every path it asks about, so + the mount list reflects the code's intent rather than the test machine. + """ + monkeypatch.setattr( + cdi_ascend, + "device_to_cdi_device_node", + lambda path, container_path=None: ConfigDeviceNode( + path=container_path or path, + host_path=path, + ), + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_device_nodes", + lambda path: [ConfigDeviceNode(path=path)], + ) + monkeypatch.setattr( + cdi_ascend, + "path_to_cdi_mount", + lambda path: ConfigMount(host_path=path, options=["ro"]), + ) + monkeypatch.setattr( + cdi_ascend, + "glob_to_cdi_mounts", + lambda pattern: [ConfigMount(host_path=pattern, options=["ro"])], + ) + + +def _mount_targets(cfg: Config) -> set[str]: + return {m.container_path for m in cfg.container_edits.mounts} + + +@pytest.mark.usefixtures("_synthetic_host") +def test_a5_omits_the_host_ranktable(): + """ + A5 loads libhccl_v2, which rejects a 1.0 ranktable with EI0014 rather than + ignoring it, and nothing here writes a 2.0 one -- so GPUStack must not + mount it. See the comment at the mount list for the measurement. + """ + cfg = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend950PR")], + ) + + assert cfg is not None + targets = _mount_targets(cfg) + assert cdi_ascend._HCCL_RANKTABLE_PATH not in targets + # The rest of the vendor's named list is untouched. + assert "/usr/local/Ascend/driver/topo" in targets + assert "/usr/local/Ascend/driver/lib64" in targets + assert "/usr/local/Ascend/driver/include" in targets + assert "/usr/local/dcmi" in targets + assert "/usr/local/bin/npu-smi" in targets + + +@pytest.mark.usefixtures("_synthetic_host") +def test_non_a5_keeps_the_host_ranktable(): + """An older generation's own ranktable is correct for it.""" + cfg = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend910B4")], + ) + + assert cdi_ascend._HCCL_RANKTABLE_PATH in _mount_targets(cfg) + + +@pytest.mark.usefixtures("_synthetic_host") +def test_only_a5_gets_the_ub_user_space_mounts(): + """ + Unchanged behaviour, asserted here because the A5 branch was restructured + around it: libnl and friends are ordinary system libraries, and mounting + them for an older generation would shadow what its image ships with. + """ + a5 = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend950PR")], + ) + a2 = cdi_ascend.AscendGenerator().generate( + devices=[_ascend_device(0, "Ascend910B4")], + ) + + assert "/usr/lib64/liburma*" in _mount_targets(a5) + assert "/usr/lib64/liburma*" not in _mount_targets(a2) + + +# --------------------------------------------------------------------------- +# The warning -- the only thing that reaches the default Env path. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name, version, soc_name, expected", + [ + ("a5 with a stale A2 table", "1.0", "Ascend950PR", True), + ("a5 with an A3 table", "1.2", "Ascend950PR", True), + ("a5 with the right table", "2.0", "Ascend950PR", False), + # An absent file is the healthy host: a node that never ran + # mindcluster-tools has nothing to be wrong. + ("a5 with no table at all", None, "Ascend950PR", False), + # Only the A5 row of the generation mapping has a measured failure. + ("a2 with its own table", "1.0", "Ascend910B4", False), + ], +) +def test_warn_incompatible_ranktable(name, version, soc_name, expected, monkeypatch): + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: version) + monkeypatch.setattr( + cdi_ascend, + "detect_devices", + # Scoped to Ascend: the warning must not pay for a whole-host probe. + lambda manufacturer: ( + [_ascend_device(0, soc_name)] + if manufacturer == ManufacturerEnum.ASCEND + else [] + ), + ) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + msg = cdi_ascend.warn_incompatible_ranktable() + + assert (msg is not None) == expected, f"case {name}" + if expected: + assert "EI0014" in msg, "the warning must name the error the operator sees" + + +def test_warn_incompatible_ranktable_does_not_detect_without_the_file(monkeypatch): + """A host without the file must not pay for device detection.""" + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: None) + + calls = [] + + def _detect(manufacturer): + calls.append(manufacturer) + return [] + + monkeypatch.setattr(cdi_ascend, "detect_devices", _detect) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + assert cdi_ascend.warn_incompatible_ranktable() is None + assert calls == [], "detection must not run when there is no ranktable" + + +def test_warn_incompatible_ranktable_warns_once(monkeypatch): + """It runs per deploy, so it has to stay quiet after the first time.""" + monkeypatch.setattr(cdi_ascend, "read_ranktable_version", lambda: "1.0") + monkeypatch.setattr( + cdi_ascend, + "detect_devices", + lambda manufacturer: [_ascend_device(0, "Ascend950PR")], # noqa: ARG005 + ) + cdi_ascend.warn_incompatible_ranktable.cache_clear() + + warnings = [] + monkeypatch.setattr(cdi_ascend.logger, "warning", warnings.append) + + for _ in range(3): + cdi_ascend.warn_incompatible_ranktable() + + assert len(warnings) == 1, f"warned {len(warnings)} times, expected once" + + +@pytest.mark.parametrize( + "name, content, expected", + [ + ("well formed", '{"version": "1.0"}', "1.0"), + ("no version key", '{"server_count": "1"}', None), + ("not an object", "[]", None), + ("not json", "{", None), + ("version not a string", '{"version": 1.0}', None), + ], +) +def test_read_ranktable_version(name, content, expected, tmp_path): + path = tmp_path / "hccl_rootinfo.json" + path.write_text(content, encoding="utf-8") + + assert cdi_ascend.read_ranktable_version(str(path)) == expected, f"case {name}" + + +def test_read_ranktable_version_missing_file(tmp_path): + assert cdi_ascend.read_ranktable_version(str(tmp_path / "absent.json")) is None + + +def test_the_warning_reads_the_path_the_mount_list_decides_on(): + """ + Both resolve the ranktable as this process sees it, so they cannot + disagree about whether it exists. A tmp_path test cannot catch a divergence + here -- it has to assert on the path the code itself reaches for. + """ + default = ( + inspect.signature(cdi_ascend.read_ranktable_version).parameters["path"].default + ) + + assert default == cdi_ascend._HCCL_RANKTABLE_PATH + assert cdi_ascend._HCCL_RANKTABLE_PATH == "/etc/hccl_rootinfo.json"