Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions gpustack_runtime/detector/ascend.py
Original file line number Diff line number Diff line change
Expand Up @@ -737,11 +737,10 @@ def _get_device_die_v2(dev_id, dev_bdf: str) -> str:
"""
Get the device's SoC die through the V2 API, falling back to its address.

The A5 driver reports neither die type -- both answer NOT_SUPPORT -- and a
die is not what the uuid is needed for: it only has to tell one device from
another, which is what the usage pass merges on and what the inventory is
keyed by. The PCI address does that on a machine whose cards have not
moved, where dropping the device would leave the NPU unusable outright.
The A5 chip's uuid is the DDie, asked for after the VDie as the vendor's
readers do. A device answering neither is identified by its PCI address:
the uuid only has to tell one device from another on this machine, and
dropping the device would leave the NPU unusable outright.

Args:
dev_id:
Expand All @@ -753,7 +752,7 @@ def _get_device_die_v2(dev_id, dev_bdf: str) -> str:
The die as a string, or the PCI address.

"""
for dev_die_type in (pydcmi.DCMI_DIE_TYPE_VDIE, pydcmi.DCMI_DIE_TYPE_NDIE):
for dev_die_type in (pydcmi.DCMI_DIE_TYPE_VDIE, pydcmi.DCMI_DIE_TYPE_DDIE):
with contextlib.suppress(pydcmi.DCMIError):
return pydcmi.dcmiv2_get_device_die_id(dev_id, dev_die_type)

Expand Down Expand Up @@ -1113,10 +1112,10 @@ def _get_toolkit_version() -> str | None:
"Ascend910_9382": 253,
"Ascend910_9372": 254,
"Ascend910_9362": 255,
"Ascend910_9363": 256,
"Ascend910_9579": 260,
"Ascend910_95": 260,
"Ascend950": 260,
"Ascend950PR": 260,
}


Expand Down Expand Up @@ -1154,9 +1153,8 @@ def _guess_soc_name_from_dev_name(dev_name: str) -> str | None:

# https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/devmanager/common/utils.go#L159-L176
#
# The A5 prefix is matched first: a name the mapping does not carry yet,
# like a later 950 variant, still belongs to the generation, and none of
# the regexes below would claim it.
# The 950 suffixes are an open set -- 950PR, 950DT, plus a die suffix on a
# CANN SOC_VERSION -- so all collapse onto one SoC, as the vendor readers do.
if soc_name.startswith(_950_PREFIX):
return "Ascend950"
if _310P_REGEX.match(dev_name):
Expand Down Expand Up @@ -1187,10 +1185,15 @@ def get_ascend_soc_version(name: str | None) -> int:
return -1

version = _soc_name_version_mapping.get(name)
if version is None:
return -1
if version is not None:
return version

# The server calls this on names that never passed through the guess above
# -- a stored device, a config file -- so the prefix has to hold here too.
if name.startswith(_950_PREFIX):
return _soc_name_version_mapping[_950_PREFIX]

return version
return -1


def get_ascend_cann_variant(name: str | None) -> str | None:
Expand Down
1 change: 1 addition & 0 deletions gpustack_runtime/detector/pydcmi/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@
## Enums ##
DCMI_DIE_TYPE_NDIE = 0
DCMI_DIE_TYPE_VDIE = 1
DCMI_DIE_TYPE_DDIE = 2

## Enums ##
DCMI_TOPO_TYPE_SELF = 0
Expand Down
48 changes: 33 additions & 15 deletions tests/gpustack_runtime/detector/test_ascend.py
Original file line number Diff line number Diff line change
Expand Up @@ -750,13 +750,12 @@ def _fake_device_node(path, **_kwargs):
@pytest.mark.parametrize(
"dev_name, soc_name, variant",
[
# An A5 chip keeps the "Ascend" prefix that the earlier generations
# drop, and is reported either way round.
("Ascend950PR", "Ascend950PR", "950"),
("950PR", "Ascend950PR", "950"),
# A 950 variant the mapping does not carry yet still belongs to the
# generation rather than falling through to nothing.
# An A5 chip keeps the "Ascend" prefix the earlier generations drop, and
# is reported either way round. Both models on sale collapse onto one.
("Ascend950PR", "Ascend950", "950"),
("950PR", "Ascend950", "950"),
("Ascend950DT", "Ascend950", "950"),
("Ascend950", "Ascend950", "950"),
# The generations that already worked, both where the mapping answers
# directly and where the regexes have to: the A5 prefix must claim
# neither.
Expand All @@ -773,6 +772,22 @@ def test_guess_soc_name_carries_the_generation(dev_name, soc_name, variant):
assert ascend.get_ascend_cann_variant(soc_name) == variant


@pytest.mark.parametrize(
"soc_name",
[
# Stored by an earlier runtime, or hand-written in a config.
"Ascend950PR",
"Ascend950DT",
# A CANN SOC_VERSION: the die suffix is what opens the name space.
"Ascend950PR_9579",
],
)
def test_cann_variant_resolves_a_950_name_the_mapping_does_not_carry(soc_name):
# The server calls this on names that never passed through the detector.
# Falling through would be served with an earlier generation's image.
assert ascend.get_ascend_cann_variant(soc_name) == "950"


def test_guess_soc_name_still_yields_nothing_for_an_unknown_chip():
assert ascend._guess_soc_name_from_dev_name("NotAnAscendChip") is None # noqa: SLF001
assert ascend.get_ascend_cann_variant(None) is None
Expand Down Expand Up @@ -828,7 +843,7 @@ def test_cdi_holds_the_ub_mounts_back_from_an_earlier_generation(monkeypatch):


def test_cdi_mounts_the_ub_libraries_for_the_a5_generation(monkeypatch):
patterns = _collect_ub_mount_patterns(monkeypatch, arch_family="Ascend950PR")
patterns = _collect_ub_mount_patterns(monkeypatch, arch_family="Ascend950")

assert any("liburma" in p for p in patterns)
assert any("libummu" in p for p in patterns)
Expand Down Expand Up @@ -1014,8 +1029,8 @@ class _UnitV2:
temperature: int = 51
power_deciwatts: int = 2012
ecc_errors: int = 0
# Which die types the driver answers for; the rest raise. A real 950PR
# answers neither, which the die-fallback tests set explicitly.
# Which die types the driver answers for; the rest raise. What a 950 really
# answers is what the die-fallback tests set explicitly.
die_types: tuple[int, ...] = (pydcmi.DCMI_DIE_TYPE_VDIE,)

@property
Expand Down Expand Up @@ -1045,6 +1060,7 @@ class _FakeDCMIV2:
DCMI_UNIT_TYPE_MCU = pydcmi.DCMI_UNIT_TYPE_MCU
DCMI_DIE_TYPE_VDIE = pydcmi.DCMI_DIE_TYPE_VDIE
DCMI_DIE_TYPE_NDIE = pydcmi.DCMI_DIE_TYPE_NDIE
DCMI_DIE_TYPE_DDIE = pydcmi.DCMI_DIE_TYPE_DDIE
DCMI_DEVICE_TYPE_HBM = pydcmi.DCMI_DEVICE_TYPE_HBM
DCMI_DEVICE_TYPE_DDR = pydcmi.DCMI_DEVICE_TYPE_DDR
DCMI_INPUT_TYPE_AICORE = pydcmi.DCMI_INPUT_TYPE_AICORE
Expand Down Expand Up @@ -1204,7 +1220,7 @@ def test_detect_info_v2_enumerates_devices_flat(fake_pydcmi_v2):
assert "card_id" not in devices[0].appendix
assert "device_id" not in devices[0].appendix
# The whole point: an A5 chip resolves to its generation.
assert devices[0].appendix["arch_family"] == "Ascend950PR"
assert devices[0].appendix["arch_family"] == "Ascend950"
assert ascend.get_ascend_cann_variant(devices[0].appendix["arch_family"]) == "950"


Expand Down Expand Up @@ -1392,17 +1408,19 @@ def test_get_topology_v2_resolves_the_api_version_when_handed_devices(fake_pydcm
assert "dcmi_get_topo_info_by_device_id" not in fake.calls


def test_detect_info_v2_falls_back_to_the_ndie(fake_pydcmi_v2):
fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=(pydcmi.DCMI_DIE_TYPE_NDIE,))])
def test_detect_info_v2_falls_back_to_the_ddie(fake_pydcmi_v2):
# The die type the vendor names as the A5 chip's uuid, and the one a 950
# answers: the VDie is asked for first and refused.
fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=(pydcmi.DCMI_DIE_TYPE_DDIE,))])

devices = AscendDetector().detect_info()

assert [d.uuid for d in devices] == ["5A 6B 7C 8D 0"]


def test_detect_info_v2_identifies_a_dieless_device_by_its_address(fake_pydcmi_v2):
# What the 950PR driver does: both die types answer NOT_SUPPORT. Dropping
# the device over it would leave eight usable NPUs invisible.
# A driver answering no die type at all. Dropping the device over it would
# leave eight usable NPUs invisible.
fake_pydcmi_v2([_UnitV2(dev_id=0, die_types=())])

devices = AscendDetector().detect_info()
Expand Down Expand Up @@ -1435,7 +1453,7 @@ def test_detect_reproduces_the_950pr_host(fake_pydcmi_v2):

assert len(devices) == 8
assert {d.name for d in devices} == {"Ascend950PR"}
assert {d.appendix["arch_family"] for d in devices} == {"Ascend950PR"}
assert {d.appendix["arch_family"] for d in devices} == {"Ascend950"}
assert {
ascend.get_ascend_cann_variant(d.appendix["arch_family"]) for d in devices
} == {"950"}
Expand Down
Loading