From 7c8cd4f09161e5675023437e4b0a9d924d869fa7 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 12:54:38 +0200 Subject: [PATCH 1/8] collect more metadata --- src/cloudai/systems/slurm/slurm-metadata.sh | 344 +++++++++++++++--- .../slurm/slurm_command_gen_strategy.py | 15 +- src/cloudai/systems/slurm/slurm_metadata.py | 9 + tests/ref_data/ai-dynamo.sbatch | 2 +- tests/ref_data/ddlb.sbatch | 2 +- tests/ref_data/deepep-benchmark.sbatch | 2 +- tests/ref_data/fio.sbatch | 2 +- tests/ref_data/gpt-no-hook.sbatch | 2 +- tests/ref_data/gpt-pre-test.sbatch | 2 +- tests/ref_data/grok-no-hook.sbatch | 2 +- tests/ref_data/grok-pre-test.sbatch | 2 +- tests/ref_data/megatron-run.sbatch | 2 +- tests/ref_data/moe-benchmark.sbatch | 2 +- tests/ref_data/nccl.sbatch | 2 +- tests/ref_data/nemo-run-no-hook.sbatch | 2 +- tests/ref_data/nemo-run-pre-test.sbatch | 2 +- tests/ref_data/nemo-run-vboost.sbatch | 2 +- tests/ref_data/nixl-ep.sbatch | 2 +- tests/ref_data/nixl-kvbench.sbatch | 2 +- tests/ref_data/nixl-perftest.sbatch | 2 +- tests/ref_data/nixl_bench.sbatch | 2 +- tests/ref_data/osu-bench.sbatch | 2 +- tests/ref_data/sglang-disagg-2nodes.sbatch | 2 +- tests/ref_data/sglang-disagg.sbatch | 2 +- tests/ref_data/sglang-multinode.sbatch | 2 +- tests/ref_data/sglang.sbatch | 2 +- tests/ref_data/sleep.sbatch | 2 +- tests/ref_data/slurm_container.sbatch | 2 +- tests/ref_data/triton-inference.sbatch | 2 +- tests/ref_data/ucc.sbatch | 2 +- tests/ref_data/vllm-disagg-2nodes.sbatch | 2 +- tests/ref_data/vllm-disagg.sbatch | 2 +- tests/ref_data/vllm-multinode.sbatch | 2 +- tests/ref_data/vllm.sbatch | 2 +- .../slurm/test_command_gen_strategy.py | 17 + tests/systems/slurm/test_metadata_script.py | 123 +++++++ tests/test_single_sbatch_runner.py | 8 +- 37 files changed, 496 insertions(+), 82 deletions(-) create mode 100644 tests/systems/slurm/test_metadata_script.py diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index 8f44b3de9..071c7f41f 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -1,45 +1,303 @@ -. /etc/os-release +#!/usr/bin/env bash + +# Metadata is best effort: a missing utility or an unsupported query must not +# fail the Slurm step or prevent the remaining fields from being collected. +set +e +set +u +set +o pipefail 2>/dev/null || true +export LC_ALL=C + +readonly UNKNOWN="unknown" + +safe_probe() { + local output + output="$("$@" 2>/dev/null)" || output="" + if [ -z "$output" ]; then + output="$UNKNOWN" + fi + printf '%s' "$output" + return 0 +} + +toml_escape() { + local value="${1:-$UNKNOWN}" + value=${value//$'\r'/ } + value=${value//$'\n'/, } + value=${value//\\/\\\\} + value=${value//\"/\\\"} + printf '%s' "$value" + return 0 +} + +emit_string() { + printf '%s = "%s"\n' "$1" "$(toml_escape "$2")" + return 0 +} + +emit_integer() { + local value="$2" + case "$value" in + '' | *[!0-9]*) value=0 ;; + esac + printf '%s = %s\n' "$1" "$value" + return 0 +} + +lscpu_field() { + local label="$1" + lscpu | awk -F: -v label="$label" '$1 == label {sub(/^[[:space:]]+/, "", $2); print $2; exit}' +} + +gpu_names_probe() { + nvidia-smi --query-gpu=name --format=csv,noheader +} + +gpu_driver_probe() { + nvidia-smi --query-gpu=driver_version --format=csv,noheader | awk 'NF {print; exit}' +} + +driver_cuda_compat_probe() { + nvidia-smi | awk ' + match($0, /CUDA Version: [0-9.]+/) { + value = substr($0, RSTART, RLENGTH) + sub(/^CUDA Version: /, "", value) + print value + exit + } + ' +} + +inventory_from_lines() { + awk ' + NF { + if (!seen[$0]++) { + order[++count] = $0 + } + } + END { + for (i = 1; i <= count; i++) { + if (i > 1) printf ", " + printf "%s x%d", order[i], seen[order[i]] + } + } + ' +} + +nonempty_line_count() { + awk 'NF {count++} END {print count + 0}' +} + +cuda_toolkit_version_probe() { + local cuda_root="${CLOUDAI_CUDA_ROOT:-/usr/local/cuda}" + local version_file + local version + + if command -v nvcc >/dev/null 2>&1; then + version="$(nvcc --version 2>/dev/null | awk ' + match($0, /release [0-9.]+/) { + value = substr($0, RSTART, RLENGTH) + sub(/^release /, "", value) + print value + exit + } + ')" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + + version_file="$cuda_root/version.json" + if [ -r "$version_file" ]; then + version="$(awk -F'"' '/"version"[[:space:]]*:/ {print $4; exit}' "$version_file" 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + + version_file="$cuda_root/version.txt" + if [ -r "$version_file" ]; then + version="$(awk 'match($0, /[0-9]+\.[0-9]+([.][0-9]+)?/) {print substr($0, RSTART, RLENGTH); exit}' \ + "$version_file" 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + + return 1 +} + +nic_lines_probe() { + lspci -Dnn | awk ' + { + line = tolower($0) + is_network = line ~ /ethernet controller|infiniband controller/ + is_nvidia = line ~ /\[15b3:/ || line ~ /mellanox/ || line ~ /nvidia/ + if (is_network && is_nvidia) print + } + ' +} + +nic_models_from_lines() { + awk ' + NF { + sub(/^[^[:space:]]+[[:space:]]+/, "") + sub(/^[^:]+:[[:space:]]*/, "") + print + } + ' +} + +hca_firmware_probe() { + local fw_file + local device_path + local device_name + local firmware + local separator="" + local sysfs_root="${CLOUDAI_INFINIBAND_SYSFS:-/sys/class/infiniband}" + + for fw_file in "$sysfs_root"/*/fw_ver; do + [ -r "$fw_file" ] || continue + device_path=${fw_file%/fw_ver} + device_name=${device_path##*/} + firmware="$(awk 'NF {print; exit}' "$fw_file" 2>/dev/null)" + [ -n "$firmware" ] || continue + printf '%s%s=%s' "$separator" "$device_name" "$firmware" + separator=", " + done + [ -n "$separator" ] +} + +doca_host_version_probe() { + local doca_root="${CLOUDAI_DOCA_ROOT:-/opt/mellanox/doca}" + local version + local version_file + + if command -v dpkg-query >/dev/null 2>&1; then + version="$(dpkg-query -W -f='${Version}' doca-host 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + + if command -v rpm >/dev/null 2>&1; then + version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' doca-host 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + + for version_file in "$doca_root/version.json" "$doca_root/version.txt"; do + [ -r "$version_file" ] || continue + version="$(awk -F'"' '/"version"[[:space:]]*:/ {print $4; exit}' "$version_file" 2>/dev/null)" + if [ -z "$version" ]; then + version="$(awk 'NF {print; exit}' "$version_file" 2>/dev/null)" + fi + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + done + + return 1 +} + +mpi_type_probe() { + mpirun --version | awk 'tolower($0) ~ /open mpi/ {print "openmpi"; exit}' +} + +mpi_version_probe() { + mpirun --version | awk 'tolower($0) ~ /open mpi/ {print $NF; exit}' +} + +ofed_version_probe() { + command -v ofed_info >/dev/null 2>&1 || return 1 + ofed_info -s | sed 's/:$//' +} + +libfabric_version_probe() { + command -v fi_info >/dev/null 2>&1 || return 1 + fi_info --version | awk 'tolower($0) ~ /libfabric/ {print $2; exit}' +} + +os_release_file="${CLOUDAI_OS_RELEASE_FILE:-/etc/os-release}" +if [ -r "$os_release_file" ]; then + # shellcheck disable=SC1090 + . "$os_release_file" 2>/dev/null || true +fi + +user="${USER:-$(safe_probe whoami)}" +kernel_version="$(safe_probe uname -r)" +cpu_model="$(safe_probe lscpu_field "Model name")" +cpu_arch="$(safe_probe lscpu_field "Architecture")" +cpu_vendor="$(safe_probe lscpu_field "Vendor ID")" + +gpu_names="$(gpu_names_probe 2>/dev/null)" || gpu_names="" +gpu_arch_type="$(printf '%s\n' "$gpu_names" | awk 'NF {print; exit}' 2>/dev/null)" +gpu_arch_type="${gpu_arch_type:-$UNKNOWN}" +gpu_count="$(printf '%s\n' "$gpu_names" | nonempty_line_count 2>/dev/null)" || gpu_count=0 +gpu_inventory="$(printf '%s\n' "$gpu_names" | inventory_from_lines 2>/dev/null)" +gpu_inventory="${gpu_inventory:-$UNKNOWN}" + +nic_lines="$(nic_lines_probe 2>/dev/null)" || nic_lines="" +nic_models="$(printf '%s\n' "$nic_lines" | nic_models_from_lines 2>/dev/null)" +nics="$(printf '%s\n' "$nic_models" | awk 'NF {print; exit}' 2>/dev/null)" +nics="${nics:-$UNKNOWN}" +nic_count="$(printf '%s\n' "$nic_lines" | nonempty_line_count 2>/dev/null)" || nic_count=0 +nic_inventory="$(printf '%s\n' "$nic_models" | inventory_from_lines 2>/dev/null)" +nic_inventory="${nic_inventory:-$UNKNOWN}" + +emit_string user "$user" +printf '\n[system]\n' +emit_string os_type "${ID:-$UNKNOWN}" +emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" +emit_string linux_kernel_version "$kernel_version" +emit_string gpu_arch_type "$gpu_arch_type" +emit_integer gpu_count "$gpu_count" +emit_string gpu_inventory "$gpu_inventory" +emit_string cpu_model_name "$cpu_model" +emit_string cpu_arch_type "$cpu_arch" +emit_string cpu_vendor "$cpu_vendor" + +printf '\n[mpi]\n' +emit_string mpi_type "$(safe_probe mpi_type_probe)" +emit_string mpi_version "$(safe_probe mpi_version_probe)" hpcx_version=${HPCX_DIR##*/} +emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" + +printf '\n[cuda]\n' +emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" +emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" +driver_version="$(safe_probe gpu_driver_probe)" +emit_string cuda_driver_version "$driver_version" +emit_string nvidia_driver_version "$driver_version" +emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" + +printf '\n[network]\n' +emit_string nics "$nics" +emit_integer nic_count "$nic_count" +emit_string nic_inventory "$nic_inventory" +emit_string hca_firmware_versions "$(safe_probe hca_firmware_probe)" +emit_string switch_type "${SWITCH:-$UNKNOWN}" +emit_string network_name "${NETWORK:-$UNKNOWN}" +emit_string mofed_version "$(safe_probe ofed_version_probe)" +emit_string doca_host_version "$(safe_probe doca_host_version_probe)" +emit_string libfabric_version "$(safe_probe libfabric_version_probe)" + +printf '\n[nccl]\n' +emit_string version "${NCCL_VERSION:-$UNKNOWN}" +emit_string commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" + +printf '\n[slurm]\n' +emit_string cluster_name "${SLURM_CLUSTER_NAME:-$UNKNOWN}" +emit_string node_list "${SLURM_NODELIST:-$UNKNOWN}" +emit_string num_nodes "${SLURM_NNODES:-$UNKNOWN}" +emit_string ntasks_per_node "${SLURM_NTASKS_PER_NODE:-$UNKNOWN}" +emit_string ntasks "${SLURM_NTASKS:-$UNKNOWN}" +emit_string job_id "${SLURM_JOBID:-$UNKNOWN}" -while IFS=: read -r value; do - echo "$value" -done < str: def _metadata_cmd(self) -> str: (self.test_run.output_path.absolute() / "metadata").mkdir(parents=True, exist_ok=True) num_nodes, _ = self.get_cached_nodes_spec() - metadata_script_path = "/cloudai_install" - if not self.image_path(): - metadata_script_path = str(self.system.install_path.absolute()) + metadata_script_path = str(self.system.install_path.absolute()) + + metadata_srun_prefix = ["srun", "--export=ALL", "--mpi=none"] + if not self.nodelist_in_use: + metadata_srun_prefix.append(f"-N{num_nodes}") + if self.system.extra_srun_args: + metadata_srun_prefix.append(self.system.extra_srun_args) + if self.test_run.extra_srun_args: + metadata_srun_prefix.append(self.test_run.extra_srun_args) + return " ".join( [ - *self.gen_srun_prefix(), + *metadata_srun_prefix, f"--ntasks={num_nodes}", "--ntasks-per-node=1", f"--output={self.test_run.output_path.absolute() / 'metadata' / 'node-%N.toml'}", diff --git a/src/cloudai/systems/slurm/slurm_metadata.py b/src/cloudai/systems/slurm/slurm_metadata.py index a3d2e9ed8..22ff5f63c 100644 --- a/src/cloudai/systems/slurm/slurm_metadata.py +++ b/src/cloudai/systems/slurm/slurm_metadata.py @@ -94,8 +94,11 @@ class MetadataSystem(BaseModel): os_version: str linux_kernel_version: str gpu_arch_type: str + gpu_count: int = 0 + gpu_inventory: str = "unknown" cpu_model_name: str cpu_arch_type: str + cpu_vendor: str = "unknown" class MetadataMPI(BaseModel): @@ -112,15 +115,21 @@ class MetadataCUDA(BaseModel): cuda_build_version: str cuda_runtime_version: str cuda_driver_version: str + nvidia_driver_version: str = "unknown" + cuda_toolkit_version: str = "unknown" class MetadataNetwork(BaseModel): """Represents the network metadata.""" nics: str + nic_count: int = 0 + nic_inventory: str = "unknown" + hca_firmware_versions: str = "unknown" switch_type: str network_name: str mofed_version: str + doca_host_version: str = "unknown" libfabric_version: str diff --git a/tests/ref_data/ai-dynamo.sbatch b/tests/ref_data/ai-dynamo.sbatch index c00906d40..3195f7db2 100644 --- a/tests/ref_data/ai-dynamo.sbatch +++ b/tests/ref_data/ai-dynamo.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N2 --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N2 --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh # Start DCGM exporter on each node. echo "Starting DCGM exporter..." diff --git a/tests/ref_data/ddlb.sbatch b/tests/ref_data/ddlb.sbatch index a8d413577..830a5a1ed 100644 --- a/tests/ref_data/ddlb.sbatch +++ b/tests/ref_data/ddlb.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python ddlb/cli/benchmark.py --primitive tp_columnwise -m 1024 -n 128 -k 1024 --dtype float16 --num-iterations 50 --num-warmups 5 --impl pytorch;backend=nccl;order=AG_before" diff --git a/tests/ref_data/deepep-benchmark.sbatch b/tests/ref_data/deepep-benchmark.sbatch index cc4237805..01bb45de0 100644 --- a/tests/ref_data/deepep-benchmark.sbatch +++ b/tests/ref_data/deepep-benchmark.sbatch @@ -25,6 +25,6 @@ export MASTER_PORT=29500 srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --nodes=2 --ntasks=2 --ntasks-per-node=1 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; torchrun --nnodes=2 --nproc_per_node=1 --rdzv_id=\${SLURM_JOB_ID:-0} --rdzv_backend=c10d --rdzv_endpoint=\${MASTER_ADDR}:\${MASTER_PORT} /workspace/DeepEP/tests/legacy/test_internode.py --num-processes 8 --num-tokens 4096 --hidden 7168 --num-topk 8 --pressure-test-mode 0 --num-experts 256" diff --git a/tests/ref_data/fio.sbatch b/tests/ref_data/fio.sbatch index a2ae28273..38145b7e2 100644 --- a/tests/ref_data/fio.sbatch +++ b/tests/ref_data/fio.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=2 --ntasks=4 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; fio --name=fio-smoke --filename=/tmp/cloudai-fio-test --rw=randwrite --bs=128k --iodepth=1 --numjobs=1 --group_reporting --thread" diff --git a/tests/ref_data/gpt-no-hook.sbatch b/tests/ref_data/gpt-no-hook.sbatch index 155e373fe..fe6becd08 100644 --- a/tests/ref_data/gpt-no-hook.sbatch +++ b/tests/ref_data/gpt-no-hook.sbatch @@ -14,7 +14,7 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/gpt-pre-test.sbatch b/tests/ref_data/gpt-pre-test.sbatch index ed28eb1a7..af20dd0c6 100644 --- a/tests/ref_data/gpt-pre-test.sbatch +++ b/tests/ref_data/gpt-pre-test.sbatch @@ -14,7 +14,7 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/grok-no-hook.sbatch b/tests/ref_data/grok-no-hook.sbatch index 03ff5c195..17a9911c8 100644 --- a/tests/ref_data/grok-no-hook.sbatch +++ b/tests/ref_data/grok-no-hook.sbatch @@ -14,7 +14,7 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass_re=.* --xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_enable_all_gather_combine_by_dim=false --xla_gpu_enable_highest_priority_async_stream=true --xla_gpu_enable_latency_hiding_scheduler=true --xla_gpu_enable_pipelined_all_gather=true --xla_gpu_enable_pipelined_all_reduce=true --xla_gpu_enable_pipelined_reduce_scatter=true --xla_gpu_enable_reduce_scatter_combine_by_dim=false --xla_gpu_enable_triton_gemm=false --xla_gpu_enable_triton_softmax_fusion=false --xla_gpu_enable_while_loop_double_buffering=true --xla_gpu_graph_level=0 --xla_gpu_pgle_profile_file_or_directory_path=/opt/paxml/workspace/pgle_output_profile.pbtxt --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD --xla_gpu_run_post_layout_collective_pipeliner=false --xla_gpu_use_memcpy_local_p2p=false" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/grok-pre-test.sbatch b/tests/ref_data/grok-pre-test.sbatch index 8567fb370..096f608d1 100644 --- a/tests/ref_data/grok-pre-test.sbatch +++ b/tests/ref_data/grok-pre-test.sbatch @@ -14,7 +14,7 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass_re=.* --xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_enable_all_gather_combine_by_dim=false --xla_gpu_enable_highest_priority_async_stream=true --xla_gpu_enable_latency_hiding_scheduler=true --xla_gpu_enable_pipelined_all_gather=true --xla_gpu_enable_pipelined_all_reduce=true --xla_gpu_enable_pipelined_reduce_scatter=true --xla_gpu_enable_reduce_scatter_combine_by_dim=false --xla_gpu_enable_triton_gemm=false --xla_gpu_enable_triton_softmax_fusion=false --xla_gpu_enable_while_loop_double_buffering=true --xla_gpu_graph_level=0 --xla_gpu_pgle_profile_file_or_directory_path=/opt/paxml/workspace/pgle_output_profile.pbtxt --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD --xla_gpu_run_post_layout_collective_pipeliner=false --xla_gpu_use_memcpy_local_p2p=false" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/megatron-run.sbatch b/tests/ref_data/megatron-run.sbatch index f56b8ec69..03d7967b7 100644 --- a/tests/ref_data/megatron-run.sbatch +++ b/tests/ref_data/megatron-run.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python __CLOUDAI_DIR__/run.py --global-batch-size 16 --hidden-size 4096 --max-position-embeddings 4096 --num-attention-heads 32 --num-layers 32 --pipeline-model-parallel-size 1 --seq-length 4096 --tensor-model-parallel-size 2 --save __CLOUDAI_DIR__ --load __CLOUDAI_DIR__ --tokenizer-model __CLOUDAI_DIR__/model.m --tensorboard-dir /cloudai_run_results/tensorboard --log-timers-to-tensorboard --log-throughput --log-memory-to-tensorboard --log-interval 1" diff --git a/tests/ref_data/moe-benchmark.sbatch b/tests/ref_data/moe-benchmark.sbatch index e9b6efa7e..a3ef519d6 100644 --- a/tests/ref_data/moe-benchmark.sbatch +++ b/tests/ref_data/moe-benchmark.sbatch @@ -14,7 +14,7 @@ export MASTER_PORT=29500 export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nccl.sbatch b/tests/ref_data/nccl.sbatch index 8f7434b61..362cc56f1 100644 --- a/tests/ref_data/nccl.sbatch +++ b/tests/ref_data/nccl.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" diff --git a/tests/ref_data/nemo-run-no-hook.sbatch b/tests/ref_data/nemo-run-no-hook.sbatch index ba0cd341d..ba884a6c8 100644 --- a/tests/ref_data/nemo-run-no-hook.sbatch +++ b/tests/ref_data/nemo-run-no-hook.sbatch @@ -14,6 +14,6 @@ export CLOUDAI_NEMO_RECIPE=llama_3b export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nemo-run-pre-test.sbatch b/tests/ref_data/nemo-run-pre-test.sbatch index a9f94dbe9..5ebe4f81b 100644 --- a/tests/ref_data/nemo-run-pre-test.sbatch +++ b/tests/ref_data/nemo-run-pre-test.sbatch @@ -14,7 +14,7 @@ export CLOUDAI_NEMO_RECIPE=llama_3b export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/nemo-run-vboost.sbatch b/tests/ref_data/nemo-run-vboost.sbatch index 73cf831e3..0d03e6735 100644 --- a/tests/ref_data/nemo-run-vboost.sbatch +++ b/tests/ref_data/nemo-run-vboost.sbatch @@ -17,6 +17,6 @@ srun --ntasks=1 --output=__OUTPUT_DIR__/output/vboost.out --error=__OUTPUT_DIR__ srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nixl-ep.sbatch b/tests/ref_data/nixl-ep.sbatch index da6bbe018..7d6096ce7 100644 --- a/tests/ref_data/nixl-ep.sbatch +++ b/tests/ref_data/nixl-ep.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export LD_LIBRARY_PATH=/workspace/rdma_core/lib:$LD_LIBRARY_PATH srun --export=ALL --mpi=pmix -N3 --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N3 --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh bash __OUTPUT_DIR__/output/nixl-ep-launch.sh diff --git a/tests/ref_data/nixl-kvbench.sbatch b/tests/ref_data/nixl-kvbench.sbatch index b5ae979ce..2a316a69d 100644 --- a/tests/ref_data/nixl-kvbench.sbatch +++ b/tests/ref_data/nixl-kvbench.sbatch @@ -14,7 +14,7 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head -n 1) srun --export=ALL --mpi=pmix -N2 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N2 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nixl-perftest.sbatch b/tests/ref_data/nixl-perftest.sbatch index 6b4d2ba88..eb809f28f 100644 --- a/tests/ref_data/nixl-perftest.sbatch +++ b/tests/ref_data/nixl-perftest.sbatch @@ -14,7 +14,7 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head -n 1) srun --export=ALL --mpi=pmix -N1 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=1 --ntasks=1 -N1 bash -c "python /workspace/nixl/benchmark/kvbench/test/inference_workload_matgen.py generate --num-user-requests=2 --batch-size=1 --num-prefill-nodes=1 --num-decode-nodes=1 --results-dir=__OUTPUT_DIR__/output/matrices --prefill-tp=1 --prefill-pp=1 --prefill-cp=1 --decode-tp=1 --decode-pp=1 --decode-cp=1 --model=model-name" srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & diff --git a/tests/ref_data/nixl_bench.sbatch b/tests/ref_data/nixl_bench.sbatch index cd59cb3f0..bf016bb6f 100644 --- a/tests/ref_data/nixl_bench.sbatch +++ b/tests/ref_data/nixl_bench.sbatch @@ -14,7 +14,7 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head -n 1) srun --export=ALL --mpi=pmix -N2 --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=url.com/docker:2 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/osu-bench.sbatch b/tests/ref_data/osu-bench.sbatch index 2abd0bd55..404568e6d 100644 --- a/tests/ref_data/osu-bench.sbatch +++ b/tests/ref_data/osu-bench.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ompi/tests/osu-micro-benchmarks/osu_allreduce --message-size 1024 --iterations 10 --full" diff --git a/tests/ref_data/sglang-disagg-2nodes.sbatch b/tests/ref_data/sglang-disagg-2nodes.sbatch index 102da888a..12c0d45a5 100644 --- a/tests/ref_data/sglang-disagg-2nodes.sbatch +++ b/tests/ref_data/sglang-disagg-2nodes.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-disagg.sbatch b/tests/ref_data/sglang-disagg.sbatch index 879d3b878..d6a4aa3eb 100644 --- a/tests/ref_data/sglang-disagg.sbatch +++ b/tests/ref_data/sglang-disagg.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-multinode.sbatch b/tests/ref_data/sglang-multinode.sbatch index 46ee15ea9..824df73b9 100644 --- a/tests/ref_data/sglang-multinode.sbatch +++ b/tests/ref_data/sglang-multinode.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_SGLANG_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang.sbatch b/tests/ref_data/sglang.sbatch index e60149d7c..cdfcbfc73 100644 --- a/tests/ref_data/sglang.sbatch +++ b/tests/ref_data/sglang.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_SGLANG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sleep.sbatch b/tests/ref_data/sleep.sbatch index 9dbea847f..4a946f492 100644 --- a/tests/ref_data/sleep.sbatch +++ b/tests/ref_data/sleep.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __OUTPUT_DIR__/install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __OUTPUT_DIR__/install/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; sleep 5" diff --git a/tests/ref_data/slurm_container.sbatch b/tests/ref_data/slurm_container.sbatch index 586cad22c..ed4b4de76 100644 --- a/tests/ref_data/slurm_container.sbatch +++ b/tests/ref_data/slurm_container.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; pwd ; ls" diff --git a/tests/ref_data/triton-inference.sbatch b/tests/ref_data/triton-inference.sbatch index 876319939..f52d3a8ed 100644 --- a/tests/ref_data/triton-inference.sbatch +++ b/tests/ref_data/triton-inference.sbatch @@ -17,7 +17,7 @@ export NIM_MODEL_NAME=__OUTPUT_DIR__/output export NIM_CACHE_PATH=__OUTPUT_DIR__/output srun --export=ALL --mpi=pmix -N3 --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __OUTPUT_DIR__/install/slurm-metadata.sh +srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __OUTPUT_DIR__/install/slurm-metadata.sh srun --export=ALL --mpi=pmix --container-image=nvcr.io/nim/deepseek-ai/deepseek-r1:1.7.2 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:__OUTPUT_DIR__/output:ro,__OUTPUT_DIR__/output:__OUTPUT_DIR__/output:rw,__OUTPUT_DIR__/output/start_server_wrapper.sh:/opt/nim/start_server_wrapper.sh:ro --nodes=2 --ntasks=2 --ntasks-per-node=1 /opt/nim/start_server_wrapper.sh & diff --git a/tests/ref_data/ucc.sbatch b/tests/ref_data/ucc.sbatch index 2f5d7e458..a4e17ba0f 100644 --- a/tests/ref_data/ucc.sbatch +++ b/tests/ref_data/ucc.sbatch @@ -12,6 +12,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ucc/bin/ucc_perftest -c alltoall -b 1 -e 8M -m cuda -F" diff --git a/tests/ref_data/vllm-disagg-2nodes.sbatch b/tests/ref_data/vllm-disagg-2nodes.sbatch index 777cdbfa1..f9191034a 100644 --- a/tests/ref_data/vllm-disagg-2nodes.sbatch +++ b/tests/ref_data/vllm-disagg-2nodes.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-disagg.sbatch b/tests/ref_data/vllm-disagg.sbatch index 7d83d196d..ffc8aedab 100644 --- a/tests/ref_data/vllm-disagg.sbatch +++ b/tests/ref_data/vllm-disagg.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-multinode.sbatch b/tests/ref_data/vllm-multinode.sbatch index 7886dca3c..7afdccd9a 100644 --- a/tests/ref_data/vllm-multinode.sbatch +++ b/tests/ref_data/vllm-multinode.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_VLLM_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm.sbatch b/tests/ref_data/vllm.sbatch index e7fa01d91..2face48a9 100644 --- a/tests/ref_data/vllm.sbatch +++ b/tests/ref_data/vllm.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash /cloudai_install/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh cleanup() { if [ "${CLOUDAI_VLLM_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/systems/slurm/test_command_gen_strategy.py b/tests/systems/slurm/test_command_gen_strategy.py index 98e103704..962855da5 100644 --- a/tests/systems/slurm/test_command_gen_strategy.py +++ b/tests/systems/slurm/test_command_gen_strategy.py @@ -334,6 +334,23 @@ def test_gen_srun_prefix_tr_extra_srun_args(strategy_fixture: SlurmCommandGenStr assert "--arg val --flag" in srun_prefix # added as a single element +def test_metadata_cmd_uses_host_without_changing_extra_args( + strategy_fixture: SlurmCommandGenStrategy, +) -> None: + strategy_fixture.image_path = Mock(return_value="workload.sqsh") + strategy_fixture.system.extra_srun_args = "--reservation keep" + strategy_fixture.test_run.extra_srun_args = "--exclusive --constraint='gpu type'" + + metadata_cmd = strategy_fixture._metadata_cmd() + + assert "--container-image" not in metadata_cmd + assert "--container-mounts" not in metadata_cmd + assert "--mpi=none" in metadata_cmd + assert "--reservation keep" in metadata_cmd + assert "--exclusive --constraint='gpu type'" in metadata_cmd + assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh" in metadata_cmd + + def test_append_distribution_and_hostfile_with_nodes(slurm_system: SlurmSystem, testrun_fixture: TestRun) -> None: slurm_system.distribution = "block" slurm_system.ntasks_per_node = 2 diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py new file mode 100644 index 000000000..e6199f129 --- /dev/null +++ b/tests/systems/slurm/test_metadata_script.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +from pathlib import Path + +import toml + +from cloudai.systems import slurm + +METADATA_SCRIPT = Path(slurm.__file__).parent / "slurm-metadata.sh" + + +def _write_command(bin_dir: Path, name: str, body: str) -> None: + command = bin_dir / name + command.write_text(f"#!/usr/bin/env bash\n{body}\n") + command.chmod(0o755) + + +def _run_collector(tmp_path: Path, commands: dict[str, str]) -> subprocess.CompletedProcess[str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for name, body in commands.items(): + _write_command(bin_dir, name, body) + + os_release = tmp_path / "os-release" + os_release.write_text('ID=ubuntu\nVERSION="24.04 LTS"\n') + infiniband_sysfs = tmp_path / "infiniband" / "mlx5_0" + infiniband_sysfs.mkdir(parents=True) + (infiniband_sysfs / "fw_ver").write_text("28.43.2026\n") + + env = { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "USER": 'metadata"user', + "CLOUDAI_OS_RELEASE_FILE": str(os_release), + "CLOUDAI_INFINIBAND_SYSFS": str(infiniband_sysfs.parent), + "CLOUDAI_CUDA_ROOT": str(tmp_path / "missing-cuda"), + "CLOUDAI_DOCA_ROOT": str(tmp_path / "missing-doca"), + "SLURM_JOBID": "12345", + } + return subprocess.run(["/bin/bash", str(METADATA_SCRIPT)], env=env, text=True, capture_output=True, check=False) + + +def test_collects_requested_host_metadata(tmp_path: Path) -> None: + result = _run_collector( + tmp_path, + { + "lscpu": """cat <<'EOF' +Architecture: x86_64 +Vendor ID: GenuineIntel +Model name: Intel(R) Xeon(R) Platinum 8573C +EOF""", + "nvidia-smi": """case "$1" in + --query-gpu=name) printf 'NVIDIA H100 80GB HBM3\\nNVIDIA H100 80GB HBM3\\n' ;; + --query-gpu=driver_version) printf '570.124.06\\n570.124.06\\n' ;; + *) printf '| NVIDIA-SMI 570.124.06 Driver Version: 570.124.06 CUDA Version: 12.8 |\\n' ;; +esac""", + "nvcc": "printf 'Cuda compilation tools, release 12.6, V12.6.85\\n'", + "lspci": """cat <<'EOF' +0000:17:00.0 Ethernet controller [0200]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] +0000:31:00.0 Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] +0000:65:00.0 VGA compatible controller [0300]: NVIDIA Corporation Device [10de:2330] +EOF""", + "dpkg-query": "printf '2.9.2-0.1.0\\n'", + "ofed_info": "printf 'MLNX_OFED_LINUX-24.10-1.1.4.0:\\n'", + "fi_info": "printf 'libfabric: 1.22.0\\n'", + "mpirun": "printf 'mpirun (Open MPI) 4.1.7a1\\n'", + }, + ) + + assert result.returncode == 0 + assert result.stderr == "" + metadata = toml.loads(result.stdout) + assert metadata["user"] == 'metadata"user' + assert metadata["system"]["linux_kernel_version"] + assert metadata["system"] == { + "os_type": "ubuntu", + "os_version": "24.04 LTS", + "linux_kernel_version": metadata["system"]["linux_kernel_version"], + "gpu_arch_type": "NVIDIA H100 80GB HBM3", + "gpu_count": 2, + "gpu_inventory": "NVIDIA H100 80GB HBM3 x2", + "cpu_model_name": "Intel(R) Xeon(R) Platinum 8573C", + "cpu_arch_type": "x86_64", + "cpu_vendor": "GenuineIntel", + } + assert metadata["cuda"] == { + "cuda_build_version": "unknown", + "cuda_runtime_version": "12.8", + "cuda_driver_version": "570.124.06", + "nvidia_driver_version": "570.124.06", + "cuda_toolkit_version": "12.6", + } + assert metadata["network"] == { + "nics": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", + "nic_count": 2, + "nic_inventory": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] x2", + "hca_firmware_versions": "mlx5_0=28.43.2026", + "switch_type": "unknown", + "network_name": "unknown", + "mofed_version": "MLNX_OFED_LINUX-24.10-1.1.4.0", + "doca_host_version": "2.9.2-0.1.0", + "libfabric_version": "1.22.0", + } + + +def test_missing_subcommands_do_not_fail_or_corrupt_output(tmp_path: Path) -> None: + failing_commands = { + command: "exit 42" + for command in ("lscpu", "nvidia-smi", "nvcc", "lspci", "dpkg-query", "rpm", "mpirun", "ofed_info", "fi_info") + } + result = _run_collector(tmp_path, failing_commands) + + assert result.returncode == 0 + metadata = toml.loads(result.stdout) + assert metadata["system"]["gpu_count"] == 0 + assert metadata["system"]["gpu_arch_type"] == "unknown" + assert metadata["cuda"]["cuda_toolkit_version"] == "unknown" + assert metadata["cuda"]["nvidia_driver_version"] == "unknown" + assert metadata["network"]["nic_count"] == 0 + assert metadata["network"]["doca_host_version"] == "unknown" diff --git a/tests/test_single_sbatch_runner.py b/tests/test_single_sbatch_runner.py index 698b63bd5..8a645f569 100644 --- a/tests/test_single_sbatch_runner.py +++ b/tests/test_single_sbatch_runner.py @@ -242,7 +242,7 @@ def test_bare_metal(self, sleep_tr: TestRun, slurm_system: SlurmSystem) -> None: assert len(aux_cmds) == 2 metadata_cmd = ( - f"srun --export=ALL --mpi=pmix -N{sleep_tr.num_nodes} --ntasks=1 --ntasks-per-node=1 " + f"srun --export=ALL --mpi=none -N{sleep_tr.num_nodes} --ntasks=1 --ntasks-per-node=1 " f"--output={runner.scenario_root}/metadata/node-%N.toml " f"--error={runner.scenario_root}/metadata/nodes.err " "bash " @@ -272,13 +272,13 @@ def test_container(self, nccl_tr: TestRun, slurm_system: SlurmSystem) -> None: f"{runner.system.output_path.absolute()}" ) metadata_cmd = ( - f"srun --export=ALL --mpi=pmix -N{nccl_tr.num_nodes} --container-image={tdef.docker_image.installed_path} " - f"{mounts} --no-container-mount-home --ntasks=2 --ntasks-per-node=1 " + f"srun --export=ALL --mpi=none -N{nccl_tr.num_nodes} --ntasks=2 --ntasks-per-node=1 " f"--output={runner.scenario_root}/metadata/node-%N.toml " f"--error={runner.scenario_root}/metadata/nodes.err " - f"bash /cloudai_install/slurm-metadata.sh" + f"bash {runner.system.install_path}/slurm-metadata.sh" ) assert aux_cmds[0] == metadata_cmd + assert "--container-image" not in aux_cmds[0] ranks_mapping_cmd = ( f"srun --export=ALL --mpi=pmix -N{nccl_tr.num_nodes} --container-image={tdef.docker_image.installed_path} " From 6be171b13ced0cf5705e55cd647c286e3c349890 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 13:51:52 +0200 Subject: [PATCH 2/8] two sruns --- src/cloudai/systems/slurm/slurm-metadata.sh | 43 +++++++++++---- .../slurm/slurm_command_gen_strategy.py | 39 +++++++++----- src/cloudai/systems/slurm/slurm_metadata.py | 30 ++++++++--- tests/ref_data/ai-dynamo.sbatch | 1 + tests/ref_data/ddlb.sbatch | 1 + tests/ref_data/deepep-benchmark.sbatch | 1 + tests/ref_data/fio.sbatch | 1 + tests/ref_data/gpt-no-hook.sbatch | 1 + tests/ref_data/gpt-pre-test.sbatch | 3 +- tests/ref_data/grok-no-hook.sbatch | 1 + tests/ref_data/grok-pre-test.sbatch | 3 +- tests/ref_data/megatron-run.sbatch | 1 + tests/ref_data/moe-benchmark.sbatch | 1 + tests/ref_data/nccl.sbatch | 1 + tests/ref_data/nemo-run-no-hook.sbatch | 1 + tests/ref_data/nemo-run-pre-test.sbatch | 3 +- tests/ref_data/nemo-run-vboost.sbatch | 1 + tests/ref_data/nixl-ep.sbatch | 1 + tests/ref_data/nixl-kvbench.sbatch | 1 + tests/ref_data/nixl-perftest.sbatch | 3 +- tests/ref_data/osu-bench.sbatch | 1 + tests/ref_data/sglang-disagg-2nodes.sbatch | 1 + tests/ref_data/sglang-disagg.sbatch | 1 + tests/ref_data/sglang-multinode.sbatch | 1 + tests/ref_data/sglang.sbatch | 1 + tests/ref_data/slurm_container.sbatch | 1 + tests/ref_data/ucc.sbatch | 1 + tests/ref_data/vllm-disagg-2nodes.sbatch | 1 + tests/ref_data/vllm-disagg.sbatch | 1 + tests/ref_data/vllm-multinode.sbatch | 1 + tests/ref_data/vllm.sbatch | 1 + .../slurm/test_command_gen_strategy.py | 20 ++++--- tests/systems/slurm/test_metadata_script.py | 54 +++++++++++++++---- tests/test_single_sbatch_runner.py | 18 ++++--- 34 files changed, 183 insertions(+), 57 deletions(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index 071c7f41f..d88b1e792 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -7,7 +7,7 @@ set +u set +o pipefail 2>/dev/null || true export LC_ALL=C -readonly UNKNOWN="unknown" +readonly UNKNOWN="null" safe_probe() { local output @@ -172,23 +172,28 @@ hca_firmware_probe() { doca_host_version_probe() { local doca_root="${CLOUDAI_DOCA_ROOT:-/opt/mellanox/doca}" + local package local version local version_file if command -v dpkg-query >/dev/null 2>&1; then - version="$(dpkg-query -W -f='${Version}' doca-host 2>/dev/null)" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi + for package in doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic doca-extra; do + version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + done fi if command -v rpm >/dev/null 2>&1; then - version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' doca-host 2>/dev/null)" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi + for package in doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic doca-extra; do + version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' "$package" 2>/dev/null)" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + done fi for version_file in "$doca_root/version.json" "$doca_root/version.txt"; do @@ -230,6 +235,22 @@ if [ -r "$os_release_file" ]; then . "$os_release_file" 2>/dev/null || true fi +if [ "${1:-host}" = "runtime" ]; then + printf '[runtime]\n' + emit_string os_type "${ID:-$UNKNOWN}" + emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" + emit_string mpi_type "$(safe_probe mpi_type_probe)" + emit_string mpi_version "$(safe_probe mpi_version_probe)" + hpcx_version=${HPCX_DIR##*/} + emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" + emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" + emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" + emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" + emit_string nccl_version "${NCCL_VERSION:-$UNKNOWN}" + emit_string nccl_commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" + exit 0 +fi + user="${USER:-$(safe_probe whoami)}" kernel_version="$(safe_probe uname -r)" cpu_model="$(safe_probe lscpu_field "Model name")" diff --git a/src/cloudai/systems/slurm/slurm_command_gen_strategy.py b/src/cloudai/systems/slurm/slurm_command_gen_strategy.py index e24d8c7c7..f07f0f52b 100644 --- a/src/cloudai/systems/slurm/slurm_command_gen_strategy.py +++ b/src/cloudai/systems/slurm/slurm_command_gen_strategy.py @@ -310,7 +310,10 @@ def _ranks_mapping_cmd(self) -> str: def _metadata_cmd(self) -> str: (self.test_run.output_path.absolute() / "metadata").mkdir(parents=True, exist_ok=True) num_nodes, _ = self.get_cached_nodes_spec() - metadata_script_path = str(self.system.install_path.absolute()) + metadata_script_path = self.system.install_path.absolute() + + metadata_output = self.test_run.output_path.absolute() / "metadata" / "node-%N.toml" + metadata_error = self.test_run.output_path.absolute() / "metadata" / "nodes.err" metadata_srun_prefix = ["srun", "--export=ALL", "--mpi=none"] if not self.nodelist_in_use: @@ -320,17 +323,29 @@ def _metadata_cmd(self) -> str: if self.test_run.extra_srun_args: metadata_srun_prefix.append(self.test_run.extra_srun_args) - return " ".join( - [ - *metadata_srun_prefix, - f"--ntasks={num_nodes}", - "--ntasks-per-node=1", - f"--output={self.test_run.output_path.absolute() / 'metadata' / 'node-%N.toml'}", - f"--error={self.test_run.output_path.absolute() / 'metadata' / 'nodes.err'}", - "bash", - f"{metadata_script_path}/slurm-metadata.sh", - ] - ) + metadata_srun_prefix = [ + *metadata_srun_prefix, + f"--ntasks={num_nodes}", + "--ntasks-per-node=1", + f"--output={metadata_output}", + f"--error={metadata_error}", + ] + + host_command = " ".join([*metadata_srun_prefix, "bash", f"{metadata_script_path}/slurm-metadata.sh"]) + + image_path = self.image_path() + if not image_path: + return host_command + + runtime_command_parts = [*metadata_srun_prefix, "--open-mode=append", f"--container-image={image_path}"] + mounts = self.container_mounts() + if mounts: + runtime_command_parts.append(f"--container-mounts={','.join(mounts)}") + if not self.system.container_mount_home: + runtime_command_parts.append("--no-container-mount-home") + runtime_command_parts.extend(["bash", f"{self.CONTAINER_MOUNT_INSTALL}/slurm-metadata.sh", "runtime"]) + runtime_command = " ".join(runtime_command_parts) + return "\n".join([host_command, runtime_command]) def _enable_vboost_cmd(self) -> str: num_nodes, _ = self.system.get_nodes_by_spec(self.test_run.nnodes, self.test_run.nodes) diff --git a/src/cloudai/systems/slurm/slurm_metadata.py b/src/cloudai/systems/slurm/slurm_metadata.py index 22ff5f63c..6c71643c7 100644 --- a/src/cloudai/systems/slurm/slurm_metadata.py +++ b/src/cloudai/systems/slurm/slurm_metadata.py @@ -95,10 +95,10 @@ class MetadataSystem(BaseModel): linux_kernel_version: str gpu_arch_type: str gpu_count: int = 0 - gpu_inventory: str = "unknown" + gpu_inventory: str = "null" cpu_model_name: str cpu_arch_type: str - cpu_vendor: str = "unknown" + cpu_vendor: str = "null" class MetadataMPI(BaseModel): @@ -115,8 +115,8 @@ class MetadataCUDA(BaseModel): cuda_build_version: str cuda_runtime_version: str cuda_driver_version: str - nvidia_driver_version: str = "unknown" - cuda_toolkit_version: str = "unknown" + nvidia_driver_version: str = "null" + cuda_toolkit_version: str = "null" class MetadataNetwork(BaseModel): @@ -124,12 +124,12 @@ class MetadataNetwork(BaseModel): nics: str nic_count: int = 0 - nic_inventory: str = "unknown" - hca_firmware_versions: str = "unknown" + nic_inventory: str = "null" + hca_firmware_versions: str = "null" switch_type: str network_name: str mofed_version: str - doca_host_version: str = "unknown" + doca_host_version: str = "null" libfabric_version: str @@ -151,6 +151,21 @@ class MetadataSlurm(BaseModel): job_id: str +class MetadataRuntime(BaseModel): + """Represents software metadata collected inside the workload container.""" + + os_type: str = "null" + os_version: str = "null" + mpi_type: str = "null" + mpi_version: str = "null" + hpcx_version: str = "null" + cuda_build_version: str = "null" + cuda_runtime_version: str = "null" + cuda_toolkit_version: str = "null" + nccl_version: str = "null" + nccl_commit_sha: str = "null" + + class SlurmSystemMetadata(BaseModel): """Represents the Slurm system metadata.""" @@ -161,3 +176,4 @@ class SlurmSystemMetadata(BaseModel): network: MetadataNetwork nccl: MetadataNCCL slurm: MetadataSlurm + runtime: MetadataRuntime = Field(default_factory=MetadataRuntime) diff --git a/tests/ref_data/ai-dynamo.sbatch b/tests/ref_data/ai-dynamo.sbatch index 3195f7db2..abbaefe9b 100644 --- a/tests/ref_data/ai-dynamo.sbatch +++ b/tests/ref_data/ai-dynamo.sbatch @@ -13,6 +13,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N2 --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp bash /cloudai_install/slurm-metadata.sh runtime # Start DCGM exporter on each node. echo "Starting DCGM exporter..." diff --git a/tests/ref_data/ddlb.sbatch b/tests/ref_data/ddlb.sbatch index 830a5a1ed..2afbb18a1 100644 --- a/tests/ref_data/ddlb.sbatch +++ b/tests/ref_data/ddlb.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python ddlb/cli/benchmark.py --primitive tp_columnwise -m 1024 -n 128 -k 1024 --dtype float16 --num-iterations 50 --num-warmups 5 --impl pytorch;backend=nccl;order=AG_before" diff --git a/tests/ref_data/deepep-benchmark.sbatch b/tests/ref_data/deepep-benchmark.sbatch index 01bb45de0..dc3dfaf57 100644 --- a/tests/ref_data/deepep-benchmark.sbatch +++ b/tests/ref_data/deepep-benchmark.sbatch @@ -26,5 +26,6 @@ export MASTER_PORT=29500 srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --nodes=2 --ntasks=2 --ntasks-per-node=1 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; torchrun --nnodes=2 --nproc_per_node=1 --rdzv_id=\${SLURM_JOB_ID:-0} --rdzv_backend=c10d --rdzv_endpoint=\${MASTER_ADDR}:\${MASTER_PORT} /workspace/DeepEP/tests/legacy/test_internode.py --num-processes 8 --num-tokens 4096 --hidden 7168 --num-topk 8 --pressure-test-mode 0 --num-experts 256" diff --git a/tests/ref_data/fio.sbatch b/tests/ref_data/fio.sbatch index 38145b7e2..d6aec262e 100644 --- a/tests/ref_data/fio.sbatch +++ b/tests/ref_data/fio.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=2 --ntasks=4 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; fio --name=fio-smoke --filename=/tmp/cloudai-fio-test --rw=randwrite --bs=128k --iodepth=1 --numjobs=1 --group_reporting --thread" diff --git a/tests/ref_data/gpt-no-hook.sbatch b/tests/ref_data/gpt-no-hook.sbatch index fe6becd08..6646f6586 100644 --- a/tests/ref_data/gpt-no-hook.sbatch +++ b/tests/ref_data/gpt-no-hook.sbatch @@ -15,6 +15,7 @@ export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOL srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/gpt-pre-test.sbatch b/tests/ref_data/gpt-pre-test.sbatch index af20dd0c6..42d2c9bf7 100644 --- a/tests/ref_data/gpt-pre-test.sbatch +++ b/tests/ref_data/gpt-pre-test.sbatch @@ -15,6 +15,7 @@ export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOL srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) @@ -32,4 +33,4 @@ if [ $PRE_TEST_SUCCESS -eq 1 ]; then --container-name=cont \ --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ \ /opt/paxml/workspace/run.sh -fi \ No newline at end of file +fi diff --git a/tests/ref_data/grok-no-hook.sbatch b/tests/ref_data/grok-no-hook.sbatch index 17a9911c8..29728ca9d 100644 --- a/tests/ref_data/grok-no-hook.sbatch +++ b/tests/ref_data/grok-no-hook.sbatch @@ -15,6 +15,7 @@ export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/grok-pre-test.sbatch b/tests/ref_data/grok-pre-test.sbatch index 096f608d1..06d59b75b 100644 --- a/tests/ref_data/grok-pre-test.sbatch +++ b/tests/ref_data/grok-pre-test.sbatch @@ -15,6 +15,7 @@ export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) @@ -32,4 +33,4 @@ if [ $PRE_TEST_SUCCESS -eq 1 ]; then --container-name=cont \ --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ \ /opt/paxml/workspace/run.sh -fi \ No newline at end of file +fi diff --git a/tests/ref_data/megatron-run.sbatch b/tests/ref_data/megatron-run.sbatch index 03d7967b7..e7d3ec85a 100644 --- a/tests/ref_data/megatron-run.sbatch +++ b/tests/ref_data/megatron-run.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python __CLOUDAI_DIR__/run.py --global-batch-size 16 --hidden-size 4096 --max-position-embeddings 4096 --num-attention-heads 32 --num-layers 32 --pipeline-model-parallel-size 1 --seq-length 4096 --tensor-model-parallel-size 2 --save __CLOUDAI_DIR__ --load __CLOUDAI_DIR__ --tokenizer-model __CLOUDAI_DIR__/model.m --tensorboard-dir /cloudai_run_results/tensorboard --log-timers-to-tensorboard --log-throughput --log-memory-to-tensorboard --log-interval 1" diff --git a/tests/ref_data/moe-benchmark.sbatch b/tests/ref_data/moe-benchmark.sbatch index a3ef519d6..f674c16c3 100644 --- a/tests/ref_data/moe-benchmark.sbatch +++ b/tests/ref_data/moe-benchmark.sbatch @@ -15,6 +15,7 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nccl.sbatch b/tests/ref_data/nccl.sbatch index 362cc56f1..2a3e674f4 100644 --- a/tests/ref_data/nccl.sbatch +++ b/tests/ref_data/nccl.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" diff --git a/tests/ref_data/nemo-run-no-hook.sbatch b/tests/ref_data/nemo-run-no-hook.sbatch index ba884a6c8..adb21b009 100644 --- a/tests/ref_data/nemo-run-no-hook.sbatch +++ b/tests/ref_data/nemo-run-no-hook.sbatch @@ -15,5 +15,6 @@ export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nemo-run-pre-test.sbatch b/tests/ref_data/nemo-run-pre-test.sbatch index 5ebe4f81b..08c448dd2 100644 --- a/tests/ref_data/nemo-run-pre-test.sbatch +++ b/tests/ref_data/nemo-run-pre-test.sbatch @@ -15,10 +15,11 @@ export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) PRE_TEST_SUCCESS=$( [ $SUCCESS_0 -eq 1 ] && echo 1 || echo 0 ) if [ $PRE_TEST_SUCCESS -eq 1 ]; then srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" -fi \ No newline at end of file +fi diff --git a/tests/ref_data/nemo-run-vboost.sbatch b/tests/ref_data/nemo-run-vboost.sbatch index 0d03e6735..0a5f3a23d 100644 --- a/tests/ref_data/nemo-run-vboost.sbatch +++ b/tests/ref_data/nemo-run-vboost.sbatch @@ -18,5 +18,6 @@ srun --ntasks=1 --output=__OUTPUT_DIR__/output/vboost.out --error=__OUTPUT_DIR__ srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nixl-ep.sbatch b/tests/ref_data/nixl-ep.sbatch index 7d6096ce7..21fad7af0 100644 --- a/tests/ref_data/nixl-ep.sbatch +++ b/tests/ref_data/nixl-ep.sbatch @@ -13,5 +13,6 @@ export LD_LIBRARY_PATH=/workspace/rdma_core/lib:$LD_LIBRARY_PATH srun --export=ALL --mpi=pmix -N3 --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime bash __OUTPUT_DIR__/output/nixl-ep-launch.sh diff --git a/tests/ref_data/nixl-kvbench.sbatch b/tests/ref_data/nixl-kvbench.sbatch index 2a316a69d..9ce83a4e6 100644 --- a/tests/ref_data/nixl-kvbench.sbatch +++ b/tests/ref_data/nixl-kvbench.sbatch @@ -15,6 +15,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N2 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nixl-perftest.sbatch b/tests/ref_data/nixl-perftest.sbatch index eb809f28f..fb0f89e54 100644 --- a/tests/ref_data/nixl-perftest.sbatch +++ b/tests/ref_data/nixl-perftest.sbatch @@ -15,6 +15,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=1 --ntasks=1 -N1 bash -c "python /workspace/nixl/benchmark/kvbench/test/inference_workload_matgen.py generate --num-user-requests=2 --batch-size=1 --num-prefill-nodes=1 --num-decode-nodes=1 --results-dir=__OUTPUT_DIR__/output/matrices --prefill-tp=1 --prefill-pp=1 --prefill-cp=1 --decode-tp=1 --decode-pp=1 --decode-cp=1 --model=model-name" srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & @@ -28,4 +29,4 @@ kill -TERM $etcd_pid timeout 60 bash -c "while kill -0 $etcd_pid 2>/dev/null; do sleep 1; done" || { echo "Failed to kill ETCD (pid=$etcd_pid) within 60 seconds"; exit 1 - } \ No newline at end of file + } diff --git a/tests/ref_data/osu-bench.sbatch b/tests/ref_data/osu-bench.sbatch index 404568e6d..010981735 100644 --- a/tests/ref_data/osu-bench.sbatch +++ b/tests/ref_data/osu-bench.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ompi/tests/osu-micro-benchmarks/osu_allreduce --message-size 1024 --iterations 10 --full" diff --git a/tests/ref_data/sglang-disagg-2nodes.sbatch b/tests/ref_data/sglang-disagg-2nodes.sbatch index 12c0d45a5..cf7834987 100644 --- a/tests/ref_data/sglang-disagg-2nodes.sbatch +++ b/tests/ref_data/sglang-disagg-2nodes.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-disagg.sbatch b/tests/ref_data/sglang-disagg.sbatch index d6a4aa3eb..af43ac0a1 100644 --- a/tests/ref_data/sglang-disagg.sbatch +++ b/tests/ref_data/sglang-disagg.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-multinode.sbatch b/tests/ref_data/sglang-multinode.sbatch index 824df73b9..08a6517a0 100644 --- a/tests/ref_data/sglang-multinode.sbatch +++ b/tests/ref_data/sglang-multinode.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_SGLANG_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang.sbatch b/tests/ref_data/sglang.sbatch index cdfcbfc73..766b83ac6 100644 --- a/tests/ref_data/sglang.sbatch +++ b/tests/ref_data/sglang.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_SGLANG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/slurm_container.sbatch b/tests/ref_data/slurm_container.sbatch index ed4b4de76..f01d7ad99 100644 --- a/tests/ref_data/slurm_container.sbatch +++ b/tests/ref_data/slurm_container.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; pwd ; ls" diff --git a/tests/ref_data/ucc.sbatch b/tests/ref_data/ucc.sbatch index a4e17ba0f..5bf66fd39 100644 --- a/tests/ref_data/ucc.sbatch +++ b/tests/ref_data/ucc.sbatch @@ -13,5 +13,6 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ucc/bin/ucc_perftest -c alltoall -b 1 -e 8M -m cuda -F" diff --git a/tests/ref_data/vllm-disagg-2nodes.sbatch b/tests/ref_data/vllm-disagg-2nodes.sbatch index f9191034a..d9f2da454 100644 --- a/tests/ref_data/vllm-disagg-2nodes.sbatch +++ b/tests/ref_data/vllm-disagg-2nodes.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-disagg.sbatch b/tests/ref_data/vllm-disagg.sbatch index ffc8aedab..14eeba4bd 100644 --- a/tests/ref_data/vllm-disagg.sbatch +++ b/tests/ref_data/vllm-disagg.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-multinode.sbatch b/tests/ref_data/vllm-multinode.sbatch index 7afdccd9a..a28b9d5e7 100644 --- a/tests/ref_data/vllm-multinode.sbatch +++ b/tests/ref_data/vllm-multinode.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_VLLM_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm.sbatch b/tests/ref_data/vllm.sbatch index 2face48a9..ac6673af3 100644 --- a/tests/ref_data/vllm.sbatch +++ b/tests/ref_data/vllm.sbatch @@ -13,6 +13,7 @@ export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime cleanup() { if [ "${CLOUDAI_VLLM_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/systems/slurm/test_command_gen_strategy.py b/tests/systems/slurm/test_command_gen_strategy.py index 962855da5..1193698f2 100644 --- a/tests/systems/slurm/test_command_gen_strategy.py +++ b/tests/systems/slurm/test_command_gen_strategy.py @@ -334,7 +334,7 @@ def test_gen_srun_prefix_tr_extra_srun_args(strategy_fixture: SlurmCommandGenStr assert "--arg val --flag" in srun_prefix # added as a single element -def test_metadata_cmd_uses_host_without_changing_extra_args( +def test_metadata_cmd_collects_host_and_container_without_changing_extra_args( strategy_fixture: SlurmCommandGenStrategy, ) -> None: strategy_fixture.image_path = Mock(return_value="workload.sqsh") @@ -343,12 +343,18 @@ def test_metadata_cmd_uses_host_without_changing_extra_args( metadata_cmd = strategy_fixture._metadata_cmd() - assert "--container-image" not in metadata_cmd - assert "--container-mounts" not in metadata_cmd - assert "--mpi=none" in metadata_cmd - assert "--reservation keep" in metadata_cmd - assert "--exclusive --constraint='gpu type'" in metadata_cmd - assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh" in metadata_cmd + host_cmd, runtime_cmd = metadata_cmd.splitlines() + assert "--container-image" not in host_cmd + assert "--container-mounts" not in host_cmd + assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh" in host_cmd + assert "--container-image=workload.sqsh" in runtime_cmd + assert "--container-mounts" in runtime_cmd + assert "--open-mode=append" in runtime_cmd + assert "bash /cloudai_install/slurm-metadata.sh runtime" in runtime_cmd + for command in (host_cmd, runtime_cmd): + assert "--mpi=none" in command + assert "--reservation keep" in command + assert "--exclusive --constraint='gpu type'" in command def test_append_distribution_and_hostfile_with_nodes(slurm_system: SlurmSystem, testrun_fixture: TestRun) -> None: diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py index e6199f129..6974c763a 100644 --- a/tests/systems/slurm/test_metadata_script.py +++ b/tests/systems/slurm/test_metadata_script.py @@ -19,7 +19,7 @@ def _write_command(bin_dir: Path, name: str, body: str) -> None: command.chmod(0o755) -def _run_collector(tmp_path: Path, commands: dict[str, str]) -> subprocess.CompletedProcess[str]: +def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "host") -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" bin_dir.mkdir() for name, body in commands.items(): @@ -40,7 +40,9 @@ def _run_collector(tmp_path: Path, commands: dict[str, str]) -> subprocess.Compl "CLOUDAI_DOCA_ROOT": str(tmp_path / "missing-doca"), "SLURM_JOBID": "12345", } - return subprocess.run(["/bin/bash", str(METADATA_SCRIPT)], env=env, text=True, capture_output=True, check=False) + return subprocess.run( + ["/bin/bash", str(METADATA_SCRIPT), mode], env=env, text=True, capture_output=True, check=False + ) def test_collects_requested_host_metadata(tmp_path: Path) -> None: @@ -63,7 +65,10 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: 0000:31:00.0 Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] 0000:65:00.0 VGA compatible controller [0300]: NVIDIA Corporation Device [10de:2330] EOF""", - "dpkg-query": "printf '2.9.2-0.1.0\\n'", + "dpkg-query": """case "$*" in + *doca-ofed) printf '2.9.2-0.1.0\\n' ;; + *) exit 1 ;; +esac""", "ofed_info": "printf 'MLNX_OFED_LINUX-24.10-1.1.4.0:\\n'", "fi_info": "printf 'libfabric: 1.22.0\\n'", "mpirun": "printf 'mpirun (Open MPI) 4.1.7a1\\n'", @@ -87,7 +92,7 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: "cpu_vendor": "GenuineIntel", } assert metadata["cuda"] == { - "cuda_build_version": "unknown", + "cuda_build_version": "null", "cuda_runtime_version": "12.8", "cuda_driver_version": "570.124.06", "nvidia_driver_version": "570.124.06", @@ -98,8 +103,8 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: "nic_count": 2, "nic_inventory": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] x2", "hca_firmware_versions": "mlx5_0=28.43.2026", - "switch_type": "unknown", - "network_name": "unknown", + "switch_type": "null", + "network_name": "null", "mofed_version": "MLNX_OFED_LINUX-24.10-1.1.4.0", "doca_host_version": "2.9.2-0.1.0", "libfabric_version": "1.22.0", @@ -116,8 +121,37 @@ def test_missing_subcommands_do_not_fail_or_corrupt_output(tmp_path: Path) -> No assert result.returncode == 0 metadata = toml.loads(result.stdout) assert metadata["system"]["gpu_count"] == 0 - assert metadata["system"]["gpu_arch_type"] == "unknown" - assert metadata["cuda"]["cuda_toolkit_version"] == "unknown" - assert metadata["cuda"]["nvidia_driver_version"] == "unknown" + assert metadata["system"]["gpu_arch_type"] == "null" + assert metadata["cuda"]["cuda_toolkit_version"] == "null" + assert metadata["cuda"]["nvidia_driver_version"] == "null" assert metadata["network"]["nic_count"] == 0 - assert metadata["network"]["doca_host_version"] == "unknown" + assert metadata["network"]["doca_host_version"] == "null" + + +def test_collects_container_metadata_as_runtime_table(tmp_path: Path) -> None: + result = _run_collector( + tmp_path, + { + "nvidia-smi": "printf 'NVIDIA-SMI CUDA Version: 13.1\\n'", + "nvcc": "printf 'Cuda compilation tools, release 13.0, V13.0.1\\n'", + "mpirun": "printf 'mpirun (Open MPI) 4.1.9a1\\n'", + }, + mode="runtime", + ) + + assert result.returncode == 0 + assert result.stderr == "" + assert toml.loads(result.stdout) == { + "runtime": { + "os_type": "ubuntu", + "os_version": "24.04 LTS", + "mpi_type": "openmpi", + "mpi_version": "4.1.9a1", + "hpcx_version": "null", + "cuda_build_version": "null", + "cuda_runtime_version": "13.1", + "cuda_toolkit_version": "13.0", + "nccl_version": "null", + "nccl_commit_sha": "null", + } + } diff --git a/tests/test_single_sbatch_runner.py b/tests/test_single_sbatch_runner.py index 8a645f569..11abdf3d8 100644 --- a/tests/test_single_sbatch_runner.py +++ b/tests/test_single_sbatch_runner.py @@ -275,10 +275,15 @@ def test_container(self, nccl_tr: TestRun, slurm_system: SlurmSystem) -> None: f"srun --export=ALL --mpi=none -N{nccl_tr.num_nodes} --ntasks=2 --ntasks-per-node=1 " f"--output={runner.scenario_root}/metadata/node-%N.toml " f"--error={runner.scenario_root}/metadata/nodes.err " - f"bash {runner.system.install_path}/slurm-metadata.sh" + f"bash {runner.system.install_path}/slurm-metadata.sh\n" + f"srun --export=ALL --mpi=none -N{nccl_tr.num_nodes} --ntasks=2 --ntasks-per-node=1 " + f"--output={runner.scenario_root}/metadata/node-%N.toml " + f"--error={runner.scenario_root}/metadata/nodes.err --open-mode=append " + f"--container-image={tdef.docker_image.installed_path} {mounts} --no-container-mount-home " + "bash /cloudai_install/slurm-metadata.sh runtime" ) assert aux_cmds[0] == metadata_cmd - assert "--container-image" not in aux_cmds[0] + assert "--container-image" not in aux_cmds[0].splitlines()[0] ranks_mapping_cmd = ( f"srun --export=ALL --mpi=pmix -N{nccl_tr.num_nodes} --container-image={tdef.docker_image.installed_path} " @@ -364,11 +369,12 @@ def test_single_case(self, slurm_system: SlurmSystem, nccl_tr: TestRun) -> None: p.replace(f"{runner.scenario_root.absolute()}", "SCENARIO_ROOT") for p in re.findall(r"--output=(\S+)", sbatch) ] - assert len(output_paths) == 4 + assert len(output_paths) == 5 assert output_paths[0] == "SCENARIO_ROOT/common.out" assert output_paths[1] == "SCENARIO_ROOT/metadata/node-%N.toml" - assert output_paths[2] == "SCENARIO_ROOT/mapping-stdout.txt" - assert output_paths[3] == f"SCENARIO_ROOT/{nccl_tr.name}/0/stdout.txt" + assert output_paths[2] == "SCENARIO_ROOT/metadata/node-%N.toml" + assert output_paths[3] == "SCENARIO_ROOT/mapping-stdout.txt" + assert output_paths[4] == f"SCENARIO_ROOT/{nccl_tr.name}/0/stdout.txt" def test_with_two_cases(self, slurm_system: SlurmSystem, nccl_tr: TestRun, sleep_tr: TestRun) -> None: nccl_tr.test.extra_env_vars["NCCL_VAR"] = "nccl_value" @@ -408,7 +414,7 @@ def test_with_two_cases(self, slurm_system: SlurmSystem, nccl_tr: TestRun, sleep p.replace(f"{runner.scenario_root.absolute()}", "SCENARIO_ROOT") for p in re.findall(r"--output=(\S+)", sbatch) ] - assert len(output_paths) == 5 + assert len(output_paths) == 6 assert output_paths[-2] != output_paths[-1] def test_dse(self, nccl_tr: TestRun, slurm_system: SlurmSystem) -> None: From 278373ec91f7416b75073838fe870c217dd167b5 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 14:21:20 +0200 Subject: [PATCH 3/8] make enhanced slurm metadata schema backwards compatible --- src/cloudai/systems/slurm/slurm-metadata.sh | 53 +++++++++---------- .../slurm/slurm_command_gen_strategy.py | 18 +++++-- src/cloudai/systems/slurm/slurm_metadata.py | 24 ++++----- tests/ref_data/ai-dynamo.sbatch | 4 +- tests/ref_data/ddlb.sbatch | 4 +- tests/ref_data/deepep-benchmark.sbatch | 4 +- tests/ref_data/fio.sbatch | 4 +- tests/ref_data/gpt-no-hook.sbatch | 4 +- tests/ref_data/gpt-pre-test.sbatch | 4 +- tests/ref_data/grok-no-hook.sbatch | 4 +- tests/ref_data/grok-pre-test.sbatch | 4 +- tests/ref_data/megatron-run.sbatch | 4 +- tests/ref_data/moe-benchmark.sbatch | 4 +- tests/ref_data/nccl.sbatch | 4 +- tests/ref_data/nemo-run-no-hook.sbatch | 4 +- tests/ref_data/nemo-run-pre-test.sbatch | 4 +- tests/ref_data/nemo-run-vboost.sbatch | 4 +- tests/ref_data/nixl-ep.sbatch | 4 +- tests/ref_data/nixl-kvbench.sbatch | 4 +- tests/ref_data/nixl-perftest.sbatch | 4 +- tests/ref_data/osu-bench.sbatch | 4 +- tests/ref_data/sglang-disagg-2nodes.sbatch | 4 +- tests/ref_data/sglang-disagg.sbatch | 4 +- tests/ref_data/sglang-multinode.sbatch | 4 +- tests/ref_data/sglang.sbatch | 4 +- tests/ref_data/slurm_container.sbatch | 4 +- tests/ref_data/ucc.sbatch | 4 +- tests/ref_data/vllm-disagg-2nodes.sbatch | 4 +- tests/ref_data/vllm-disagg.sbatch | 4 +- tests/ref_data/vllm-multinode.sbatch | 4 +- tests/ref_data/vllm.sbatch | 4 +- .../slurm/test_command_gen_strategy.py | 6 +-- tests/systems/slurm/test_metadata_script.py | 52 ++++++++++++------ tests/test_single_sbatch_runner.py | 8 +-- 34 files changed, 146 insertions(+), 127 deletions(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index d88b1e792..345e61430 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -235,22 +235,6 @@ if [ -r "$os_release_file" ]; then . "$os_release_file" 2>/dev/null || true fi -if [ "${1:-host}" = "runtime" ]; then - printf '[runtime]\n' - emit_string os_type "${ID:-$UNKNOWN}" - emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" - emit_string mpi_type "$(safe_probe mpi_type_probe)" - emit_string mpi_version "$(safe_probe mpi_version_probe)" - hpcx_version=${HPCX_DIR##*/} - emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" - emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" - emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" - emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" - emit_string nccl_version "${NCCL_VERSION:-$UNKNOWN}" - emit_string nccl_commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" - exit 0 -fi - user="${USER:-$(safe_probe whoami)}" kernel_version="$(safe_probe uname -r)" cpu_model="$(safe_probe lscpu_field "Model name")" @@ -272,8 +256,17 @@ nic_count="$(printf '%s\n' "$nic_lines" | nonempty_line_count 2>/dev/null)" || n nic_inventory="$(printf '%s\n' "$nic_models" | inventory_from_lines 2>/dev/null)" nic_inventory="${nic_inventory:-$UNKNOWN}" -emit_string user "$user" -printf '\n[system]\n' +mode="${1:-runtime}" +if [ "$mode" = "host" ]; then + printf '[host]\n' + emit_string user "$user" + table_prefix="host." +else + emit_string user "$user" + table_prefix="" +fi + +printf '\n[%ssystem]\n' "$table_prefix" emit_string os_type "${ID:-$UNKNOWN}" emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" emit_string linux_kernel_version "$kernel_version" @@ -284,13 +277,13 @@ emit_string cpu_model_name "$cpu_model" emit_string cpu_arch_type "$cpu_arch" emit_string cpu_vendor "$cpu_vendor" -printf '\n[mpi]\n' +printf '\n[%smpi]\n' "$table_prefix" emit_string mpi_type "$(safe_probe mpi_type_probe)" emit_string mpi_version "$(safe_probe mpi_version_probe)" hpcx_version=${HPCX_DIR##*/} emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" -printf '\n[cuda]\n' +printf '\n[%scuda]\n' "$table_prefix" emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" driver_version="$(safe_probe gpu_driver_probe)" @@ -298,7 +291,7 @@ emit_string cuda_driver_version "$driver_version" emit_string nvidia_driver_version "$driver_version" emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" -printf '\n[network]\n' +printf '\n[%snetwork]\n' "$table_prefix" emit_string nics "$nics" emit_integer nic_count "$nic_count" emit_string nic_inventory "$nic_inventory" @@ -309,16 +302,18 @@ emit_string mofed_version "$(safe_probe ofed_version_probe)" emit_string doca_host_version "$(safe_probe doca_host_version_probe)" emit_string libfabric_version "$(safe_probe libfabric_version_probe)" -printf '\n[nccl]\n' +printf '\n[%snccl]\n' "$table_prefix" emit_string version "${NCCL_VERSION:-$UNKNOWN}" emit_string commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" -printf '\n[slurm]\n' -emit_string cluster_name "${SLURM_CLUSTER_NAME:-$UNKNOWN}" -emit_string node_list "${SLURM_NODELIST:-$UNKNOWN}" -emit_string num_nodes "${SLURM_NNODES:-$UNKNOWN}" -emit_string ntasks_per_node "${SLURM_NTASKS_PER_NODE:-$UNKNOWN}" -emit_string ntasks "${SLURM_NTASKS:-$UNKNOWN}" -emit_string job_id "${SLURM_JOBID:-$UNKNOWN}" +if [ "$mode" != "host" ]; then + printf '\n[slurm]\n' + emit_string cluster_name "${SLURM_CLUSTER_NAME:-$UNKNOWN}" + emit_string node_list "${SLURM_NODELIST:-$UNKNOWN}" + emit_string num_nodes "${SLURM_NNODES:-$UNKNOWN}" + emit_string ntasks_per_node "${SLURM_NTASKS_PER_NODE:-$UNKNOWN}" + emit_string ntasks "${SLURM_NTASKS:-$UNKNOWN}" + emit_string job_id "${SLURM_JOBID:-$UNKNOWN}" +fi exit 0 diff --git a/src/cloudai/systems/slurm/slurm_command_gen_strategy.py b/src/cloudai/systems/slurm/slurm_command_gen_strategy.py index f07f0f52b..2c5067954 100644 --- a/src/cloudai/systems/slurm/slurm_command_gen_strategy.py +++ b/src/cloudai/systems/slurm/slurm_command_gen_strategy.py @@ -331,13 +331,11 @@ def _metadata_cmd(self) -> str: f"--error={metadata_error}", ] - host_command = " ".join([*metadata_srun_prefix, "bash", f"{metadata_script_path}/slurm-metadata.sh"]) - image_path = self.image_path() if not image_path: - return host_command + return " ".join([*metadata_srun_prefix, "bash", f"{metadata_script_path}/slurm-metadata.sh"]) - runtime_command_parts = [*metadata_srun_prefix, "--open-mode=append", f"--container-image={image_path}"] + runtime_command_parts = [*metadata_srun_prefix, f"--container-image={image_path}"] mounts = self.container_mounts() if mounts: runtime_command_parts.append(f"--container-mounts={','.join(mounts)}") @@ -345,7 +343,17 @@ def _metadata_cmd(self) -> str: runtime_command_parts.append("--no-container-mount-home") runtime_command_parts.extend(["bash", f"{self.CONTAINER_MOUNT_INSTALL}/slurm-metadata.sh", "runtime"]) runtime_command = " ".join(runtime_command_parts) - return "\n".join([host_command, runtime_command]) + + host_command = " ".join( + [ + *metadata_srun_prefix, + "--open-mode=append", + "bash", + f"{metadata_script_path}/slurm-metadata.sh", + "host", + ] + ) + return "\n".join([runtime_command, host_command]) def _enable_vboost_cmd(self) -> str: num_nodes, _ = self.system.get_nodes_by_spec(self.test_run.nnodes, self.test_run.nodes) diff --git a/src/cloudai/systems/slurm/slurm_metadata.py b/src/cloudai/systems/slurm/slurm_metadata.py index 6c71643c7..29707ce93 100644 --- a/src/cloudai/systems/slurm/slurm_metadata.py +++ b/src/cloudai/systems/slurm/slurm_metadata.py @@ -151,19 +151,15 @@ class MetadataSlurm(BaseModel): job_id: str -class MetadataRuntime(BaseModel): - """Represents software metadata collected inside the workload container.""" - - os_type: str = "null" - os_version: str = "null" - mpi_type: str = "null" - mpi_version: str = "null" - hpcx_version: str = "null" - cuda_build_version: str = "null" - cuda_runtime_version: str = "null" - cuda_toolkit_version: str = "null" - nccl_version: str = "null" - nccl_commit_sha: str = "null" +class MetadataHost(BaseModel): + """Represents metadata collected directly from the Slurm host.""" + + user: str + system: MetadataSystem + mpi: MetadataMPI + cuda: MetadataCUDA + network: MetadataNetwork + nccl: MetadataNCCL class SlurmSystemMetadata(BaseModel): @@ -176,4 +172,4 @@ class SlurmSystemMetadata(BaseModel): network: MetadataNetwork nccl: MetadataNCCL slurm: MetadataSlurm - runtime: MetadataRuntime = Field(default_factory=MetadataRuntime) + host: MetadataHost | None = None diff --git a/tests/ref_data/ai-dynamo.sbatch b/tests/ref_data/ai-dynamo.sbatch index abbaefe9b..353053c57 100644 --- a/tests/ref_data/ai-dynamo.sbatch +++ b/tests/ref_data/ai-dynamo.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N2 --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/ai-dynamo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/cloudai_install/huggingface,/tmp:/tmp bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host # Start DCGM exporter on each node. echo "Starting DCGM exporter..." diff --git a/tests/ref_data/ddlb.sbatch b/tests/ref_data/ddlb.sbatch index 2afbb18a1..4635b0ca2 100644 --- a/tests/ref_data/ddlb.sbatch +++ b/tests/ref_data/ddlb.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python ddlb/cli/benchmark.py --primitive tp_columnwise -m 1024 -n 128 -k 1024 --dtype float16 --num-iterations 50 --num-warmups 5 --impl pytorch;backend=nccl;order=AG_before" diff --git a/tests/ref_data/deepep-benchmark.sbatch b/tests/ref_data/deepep-benchmark.sbatch index dc3dfaf57..b346c5e41 100644 --- a/tests/ref_data/deepep-benchmark.sbatch +++ b/tests/ref_data/deepep-benchmark.sbatch @@ -25,7 +25,7 @@ export MASTER_PORT=29500 srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --nodes=2 --ntasks=2 --ntasks-per-node=1 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; torchrun --nnodes=2 --nproc_per_node=1 --rdzv_id=\${SLURM_JOB_ID:-0} --rdzv_backend=c10d --rdzv_endpoint=\${MASTER_ADDR}:\${MASTER_PORT} /workspace/DeepEP/tests/legacy/test_internode.py --num-processes 8 --num-tokens 4096 --hidden 7168 --num-topk 8 --pressure-test-mode 0 --num-experts 256" diff --git a/tests/ref_data/fio.sbatch b/tests/ref_data/fio.sbatch index d6aec262e..7bbfb6e2d 100644 --- a/tests/ref_data/fio.sbatch +++ b/tests/ref_data/fio.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL -N2 --container-image=openeuler/fio:3.42-oe2403sp3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=2 --ntasks=4 bash -c "source __OUTPUT_DIR__/output/env_vars.sh; fio --name=fio-smoke --filename=/tmp/cloudai-fio-test --rw=randwrite --bs=128k --iodepth=1 --numjobs=1 --group_reporting --thread" diff --git a/tests/ref_data/gpt-no-hook.sbatch b/tests/ref_data/gpt-no-hook.sbatch index 6646f6586..a83cbe616 100644 --- a/tests/ref_data/gpt-no-hook.sbatch +++ b/tests/ref_data/gpt-no-hook.sbatch @@ -14,8 +14,8 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/gpt-pre-test.sbatch b/tests/ref_data/gpt-pre-test.sbatch index 42d2c9bf7..0b6445ff1 100644 --- a/tests/ref_data/gpt-pre-test.sbatch +++ b/tests/ref_data/gpt-pre-test.sbatch @@ -14,8 +14,8 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/grok-no-hook.sbatch b/tests/ref_data/grok-no-hook.sbatch index 29728ca9d..dca9f7a89 100644 --- a/tests/ref_data/grok-no-hook.sbatch +++ b/tests/ref_data/grok-no-hook.sbatch @@ -14,8 +14,8 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass_re=.* --xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_enable_all_gather_combine_by_dim=false --xla_gpu_enable_highest_priority_async_stream=true --xla_gpu_enable_latency_hiding_scheduler=true --xla_gpu_enable_pipelined_all_gather=true --xla_gpu_enable_pipelined_all_reduce=true --xla_gpu_enable_pipelined_reduce_scatter=true --xla_gpu_enable_reduce_scatter_combine_by_dim=false --xla_gpu_enable_triton_gemm=false --xla_gpu_enable_triton_softmax_fusion=false --xla_gpu_enable_while_loop_double_buffering=true --xla_gpu_graph_level=0 --xla_gpu_pgle_profile_file_or_directory_path=/opt/paxml/workspace/pgle_output_profile.pbtxt --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD --xla_gpu_run_post_layout_collective_pipeliner=false --xla_gpu_use_memcpy_local_p2p=false" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host echo "Loading container with srun command" srun --mpi=none --container-image=https://docker/url --container-name=cont true diff --git a/tests/ref_data/grok-pre-test.sbatch b/tests/ref_data/grok-pre-test.sbatch index 06d59b75b..689282742 100644 --- a/tests/ref_data/grok-pre-test.sbatch +++ b/tests/ref_data/grok-pre-test.sbatch @@ -14,8 +14,8 @@ export PER_GPU_COMBINE_THRESHOLD=0 export XLA_FLAGS="--xla_disable_hlo_passes=rematerialization --xla_dump_hlo_pass_re=.* --xla_gpu_all_gather_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_all_reduce_combine_threshold_bytes=$COMBINE_THRESHOLD --xla_gpu_enable_all_gather_combine_by_dim=false --xla_gpu_enable_highest_priority_async_stream=true --xla_gpu_enable_latency_hiding_scheduler=true --xla_gpu_enable_pipelined_all_gather=true --xla_gpu_enable_pipelined_all_reduce=true --xla_gpu_enable_pipelined_reduce_scatter=true --xla_gpu_enable_reduce_scatter_combine_by_dim=false --xla_gpu_enable_triton_gemm=false --xla_gpu_enable_triton_softmax_fusion=false --xla_gpu_enable_while_loop_double_buffering=true --xla_gpu_graph_level=0 --xla_gpu_pgle_profile_file_or_directory_path=/opt/paxml/workspace/pgle_output_profile.pbtxt --xla_gpu_reduce_scatter_combine_threshold_bytes=$PER_GPU_COMBINE_THRESHOLD --xla_gpu_run_post_layout_collective_pipeliner=false --xla_gpu_use_memcpy_local_p2p=false" srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output:/opt/paxml/workspace/ bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/megatron-run.sbatch b/tests/ref_data/megatron-run.sbatch index e7d3ec85a..7b25f4bd2 100644 --- a/tests/ref_data/megatron-run.sbatch +++ b/tests/ref_data/megatron-run.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/megatron:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,$PWD bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python __CLOUDAI_DIR__/run.py --global-batch-size 16 --hidden-size 4096 --max-position-embeddings 4096 --num-attention-heads 32 --num-layers 32 --pipeline-model-parallel-size 1 --seq-length 4096 --tensor-model-parallel-size 2 --save __CLOUDAI_DIR__ --load __CLOUDAI_DIR__ --tokenizer-model __CLOUDAI_DIR__/model.m --tensorboard-dir /cloudai_run_results/tensorboard --log-timers-to-tensorboard --log-throughput --log-memory-to-tensorboard --log-interval 1" diff --git a/tests/ref_data/moe-benchmark.sbatch b/tests/ref_data/moe-benchmark.sbatch index f674c16c3..90da9f4de 100644 --- a/tests/ref_data/moe-benchmark.sbatch +++ b/tests/ref_data/moe-benchmark.sbatch @@ -14,8 +14,8 @@ export MASTER_PORT=29500 export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" srun --export=ALL --mpi=pmix -N2 --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix --container-image=docker/image:url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output,__OUTPUT_DIR__/output/config.yaml:/tmp/config.yaml,__OUTPUT_DIR__/output:/workspace/DeepEP/results --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nccl.sbatch b/tests/ref_data/nccl.sbatch index 2a3e674f4..356ea400e 100644 --- a/tests/ref_data/nccl.sbatch +++ b/tests/ref_data/nccl.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" diff --git a/tests/ref_data/nemo-run-no-hook.sbatch b/tests/ref_data/nemo-run-no-hook.sbatch index adb21b009..d03fb6868 100644 --- a/tests/ref_data/nemo-run-no-hook.sbatch +++ b/tests/ref_data/nemo-run-no-hook.sbatch @@ -14,7 +14,7 @@ export CLOUDAI_NEMO_RECIPE=llama_3b export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nemo-run-pre-test.sbatch b/tests/ref_data/nemo-run-pre-test.sbatch index 08c448dd2..99493df49 100644 --- a/tests/ref_data/nemo-run-pre-test.sbatch +++ b/tests/ref_data/nemo-run-pre-test.sbatch @@ -14,8 +14,8 @@ export CLOUDAI_NEMO_RECIPE=llama_3b export NEMO_LOG_MEMORY_USAGE=1 srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --output=__OUTPUT_DIR__/output/pre_test/nccl/stdout.txt --error=__OUTPUT_DIR__/output/pre_test/nccl/stderr.txt --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output/pre_test/nccl:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output/pre_test/nccl bash -c "source __OUTPUT_DIR__/output/pre_test/nccl/env_vars.sh; all_reduce_perf_mpi --nthreads 1 --ngpus 1 --minbytes 32M --maxbytes 32M --stepbytes 1M --op sum --datatype float --root 0 --iters 20 --warmup_iters 5 --agg_iters 1 --average 1 --parallel_init 0 --check 1 --blocking 0 --cudagraph 0" SUCCESS_0=$(grep -q "Avg bus bandwidth" __OUTPUT_DIR__/output/pre_test/nccl/stdout.txt && echo 1 || echo 0) diff --git a/tests/ref_data/nemo-run-vboost.sbatch b/tests/ref_data/nemo-run-vboost.sbatch index 0a5f3a23d..205e4e1eb 100644 --- a/tests/ref_data/nemo-run-vboost.sbatch +++ b/tests/ref_data/nemo-run-vboost.sbatch @@ -17,7 +17,7 @@ srun --ntasks=1 --output=__OUTPUT_DIR__/output/vboost.out --error=__OUTPUT_DIR__ srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io/nvidia/nemo:24.09 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__CLOUDAI_DIR__/src/cloudai/workloads/nemo_run:/cloudai_workspace bash -c "source __OUTPUT_DIR__/output/env_vars.sh; python /cloudai_install/cloudai_nemorun.py --factory llama_3b -y trainer.max_steps=100 trainer.val_check_interval=1000 trainer.num_nodes=1 trainer.strategy.tensor_model_parallel_size=1 trainer.strategy.pipeline_model_parallel_size=1 trainer.strategy.context_parallel_size=2 trainer.log_every_n_steps=1 trainer.devices=8 data.seq_length=8192 data.micro_batch_size=1 data.global_batch_size=1 data.num_train_samples=100" diff --git a/tests/ref_data/nixl-ep.sbatch b/tests/ref_data/nixl-ep.sbatch index 21fad7af0..40d0edfb4 100644 --- a/tests/ref_data/nixl-ep.sbatch +++ b/tests/ref_data/nixl-ep.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export LD_LIBRARY_PATH=/workspace/rdma_core/lib:$LD_LIBRARY_PATH srun --export=ALL --mpi=pmix -N3 --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker.io/nvidia/nixl-ep:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__INSTALL_DIR__:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N3 --ntasks=3 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host bash __OUTPUT_DIR__/output/nixl-ep-launch.sh diff --git a/tests/ref_data/nixl-kvbench.sbatch b/tests/ref_data/nixl-kvbench.sbatch index 9ce83a4e6..2cf096063 100644 --- a/tests/ref_data/nixl-kvbench.sbatch +++ b/tests/ref_data/nixl-kvbench.sbatch @@ -14,8 +14,8 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head -n 1) srun --export=ALL --mpi=pmix -N2 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & etcd_pid=$! diff --git a/tests/ref_data/nixl-perftest.sbatch b/tests/ref_data/nixl-perftest.sbatch index fb0f89e54..5938c60be 100644 --- a/tests/ref_data/nixl-perftest.sbatch +++ b/tests/ref_data/nixl-perftest.sbatch @@ -14,8 +14,8 @@ export NIXL_ETCD_ENDPOINTS="$SLURM_JOB_MASTER_NODE:2379" export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head -n 1) srun --export=ALL --mpi=pmix -N1 --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --ntasks-per-node=1 --ntasks=1 -N1 bash -c "python /workspace/nixl/benchmark/kvbench/test/inference_workload_matgen.py generate --num-user-requests=2 --batch-size=1 --num-prefill-nodes=1 --num-decode-nodes=1 --results-dir=__OUTPUT_DIR__/output/matrices --prefill-tp=1 --prefill-pp=1 --prefill-cp=1 --decode-tp=1 --decode-pp=1 --decode-cp=1 --model=model-name" srun --export=ALL --mpi=pmix --container-image=url.com/docker:tag --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/etcd.log --overlap --ntasks-per-node=1 --ntasks=1 --nodelist=$SLURM_JOB_MASTER_NODE -N1 etcd --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://$SLURM_JOB_MASTER_NODE:2379 --listen-peer-urls=http://0.0.0.0:2380 --initial-advertise-peer-urls=http://$SLURM_JOB_MASTER_NODE:2380 --initial-cluster="default=http://$SLURM_JOB_MASTER_NODE:2380" --initial-cluster-state=new & diff --git a/tests/ref_data/osu-bench.sbatch b/tests/ref_data/osu-bench.sbatch index 010981735..b08c8a2e5 100644 --- a/tests/ref_data/osu-bench.sbatch +++ b/tests/ref_data/osu-bench.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ompi/tests/osu-micro-benchmarks/osu_allreduce --message-size 1024 --iterations 10 --full" diff --git a/tests/ref_data/sglang-disagg-2nodes.sbatch b/tests/ref_data/sglang-disagg-2nodes.sbatch index cf7834987..545dcb8f6 100644 --- a/tests/ref_data/sglang-disagg-2nodes.sbatch +++ b/tests/ref_data/sglang-disagg-2nodes.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-disagg.sbatch b/tests/ref_data/sglang-disagg.sbatch index af43ac0a1..fed39454b 100644 --- a/tests/ref_data/sglang-disagg.sbatch +++ b/tests/ref_data/sglang-disagg.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_SGLANG_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang-multinode.sbatch b/tests/ref_data/sglang-multinode.sbatch index 08a6517a0..b9fdc5ab6 100644 --- a/tests/ref_data/sglang-multinode.sbatch +++ b/tests/ref_data/sglang-multinode.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_SGLANG_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/sglang.sbatch b/tests/ref_data/sglang.sbatch index 766b83ac6..070fc9381 100644 --- a/tests/ref_data/sglang.sbatch +++ b/tests/ref_data/sglang.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=docker.io/lmsysorg/sglang:dev --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_SGLANG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/slurm_container.sbatch b/tests/ref_data/slurm_container.sbatch index f01d7ad99..7690e4d23 100644 --- a/tests/ref_data/slurm_container.sbatch +++ b/tests/ref_data/slurm_container.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=https://docker/url --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; pwd ; ls" diff --git a/tests/ref_data/ucc.sbatch b/tests/ref_data/ucc.sbatch index 5bf66fd39..390130eda 100644 --- a/tests/ref_data/ucc.sbatch +++ b/tests/ref_data/ucc.sbatch @@ -12,7 +12,7 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host srun --export=ALL --mpi=pmix -N1 --container-image=nvcr.io#nvidia/pytorch:24.02-py3 --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output bash -c "source __OUTPUT_DIR__/output/env_vars.sh; /opt/hpcx/ucc/bin/ucc_perftest -c alltoall -b 1 -e 8M -m cuda -F" diff --git a/tests/ref_data/vllm-disagg-2nodes.sbatch b/tests/ref_data/vllm-disagg-2nodes.sbatch index d9f2da454..6d020faa5 100644 --- a/tests/ref_data/vllm-disagg-2nodes.sbatch +++ b/tests/ref_data/vllm-disagg-2nodes.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_2NODES_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-disagg.sbatch b/tests/ref_data/vllm-disagg.sbatch index 14eeba4bd..ef5060196 100644 --- a/tests/ref_data/vllm-disagg.sbatch +++ b/tests/ref_data/vllm-disagg.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_VLLM_DISAGG_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm-multinode.sbatch b/tests/ref_data/vllm-multinode.sbatch index a28b9d5e7..2cb2a91ce 100644 --- a/tests/ref_data/vllm-multinode.sbatch +++ b/tests/ref_data/vllm-multinode.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0,1,2,3 srun --export=ALL --mpi=none -N2 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N2 --ntasks=2 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_VLLM_MULTINODE_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/ref_data/vllm.sbatch b/tests/ref_data/vllm.sbatch index ac6673af3..54df16369 100644 --- a/tests/ref_data/vllm.sbatch +++ b/tests/ref_data/vllm.sbatch @@ -12,8 +12,8 @@ export SLURM_JOB_MASTER_NODE=$(scontrol show hostname $SLURM_JOB_NODELIST | head export CUDA_VISIBLE_DEVICES=0 srun --export=ALL --mpi=none -N1 --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface --output=__OUTPUT_DIR__/output/mapping-stdout.txt --error=__OUTPUT_DIR__/output/mapping-stderr.txt bash -c "echo \$(date): \$(hostname):node \${SLURM_NODEID}:rank \${SLURM_PROCID}." -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err bash __INSTALL_DIR__/slurm-metadata.sh -srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --container-image=nvcr.io/nvidia/vllm:latest --container-mounts=__OUTPUT_DIR__/output:/cloudai_run_results,__OUTPUT_DIR__/install:/cloudai_install,__OUTPUT_DIR__/output,__INSTALL_DIR__/huggingface:/root/.cache/huggingface bash /cloudai_install/slurm-metadata.sh runtime +srun --export=ALL --mpi=none -N1 --ntasks=1 --ntasks-per-node=1 --output=__OUTPUT_DIR__/output/metadata/node-%N.toml --error=__OUTPUT_DIR__/output/metadata/nodes.err --open-mode=append bash __INSTALL_DIR__/slurm-metadata.sh host cleanup() { if [ "${CLOUDAI_VLLM_0_CLEANUP_DONE:-0}" = "1" ]; then diff --git a/tests/systems/slurm/test_command_gen_strategy.py b/tests/systems/slurm/test_command_gen_strategy.py index 1193698f2..37ff835ad 100644 --- a/tests/systems/slurm/test_command_gen_strategy.py +++ b/tests/systems/slurm/test_command_gen_strategy.py @@ -343,13 +343,13 @@ def test_metadata_cmd_collects_host_and_container_without_changing_extra_args( metadata_cmd = strategy_fixture._metadata_cmd() - host_cmd, runtime_cmd = metadata_cmd.splitlines() + runtime_cmd, host_cmd = metadata_cmd.splitlines() assert "--container-image" not in host_cmd assert "--container-mounts" not in host_cmd - assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh" in host_cmd + assert "--open-mode=append" in host_cmd + assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh host" in host_cmd assert "--container-image=workload.sqsh" in runtime_cmd assert "--container-mounts" in runtime_cmd - assert "--open-mode=append" in runtime_cmd assert "bash /cloudai_install/slurm-metadata.sh runtime" in runtime_cmd for command in (host_cmd, runtime_cmd): assert "--mpi=none" in command diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py index 6974c763a..f7a21b35c 100644 --- a/tests/systems/slurm/test_metadata_script.py +++ b/tests/systems/slurm/test_metadata_script.py @@ -9,6 +9,7 @@ import toml from cloudai.systems import slurm +from cloudai.systems.slurm import SlurmSystemMetadata METADATA_SCRIPT = Path(slurm.__file__).parent / "slurm-metadata.sh" @@ -19,7 +20,7 @@ def _write_command(bin_dir: Path, name: str, body: str) -> None: command.chmod(0o755) -def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "host") -> subprocess.CompletedProcess[str]: +def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "runtime") -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" bin_dir.mkdir() for name, body in commands.items(): @@ -128,7 +129,7 @@ def test_missing_subcommands_do_not_fail_or_corrupt_output(tmp_path: Path) -> No assert metadata["network"]["doca_host_version"] == "null" -def test_collects_container_metadata_as_runtime_table(tmp_path: Path) -> None: +def test_collects_container_metadata_in_legacy_sections(tmp_path: Path) -> None: result = _run_collector( tmp_path, { @@ -141,17 +142,36 @@ def test_collects_container_metadata_as_runtime_table(tmp_path: Path) -> None: assert result.returncode == 0 assert result.stderr == "" - assert toml.loads(result.stdout) == { - "runtime": { - "os_type": "ubuntu", - "os_version": "24.04 LTS", - "mpi_type": "openmpi", - "mpi_version": "4.1.9a1", - "hpcx_version": "null", - "cuda_build_version": "null", - "cuda_runtime_version": "13.1", - "cuda_toolkit_version": "13.0", - "nccl_version": "null", - "nccl_commit_sha": "null", - } - } + metadata = toml.loads(result.stdout) + assert "runtime" not in metadata + assert metadata["system"]["os_version"] == "24.04 LTS" + assert metadata["mpi"] == {"mpi_type": "openmpi", "mpi_version": "4.1.9a1", "hpcx_version": "null"} + assert metadata["cuda"]["cuda_runtime_version"] == "13.1" + assert metadata["cuda"]["cuda_toolkit_version"] == "13.0" + + +def test_collects_host_metadata_under_host_namespace(tmp_path: Path) -> None: + result = _run_collector(tmp_path, {}, mode="host") + + assert result.returncode == 0 + metadata = toml.loads(result.stdout) + assert list(metadata) == ["host"] + assert metadata["host"]["user"] == 'metadata"user' + assert metadata["host"]["system"]["os_type"] == "ubuntu" + assert "slurm" not in metadata["host"] + + +def test_runtime_and_host_outputs_form_one_valid_metadata_document(tmp_path: Path) -> None: + runtime_dir = tmp_path / "runtime" + host_dir = tmp_path / "host" + runtime_dir.mkdir() + host_dir.mkdir() + + runtime_result = _run_collector(runtime_dir, {}, mode="runtime") + host_result = _run_collector(host_dir, {}, mode="host") + metadata = toml.loads(f"{runtime_result.stdout}\n{host_result.stdout}") + + parsed = SlurmSystemMetadata.model_validate(metadata) + assert parsed.host is not None + assert parsed.system.os_type == "ubuntu" + assert parsed.host.system.os_type == "ubuntu" diff --git a/tests/test_single_sbatch_runner.py b/tests/test_single_sbatch_runner.py index 11abdf3d8..4afe91b96 100644 --- a/tests/test_single_sbatch_runner.py +++ b/tests/test_single_sbatch_runner.py @@ -275,15 +275,15 @@ def test_container(self, nccl_tr: TestRun, slurm_system: SlurmSystem) -> None: f"srun --export=ALL --mpi=none -N{nccl_tr.num_nodes} --ntasks=2 --ntasks-per-node=1 " f"--output={runner.scenario_root}/metadata/node-%N.toml " f"--error={runner.scenario_root}/metadata/nodes.err " - f"bash {runner.system.install_path}/slurm-metadata.sh\n" + f"--container-image={tdef.docker_image.installed_path} {mounts} --no-container-mount-home " + "bash /cloudai_install/slurm-metadata.sh runtime\n" f"srun --export=ALL --mpi=none -N{nccl_tr.num_nodes} --ntasks=2 --ntasks-per-node=1 " f"--output={runner.scenario_root}/metadata/node-%N.toml " f"--error={runner.scenario_root}/metadata/nodes.err --open-mode=append " - f"--container-image={tdef.docker_image.installed_path} {mounts} --no-container-mount-home " - "bash /cloudai_install/slurm-metadata.sh runtime" + f"bash {runner.system.install_path}/slurm-metadata.sh host" ) assert aux_cmds[0] == metadata_cmd - assert "--container-image" not in aux_cmds[0].splitlines()[0] + assert "--container-image" not in aux_cmds[0].splitlines()[1] ranks_mapping_cmd = ( f"srun --export=ALL --mpi=pmix -N{nccl_tr.num_nodes} --container-image={tdef.docker_image.installed_path} " From 50432bc14a6bd8ea50de6d5dfc0ed4e09d2a53f6 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 15:09:02 +0200 Subject: [PATCH 4/8] remove duplicated sections; enhanced commands to collect network devices --- src/cloudai/systems/slurm/slurm-metadata.sh | 228 +++++++++++++++----- src/cloudai/systems/slurm/slurm_metadata.py | 12 -- tests/systems/slurm/test_metadata_script.py | 28 +-- 3 files changed, 194 insertions(+), 74 deletions(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index 345e61430..376edf11e 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -19,6 +19,16 @@ safe_probe() { return 0 } +bounded_command() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + else + "$@" + fi +} + toml_escape() { local value="${1:-$UNKNOWN}" value=${value//$'\r'/ } @@ -176,8 +186,29 @@ doca_host_version_probe() { local version local version_file + if command -v doca-info >/dev/null 2>&1; then + version="$(bounded_command 10 doca-info 2>/dev/null | awk ' + /^DOCA:/ {in_doca = 1; next} + in_doca && /^- / { + for (i = 1; i <= NF; i++) { + if ($i ~ /^[0-9]+\.[0-9]+/) { + print $i + exit + } + } + } + in_doca && /^[^ -]/ {exit} + ')" + if [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi + fi + if command -v dpkg-query >/dev/null 2>&1; then - for package in doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic doca-extra; do + for package in \ + doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ + doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" if [ -n "$version" ]; then printf '%s' "$version" @@ -187,7 +218,9 @@ doca_host_version_probe() { fi if command -v rpm >/dev/null 2>&1; then - for package in doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic doca-extra; do + for package in \ + doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ + doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' "$package" 2>/dev/null)" if [ -n "$version" ]; then printf '%s' "$version" @@ -225,8 +258,85 @@ ofed_version_probe() { } libfabric_version_probe() { - command -v fi_info >/dev/null 2>&1 || return 1 - fi_info --version | awk 'tolower($0) ~ /libfabric/ {print $2; exit}' + local version + + if command -v fi_info >/dev/null 2>&1; then + version="$(bounded_command 10 fi_info --version 2>/dev/null | awk ' + tolower($0) ~ /libfabric|fi_info/ { + for (i = 1; i <= NF; i++) { + if ($i ~ /^[0-9]+\.[0-9]+/) { + print $i + exit + } + } + } + ')" + [ -z "$version" ] || { printf '%s' "$version"; return 0; } + fi + + if command -v pkg-config >/dev/null 2>&1; then + version="$(pkg-config --modversion libfabric 2>/dev/null)" + [ -z "$version" ] || { printf '%s' "$version"; return 0; } + fi + + if command -v dpkg-query >/dev/null 2>&1; then + for package in libfabric1 libfabric-dev; do + version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" + [ -z "$version" ] || { printf '%s' "$version"; return 0; } + done + fi + + if command -v rpm >/dev/null 2>&1; then + version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' libfabric 2>/dev/null)" + [ -z "$version" ] || { printf '%s' "$version"; return 0; } + fi + + return 1 +} + +lldp_field_probe() { + local field="$1" + local output + + if command -v lldpctl >/dev/null 2>&1; then + output="$(lldpctl -f keyvalue 2>/dev/null)" + elif command -v lldpcli >/dev/null 2>&1; then + output="$(lldpcli show neighbors -f keyvalue 2>/dev/null)" + else + return 1 + fi + + printf '%s\n' "$output" | awk -F= -v field="$field" ' + index($1, field) {print $2} + ' | inventory_from_lines +} + +switch_type_probe() { + local value + + value="$(lldp_field_probe '.chassis.descr')" + [ -z "$value" ] || { printf '%s' "$value"; return 0; } + + if command -v ibswitches >/dev/null 2>&1; then + value="$(bounded_command 10 ibswitches 2>/dev/null | awk -F'"' 'NF >= 2 {print $2}' | inventory_from_lines)" + [ -z "$value" ] || { printf '%s' "$value"; return 0; } + fi + + return 1 +} + +network_name_probe() { + local value + + value="$(lldp_field_probe '.chassis.name')" + [ -z "$value" ] || { printf '%s' "$value"; return 0; } + + if command -v fi_info >/dev/null 2>&1; then + value="$(bounded_command 10 fi_info 2>/dev/null | awk -F': ' '$1 ~ /^[[:space:]]*fabric$/ {print $2}' | inventory_from_lines)" + [ -z "$value" ] || { printf '%s' "$value"; return 0; } + fi + + return 1 } os_release_file="${CLOUDAI_OS_RELEASE_FILE:-/etc/os-release}" @@ -256,57 +366,62 @@ nic_count="$(printf '%s\n' "$nic_lines" | nonempty_line_count 2>/dev/null)" || n nic_inventory="$(printf '%s\n' "$nic_models" | inventory_from_lines 2>/dev/null)" nic_inventory="${nic_inventory:-$UNKNOWN}" -mode="${1:-runtime}" -if [ "$mode" = "host" ]; then - printf '[host]\n' - emit_string user "$user" - table_prefix="host." -else +emit_system_metadata() { + emit_string os_type "${ID:-$UNKNOWN}" + emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" + emit_string linux_kernel_version "$kernel_version" + emit_string gpu_arch_type "$gpu_arch_type" + emit_integer gpu_count "$gpu_count" + emit_string gpu_inventory "$gpu_inventory" + emit_string cpu_model_name "$cpu_model" + emit_string cpu_arch_type "$cpu_arch" + emit_string cpu_vendor "$cpu_vendor" +} + +emit_physical_network_metadata() { + emit_string nics "$nics" + emit_integer nic_count "$nic_count" + emit_string nic_inventory "$nic_inventory" + emit_string hca_firmware_versions "$(safe_probe hca_firmware_probe)" + emit_string switch_type "${SWITCH:-$(safe_probe switch_type_probe)}" + emit_string network_name "${NETWORK:-$(safe_probe network_name_probe)}" + emit_string mofed_version "$(safe_probe ofed_version_probe)" + emit_string doca_host_version "$(safe_probe doca_host_version_probe)" +} + +mode="${1:-all}" +if [ "$mode" != "host" ]; then emit_string user "$user" - table_prefix="" fi -printf '\n[%ssystem]\n' "$table_prefix" -emit_string os_type "${ID:-$UNKNOWN}" -emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" -emit_string linux_kernel_version "$kernel_version" -emit_string gpu_arch_type "$gpu_arch_type" -emit_integer gpu_count "$gpu_count" -emit_string gpu_inventory "$gpu_inventory" -emit_string cpu_model_name "$cpu_model" -emit_string cpu_arch_type "$cpu_arch" -emit_string cpu_vendor "$cpu_vendor" - -printf '\n[%smpi]\n' "$table_prefix" -emit_string mpi_type "$(safe_probe mpi_type_probe)" -emit_string mpi_version "$(safe_probe mpi_version_probe)" -hpcx_version=${HPCX_DIR##*/} -emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" - -printf '\n[%scuda]\n' "$table_prefix" -emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" -emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" -driver_version="$(safe_probe gpu_driver_probe)" -emit_string cuda_driver_version "$driver_version" -emit_string nvidia_driver_version "$driver_version" -emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" - -printf '\n[%snetwork]\n' "$table_prefix" -emit_string nics "$nics" -emit_integer nic_count "$nic_count" -emit_string nic_inventory "$nic_inventory" -emit_string hca_firmware_versions "$(safe_probe hca_firmware_probe)" -emit_string switch_type "${SWITCH:-$UNKNOWN}" -emit_string network_name "${NETWORK:-$UNKNOWN}" -emit_string mofed_version "$(safe_probe ofed_version_probe)" -emit_string doca_host_version "$(safe_probe doca_host_version_probe)" -emit_string libfabric_version "$(safe_probe libfabric_version_probe)" - -printf '\n[%snccl]\n' "$table_prefix" -emit_string version "${NCCL_VERSION:-$UNKNOWN}" -emit_string commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" +if [ "$mode" = "all" ]; then + printf '\n[system]\n' + emit_system_metadata + + printf '\n[network]\n' + emit_physical_network_metadata + emit_string libfabric_version "$(safe_probe libfabric_version_probe)" +fi if [ "$mode" != "host" ]; then + printf '\n[mpi]\n' + emit_string mpi_type "$(safe_probe mpi_type_probe)" + emit_string mpi_version "$(safe_probe mpi_version_probe)" + hpcx_version=${HPCX_DIR##*/} + emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" + + printf '\n[cuda]\n' + emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" + emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" + driver_version="$(safe_probe gpu_driver_probe)" + emit_string cuda_driver_version "$driver_version" + emit_string nvidia_driver_version "$driver_version" + emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" + + printf '\n[nccl]\n' + emit_string version "${NCCL_VERSION:-$UNKNOWN}" + emit_string commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" + printf '\n[slurm]\n' emit_string cluster_name "${SLURM_CLUSTER_NAME:-$UNKNOWN}" emit_string node_list "${SLURM_NODELIST:-$UNKNOWN}" @@ -314,6 +429,19 @@ if [ "$mode" != "host" ]; then emit_string ntasks_per_node "${SLURM_NTASKS_PER_NODE:-$UNKNOWN}" emit_string ntasks "${SLURM_NTASKS:-$UNKNOWN}" emit_string job_id "${SLURM_JOBID:-$UNKNOWN}" + + if [ "$mode" = "runtime" ]; then + printf '\n[network]\n' + emit_string libfabric_version "$(safe_probe libfabric_version_probe)" + fi +else + # The runtime collector leaves [network] open so the host can append the + # physical fabric fields without declaring the TOML table a second time. + printf '\n' + emit_physical_network_metadata + + printf '\n[system]\n' + emit_system_metadata fi exit 0 diff --git a/src/cloudai/systems/slurm/slurm_metadata.py b/src/cloudai/systems/slurm/slurm_metadata.py index 29707ce93..74a50e2be 100644 --- a/src/cloudai/systems/slurm/slurm_metadata.py +++ b/src/cloudai/systems/slurm/slurm_metadata.py @@ -151,17 +151,6 @@ class MetadataSlurm(BaseModel): job_id: str -class MetadataHost(BaseModel): - """Represents metadata collected directly from the Slurm host.""" - - user: str - system: MetadataSystem - mpi: MetadataMPI - cuda: MetadataCUDA - network: MetadataNetwork - nccl: MetadataNCCL - - class SlurmSystemMetadata(BaseModel): """Represents the Slurm system metadata.""" @@ -172,4 +161,3 @@ class SlurmSystemMetadata(BaseModel): network: MetadataNetwork nccl: MetadataNCCL slurm: MetadataSlurm - host: MetadataHost | None = None diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py index f7a21b35c..17c89052f 100644 --- a/tests/systems/slurm/test_metadata_script.py +++ b/tests/systems/slurm/test_metadata_script.py @@ -20,7 +20,7 @@ def _write_command(bin_dir: Path, name: str, body: str) -> None: command.chmod(0o755) -def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "runtime") -> subprocess.CompletedProcess[str]: +def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "all") -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" bin_dir.mkdir() for name, body in commands.items(): @@ -72,6 +72,10 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: esac""", "ofed_info": "printf 'MLNX_OFED_LINUX-24.10-1.1.4.0:\\n'", "fi_info": "printf 'libfabric: 1.22.0\\n'", + "lldpctl": """cat <<'EOF' +lldp.eth0.chassis.name=eos-leaf-01 +lldp.eth0.chassis.descr=NVIDIA Spectrum-4 +EOF""", "mpirun": "printf 'mpirun (Open MPI) 4.1.7a1\\n'", }, ) @@ -104,8 +108,8 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: "nic_count": 2, "nic_inventory": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] x2", "hca_firmware_versions": "mlx5_0=28.43.2026", - "switch_type": "null", - "network_name": "null", + "switch_type": "NVIDIA Spectrum-4 x1", + "network_name": "eos-leaf-01 x1", "mofed_version": "MLNX_OFED_LINUX-24.10-1.1.4.0", "doca_host_version": "2.9.2-0.1.0", "libfabric_version": "1.22.0", @@ -144,21 +148,22 @@ def test_collects_container_metadata_in_legacy_sections(tmp_path: Path) -> None: assert result.stderr == "" metadata = toml.loads(result.stdout) assert "runtime" not in metadata - assert metadata["system"]["os_version"] == "24.04 LTS" + assert "system" not in metadata + assert metadata["network"] == {"libfabric_version": "null"} assert metadata["mpi"] == {"mpi_type": "openmpi", "mpi_version": "4.1.9a1", "hpcx_version": "null"} assert metadata["cuda"]["cuda_runtime_version"] == "13.1" assert metadata["cuda"]["cuda_toolkit_version"] == "13.0" -def test_collects_host_metadata_under_host_namespace(tmp_path: Path) -> None: +def test_collects_only_host_owned_sections_in_host_mode(tmp_path: Path) -> None: result = _run_collector(tmp_path, {}, mode="host") assert result.returncode == 0 - metadata = toml.loads(result.stdout) - assert list(metadata) == ["host"] - assert metadata["host"]["user"] == 'metadata"user' - assert metadata["host"]["system"]["os_type"] == "ubuntu" - assert "slurm" not in metadata["host"] + assert result.stdout.startswith("\nnics = ") + assert "\n[system]\n" in result.stdout + assert "\n[mpi]\n" not in result.stdout + assert "\n[nccl]\n" not in result.stdout + assert "libfabric_version" not in result.stdout def test_runtime_and_host_outputs_form_one_valid_metadata_document(tmp_path: Path) -> None: @@ -172,6 +177,5 @@ def test_runtime_and_host_outputs_form_one_valid_metadata_document(tmp_path: Pat metadata = toml.loads(f"{runtime_result.stdout}\n{host_result.stdout}") parsed = SlurmSystemMetadata.model_validate(metadata) - assert parsed.host is not None assert parsed.system.os_type == "ubuntu" - assert parsed.host.system.os_type == "ubuntu" + assert parsed.mpi.mpi_type == "null" From 4dd92a99bdfb1da64f1becfa876158f30f06ec90 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 18:46:58 +0200 Subject: [PATCH 5/8] simplify script --- src/cloudai/systems/slurm/slurm-metadata.sh | 57 ++++----------------- tests/systems/slurm/test_metadata_script.py | 17 +++--- 2 files changed, 18 insertions(+), 56 deletions(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index 376edf11e..c887c186a 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -182,12 +182,16 @@ hca_firmware_probe() { doca_host_version_probe() { local doca_root="${CLOUDAI_DOCA_ROOT:-/opt/mellanox/doca}" + local doca_info="$doca_root/tools/doca-info" local package local version local version_file if command -v doca-info >/dev/null 2>&1; then - version="$(bounded_command 10 doca-info 2>/dev/null | awk ' + doca_info="$(command -v doca-info)" + fi + if [ -x "$doca_info" ]; then + version="$(bounded_command 10 "$doca_info" 2>/dev/null | awk ' /^DOCA:/ {in_doca = 1; next} in_doca && /^- / { for (i = 1; i <= NF; i++) { @@ -294,51 +298,6 @@ libfabric_version_probe() { return 1 } -lldp_field_probe() { - local field="$1" - local output - - if command -v lldpctl >/dev/null 2>&1; then - output="$(lldpctl -f keyvalue 2>/dev/null)" - elif command -v lldpcli >/dev/null 2>&1; then - output="$(lldpcli show neighbors -f keyvalue 2>/dev/null)" - else - return 1 - fi - - printf '%s\n' "$output" | awk -F= -v field="$field" ' - index($1, field) {print $2} - ' | inventory_from_lines -} - -switch_type_probe() { - local value - - value="$(lldp_field_probe '.chassis.descr')" - [ -z "$value" ] || { printf '%s' "$value"; return 0; } - - if command -v ibswitches >/dev/null 2>&1; then - value="$(bounded_command 10 ibswitches 2>/dev/null | awk -F'"' 'NF >= 2 {print $2}' | inventory_from_lines)" - [ -z "$value" ] || { printf '%s' "$value"; return 0; } - fi - - return 1 -} - -network_name_probe() { - local value - - value="$(lldp_field_probe '.chassis.name')" - [ -z "$value" ] || { printf '%s' "$value"; return 0; } - - if command -v fi_info >/dev/null 2>&1; then - value="$(bounded_command 10 fi_info 2>/dev/null | awk -F': ' '$1 ~ /^[[:space:]]*fabric$/ {print $2}' | inventory_from_lines)" - [ -z "$value" ] || { printf '%s' "$value"; return 0; } - fi - - return 1 -} - os_release_file="${CLOUDAI_OS_RELEASE_FILE:-/etc/os-release}" if [ -r "$os_release_file" ]; then # shellcheck disable=SC1090 @@ -383,8 +342,10 @@ emit_physical_network_metadata() { emit_integer nic_count "$nic_count" emit_string nic_inventory "$nic_inventory" emit_string hca_firmware_versions "$(safe_probe hca_firmware_probe)" - emit_string switch_type "${SWITCH:-$(safe_probe switch_type_probe)}" - emit_string network_name "${NETWORK:-$(safe_probe network_name_probe)}" + # These are deployment topology labels, not facts a compute node can infer + # reliably. Preserve the original explicit configuration contract. + emit_string switch_type "${SWITCH:-$UNKNOWN}" + emit_string network_name "${NETWORK:-$UNKNOWN}" emit_string mofed_version "$(safe_probe ofed_version_probe)" emit_string doca_host_version "$(safe_probe doca_host_version_probe)" } diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py index 17c89052f..d4149c958 100644 --- a/tests/systems/slurm/test_metadata_script.py +++ b/tests/systems/slurm/test_metadata_script.py @@ -20,7 +20,9 @@ def _write_command(bin_dir: Path, name: str, body: str) -> None: command.chmod(0o755) -def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "all") -> subprocess.CompletedProcess[str]: +def _run_collector( + tmp_path: Path, commands: dict[str, str], mode: str = "all", extra_env: dict[str, str] | None = None +) -> subprocess.CompletedProcess[str]: bin_dir = tmp_path / "bin" bin_dir.mkdir() for name, body in commands.items(): @@ -31,7 +33,6 @@ def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "all") infiniband_sysfs = tmp_path / "infiniband" / "mlx5_0" infiniband_sysfs.mkdir(parents=True) (infiniband_sysfs / "fw_ver").write_text("28.43.2026\n") - env = { "PATH": f"{bin_dir}:{os.environ['PATH']}", "USER": 'metadata"user', @@ -41,6 +42,7 @@ def _run_collector(tmp_path: Path, commands: dict[str, str], mode: str = "all") "CLOUDAI_DOCA_ROOT": str(tmp_path / "missing-doca"), "SLURM_JOBID": "12345", } + env.update(extra_env or {}) return subprocess.run( ["/bin/bash", str(METADATA_SCRIPT), mode], env=env, text=True, capture_output=True, check=False ) @@ -72,12 +74,9 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: esac""", "ofed_info": "printf 'MLNX_OFED_LINUX-24.10-1.1.4.0:\\n'", "fi_info": "printf 'libfabric: 1.22.0\\n'", - "lldpctl": """cat <<'EOF' -lldp.eth0.chassis.name=eos-leaf-01 -lldp.eth0.chassis.descr=NVIDIA Spectrum-4 -EOF""", "mpirun": "printf 'mpirun (Open MPI) 4.1.7a1\\n'", }, + extra_env={"SWITCH": "Quantum-2", "NETWORK": "compute-fabric"}, ) assert result.returncode == 0 @@ -108,8 +107,8 @@ def test_collects_requested_host_metadata(tmp_path: Path) -> None: "nic_count": 2, "nic_inventory": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] x2", "hca_firmware_versions": "mlx5_0=28.43.2026", - "switch_type": "NVIDIA Spectrum-4 x1", - "network_name": "eos-leaf-01 x1", + "switch_type": "Quantum-2", + "network_name": "compute-fabric", "mofed_version": "MLNX_OFED_LINUX-24.10-1.1.4.0", "doca_host_version": "2.9.2-0.1.0", "libfabric_version": "1.22.0", @@ -130,6 +129,8 @@ def test_missing_subcommands_do_not_fail_or_corrupt_output(tmp_path: Path) -> No assert metadata["cuda"]["cuda_toolkit_version"] == "null" assert metadata["cuda"]["nvidia_driver_version"] == "null" assert metadata["network"]["nic_count"] == 0 + assert metadata["network"]["switch_type"] == "null" + assert metadata["network"]["network_name"] == "null" assert metadata["network"]["doca_host_version"] == "null" From 8899f3f55791d099047a166d87797c70755c3937 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 22:38:54 +0200 Subject: [PATCH 6/8] remove unnecessary tests --- src/cloudai/systems/slurm/slurm-metadata.sh | 18 +- .../slurm/test_command_gen_strategy.py | 23 --- tests/systems/slurm/test_metadata_script.py | 182 ------------------ 3 files changed, 10 insertions(+), 213 deletions(-) delete mode 100644 tests/systems/slurm/test_metadata_script.py diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index c887c186a..d1a10346b 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -213,8 +213,7 @@ doca_host_version_probe() { for package in \ doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do - version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" - if [ -n "$version" ]; then + if version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" && [ -n "$version" ]; then printf '%s' "$version" return 0 fi @@ -225,8 +224,7 @@ doca_host_version_probe() { for package in \ doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do - version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' "$package" 2>/dev/null)" - if [ -n "$version" ]; then + if version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' "$package" 2>/dev/null)" && [ -n "$version" ]; then printf '%s' "$version" return 0 fi @@ -285,14 +283,18 @@ libfabric_version_probe() { if command -v dpkg-query >/dev/null 2>&1; then for package in libfabric1 libfabric-dev; do - version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" - [ -z "$version" ] || { printf '%s' "$version"; return 0; } + if version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" && [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi done fi if command -v rpm >/dev/null 2>&1; then - version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' libfabric 2>/dev/null)" - [ -z "$version" ] || { printf '%s' "$version"; return 0; } + if version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' libfabric 2>/dev/null)" && [ -n "$version" ]; then + printf '%s' "$version" + return 0 + fi fi return 1 diff --git a/tests/systems/slurm/test_command_gen_strategy.py b/tests/systems/slurm/test_command_gen_strategy.py index 37ff835ad..98e103704 100644 --- a/tests/systems/slurm/test_command_gen_strategy.py +++ b/tests/systems/slurm/test_command_gen_strategy.py @@ -334,29 +334,6 @@ def test_gen_srun_prefix_tr_extra_srun_args(strategy_fixture: SlurmCommandGenStr assert "--arg val --flag" in srun_prefix # added as a single element -def test_metadata_cmd_collects_host_and_container_without_changing_extra_args( - strategy_fixture: SlurmCommandGenStrategy, -) -> None: - strategy_fixture.image_path = Mock(return_value="workload.sqsh") - strategy_fixture.system.extra_srun_args = "--reservation keep" - strategy_fixture.test_run.extra_srun_args = "--exclusive --constraint='gpu type'" - - metadata_cmd = strategy_fixture._metadata_cmd() - - runtime_cmd, host_cmd = metadata_cmd.splitlines() - assert "--container-image" not in host_cmd - assert "--container-mounts" not in host_cmd - assert "--open-mode=append" in host_cmd - assert f"bash {strategy_fixture.system.install_path}/slurm-metadata.sh host" in host_cmd - assert "--container-image=workload.sqsh" in runtime_cmd - assert "--container-mounts" in runtime_cmd - assert "bash /cloudai_install/slurm-metadata.sh runtime" in runtime_cmd - for command in (host_cmd, runtime_cmd): - assert "--mpi=none" in command - assert "--reservation keep" in command - assert "--exclusive --constraint='gpu type'" in command - - def test_append_distribution_and_hostfile_with_nodes(slurm_system: SlurmSystem, testrun_fixture: TestRun) -> None: slurm_system.distribution = "block" slurm_system.ntasks_per_node = 2 diff --git a/tests/systems/slurm/test_metadata_script.py b/tests/systems/slurm/test_metadata_script.py deleted file mode 100644 index d4149c958..000000000 --- a/tests/systems/slurm/test_metadata_script.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES -# Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import os -import subprocess -from pathlib import Path - -import toml - -from cloudai.systems import slurm -from cloudai.systems.slurm import SlurmSystemMetadata - -METADATA_SCRIPT = Path(slurm.__file__).parent / "slurm-metadata.sh" - - -def _write_command(bin_dir: Path, name: str, body: str) -> None: - command = bin_dir / name - command.write_text(f"#!/usr/bin/env bash\n{body}\n") - command.chmod(0o755) - - -def _run_collector( - tmp_path: Path, commands: dict[str, str], mode: str = "all", extra_env: dict[str, str] | None = None -) -> subprocess.CompletedProcess[str]: - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - for name, body in commands.items(): - _write_command(bin_dir, name, body) - - os_release = tmp_path / "os-release" - os_release.write_text('ID=ubuntu\nVERSION="24.04 LTS"\n') - infiniband_sysfs = tmp_path / "infiniband" / "mlx5_0" - infiniband_sysfs.mkdir(parents=True) - (infiniband_sysfs / "fw_ver").write_text("28.43.2026\n") - env = { - "PATH": f"{bin_dir}:{os.environ['PATH']}", - "USER": 'metadata"user', - "CLOUDAI_OS_RELEASE_FILE": str(os_release), - "CLOUDAI_INFINIBAND_SYSFS": str(infiniband_sysfs.parent), - "CLOUDAI_CUDA_ROOT": str(tmp_path / "missing-cuda"), - "CLOUDAI_DOCA_ROOT": str(tmp_path / "missing-doca"), - "SLURM_JOBID": "12345", - } - env.update(extra_env or {}) - return subprocess.run( - ["/bin/bash", str(METADATA_SCRIPT), mode], env=env, text=True, capture_output=True, check=False - ) - - -def test_collects_requested_host_metadata(tmp_path: Path) -> None: - result = _run_collector( - tmp_path, - { - "lscpu": """cat <<'EOF' -Architecture: x86_64 -Vendor ID: GenuineIntel -Model name: Intel(R) Xeon(R) Platinum 8573C -EOF""", - "nvidia-smi": """case "$1" in - --query-gpu=name) printf 'NVIDIA H100 80GB HBM3\\nNVIDIA H100 80GB HBM3\\n' ;; - --query-gpu=driver_version) printf '570.124.06\\n570.124.06\\n' ;; - *) printf '| NVIDIA-SMI 570.124.06 Driver Version: 570.124.06 CUDA Version: 12.8 |\\n' ;; -esac""", - "nvcc": "printf 'Cuda compilation tools, release 12.6, V12.6.85\\n'", - "lspci": """cat <<'EOF' -0000:17:00.0 Ethernet controller [0200]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] -0000:31:00.0 Infiniband controller [0207]: Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] -0000:65:00.0 VGA compatible controller [0300]: NVIDIA Corporation Device [10de:2330] -EOF""", - "dpkg-query": """case "$*" in - *doca-ofed) printf '2.9.2-0.1.0\\n' ;; - *) exit 1 ;; -esac""", - "ofed_info": "printf 'MLNX_OFED_LINUX-24.10-1.1.4.0:\\n'", - "fi_info": "printf 'libfabric: 1.22.0\\n'", - "mpirun": "printf 'mpirun (Open MPI) 4.1.7a1\\n'", - }, - extra_env={"SWITCH": "Quantum-2", "NETWORK": "compute-fabric"}, - ) - - assert result.returncode == 0 - assert result.stderr == "" - metadata = toml.loads(result.stdout) - assert metadata["user"] == 'metadata"user' - assert metadata["system"]["linux_kernel_version"] - assert metadata["system"] == { - "os_type": "ubuntu", - "os_version": "24.04 LTS", - "linux_kernel_version": metadata["system"]["linux_kernel_version"], - "gpu_arch_type": "NVIDIA H100 80GB HBM3", - "gpu_count": 2, - "gpu_inventory": "NVIDIA H100 80GB HBM3 x2", - "cpu_model_name": "Intel(R) Xeon(R) Platinum 8573C", - "cpu_arch_type": "x86_64", - "cpu_vendor": "GenuineIntel", - } - assert metadata["cuda"] == { - "cuda_build_version": "null", - "cuda_runtime_version": "12.8", - "cuda_driver_version": "570.124.06", - "nvidia_driver_version": "570.124.06", - "cuda_toolkit_version": "12.6", - } - assert metadata["network"] == { - "nics": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021]", - "nic_count": 2, - "nic_inventory": "Mellanox Technologies MT2910 Family [ConnectX-7] [15b3:1021] x2", - "hca_firmware_versions": "mlx5_0=28.43.2026", - "switch_type": "Quantum-2", - "network_name": "compute-fabric", - "mofed_version": "MLNX_OFED_LINUX-24.10-1.1.4.0", - "doca_host_version": "2.9.2-0.1.0", - "libfabric_version": "1.22.0", - } - - -def test_missing_subcommands_do_not_fail_or_corrupt_output(tmp_path: Path) -> None: - failing_commands = { - command: "exit 42" - for command in ("lscpu", "nvidia-smi", "nvcc", "lspci", "dpkg-query", "rpm", "mpirun", "ofed_info", "fi_info") - } - result = _run_collector(tmp_path, failing_commands) - - assert result.returncode == 0 - metadata = toml.loads(result.stdout) - assert metadata["system"]["gpu_count"] == 0 - assert metadata["system"]["gpu_arch_type"] == "null" - assert metadata["cuda"]["cuda_toolkit_version"] == "null" - assert metadata["cuda"]["nvidia_driver_version"] == "null" - assert metadata["network"]["nic_count"] == 0 - assert metadata["network"]["switch_type"] == "null" - assert metadata["network"]["network_name"] == "null" - assert metadata["network"]["doca_host_version"] == "null" - - -def test_collects_container_metadata_in_legacy_sections(tmp_path: Path) -> None: - result = _run_collector( - tmp_path, - { - "nvidia-smi": "printf 'NVIDIA-SMI CUDA Version: 13.1\\n'", - "nvcc": "printf 'Cuda compilation tools, release 13.0, V13.0.1\\n'", - "mpirun": "printf 'mpirun (Open MPI) 4.1.9a1\\n'", - }, - mode="runtime", - ) - - assert result.returncode == 0 - assert result.stderr == "" - metadata = toml.loads(result.stdout) - assert "runtime" not in metadata - assert "system" not in metadata - assert metadata["network"] == {"libfabric_version": "null"} - assert metadata["mpi"] == {"mpi_type": "openmpi", "mpi_version": "4.1.9a1", "hpcx_version": "null"} - assert metadata["cuda"]["cuda_runtime_version"] == "13.1" - assert metadata["cuda"]["cuda_toolkit_version"] == "13.0" - - -def test_collects_only_host_owned_sections_in_host_mode(tmp_path: Path) -> None: - result = _run_collector(tmp_path, {}, mode="host") - - assert result.returncode == 0 - assert result.stdout.startswith("\nnics = ") - assert "\n[system]\n" in result.stdout - assert "\n[mpi]\n" not in result.stdout - assert "\n[nccl]\n" not in result.stdout - assert "libfabric_version" not in result.stdout - - -def test_runtime_and_host_outputs_form_one_valid_metadata_document(tmp_path: Path) -> None: - runtime_dir = tmp_path / "runtime" - host_dir = tmp_path / "host" - runtime_dir.mkdir() - host_dir.mkdir() - - runtime_result = _run_collector(runtime_dir, {}, mode="runtime") - host_result = _run_collector(host_dir, {}, mode="host") - metadata = toml.loads(f"{runtime_result.stdout}\n{host_result.stdout}") - - parsed = SlurmSystemMetadata.model_validate(metadata) - assert parsed.system.os_type == "ubuntu" - assert parsed.mpi.mpi_type == "null" From 0670b77773f1adf574b769de955c4a5a48fcdd8d Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Fri, 21 Aug 2026 22:51:25 +0200 Subject: [PATCH 7/8] shorten slurm metadata script --- src/cloudai/systems/slurm/slurm-metadata.sh | 534 ++++++-------------- 1 file changed, 148 insertions(+), 386 deletions(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index d1a10346b..48cec31b3 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -1,410 +1,172 @@ #!/usr/bin/env bash -# Metadata is best effort: a missing utility or an unsupported query must not -# fail the Slurm step or prevent the remaining fields from being collected. -set +e -set +u -set +o pipefail 2>/dev/null || true export LC_ALL=C +. "${CLOUDAI_OS_RELEASE_FILE:-/etc/os-release}" 2>/dev/null || true -readonly UNKNOWN="null" - -safe_probe() { - local output - output="$("$@" 2>/dev/null)" || output="" - if [ -z "$output" ]; then - output="$UNKNOWN" - fi - printf '%s' "$output" - return 0 -} - -bounded_command() { - local seconds="$1" - shift - if command -v timeout >/dev/null 2>&1; then - timeout "$seconds" "$@" - else - "$@" - fi -} - -toml_escape() { - local value="${1:-$UNKNOWN}" - value=${value//$'\r'/ } - value=${value//$'\n'/, } - value=${value//\\/\\\\} - value=${value//\"/\\\"} - printf '%s' "$value" - return 0 -} - -emit_string() { - printf '%s = "%s"\n' "$1" "$(toml_escape "$2")" - return 0 -} - -emit_integer() { - local value="$2" - case "$value" in - '' | *[!0-9]*) value=0 ;; - esac - printf '%s = %s\n' "$1" "$value" - return 0 -} - -lscpu_field() { - local label="$1" - lscpu | awk -F: -v label="$label" '$1 == label {sub(/^[[:space:]]+/, "", $2); print $2; exit}' -} - -gpu_names_probe() { - nvidia-smi --query-gpu=name --format=csv,noheader +inventory() { + awk 'NF { count[$0]++; if (count[$0] == 1) order[++n] = $0 } + END { for (i = 1; i <= n; i++) printf "%s%s x%d", (i > 1 ? ", " : ""), order[i], count[order[i]] }' } -gpu_driver_probe() { - nvidia-smi --query-gpu=driver_version --format=csv,noheader | awk 'NF {print; exit}' +first_or_null() { + awk 'NF { print; found = 1; exit } END { if (!found) print "null" }' } -driver_cuda_compat_probe() { - nvidia-smi | awk ' - match($0, /CUDA Version: [0-9.]+/) { - value = substr($0, RSTART, RLENGTH) - sub(/^CUDA Version: /, "", value) - print value - exit - } - ' +ofed_version() { + command -v ofed_info >/dev/null || { echo null; return; } + ofed_info -s 2>/dev/null | sed 's/:$//' | first_or_null } -inventory_from_lines() { - awk ' - NF { - if (!seen[$0]++) { - order[++count] = $0 - } - } - END { - for (i = 1; i <= count; i++) { - if (i > 1) printf ", " - printf "%s x%d", order[i], seen[order[i]] - } - } - ' +libfabric_version() { + command -v fi_info >/dev/null || { echo null; return; } + fi_info --version 2>/dev/null | awk 'tolower($0) ~ /libfabric/ { print $2; exit }' | first_or_null } -nonempty_line_count() { - awk 'NF {count++} END {print count + 0}' -} - -cuda_toolkit_version_probe() { - local cuda_root="${CLOUDAI_CUDA_ROOT:-/usr/local/cuda}" - local version_file +cuda_toolkit_version() { local version - - if command -v nvcc >/dev/null 2>&1; then - version="$(nvcc --version 2>/dev/null | awk ' - match($0, /release [0-9.]+/) { - value = substr($0, RSTART, RLENGTH) - sub(/^release /, "", value) - print value - exit - } - ')" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - fi - - version_file="$cuda_root/version.json" - if [ -r "$version_file" ]; then - version="$(awk -F'"' '/"version"[[:space:]]*:/ {print $4; exit}' "$version_file" 2>/dev/null)" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - fi - - version_file="$cuda_root/version.txt" - if [ -r "$version_file" ]; then - version="$(awk 'match($0, /[0-9]+\.[0-9]+([.][0-9]+)?/) {print substr($0, RSTART, RLENGTH); exit}' \ - "$version_file" 2>/dev/null)" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - fi - - return 1 -} - -nic_lines_probe() { - lspci -Dnn | awk ' - { - line = tolower($0) - is_network = line ~ /ethernet controller|infiniband controller/ - is_nvidia = line ~ /\[15b3:/ || line ~ /mellanox/ || line ~ /nvidia/ - if (is_network && is_nvidia) print - } - ' -} - -nic_models_from_lines() { - awk ' - NF { - sub(/^[^[:space:]]+[[:space:]]+/, "") - sub(/^[^:]+:[[:space:]]*/, "") - print - } - ' -} - -hca_firmware_probe() { - local fw_file - local device_path - local device_name - local firmware - local separator="" - local sysfs_root="${CLOUDAI_INFINIBAND_SYSFS:-/sys/class/infiniband}" - - for fw_file in "$sysfs_root"/*/fw_ver; do - [ -r "$fw_file" ] || continue - device_path=${fw_file%/fw_ver} - device_name=${device_path##*/} - firmware="$(awk 'NF {print; exit}' "$fw_file" 2>/dev/null)" - [ -n "$firmware" ] || continue - printf '%s%s=%s' "$separator" "$device_name" "$firmware" - separator=", " - done - [ -n "$separator" ] + version="$(nvcc --version 2>/dev/null | sed -n 's/.*release \([0-9.]*\).*/\1/p' | head -n1)" + [ -n "$version" ] || version="$(sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "${CLOUDAI_CUDA_ROOT:-/usr/local/cuda}/version.json" 2>/dev/null | head -n1)" + [ -n "$version" ] || version="$(sed -n 's/[^0-9]*\([0-9][0-9.]*\).*/\1/p' \ + "${CLOUDAI_CUDA_ROOT:-/usr/local/cuda}/version.txt" 2>/dev/null | head -n1)" + printf '%s' "${version:-null}" } -doca_host_version_probe() { +doca_host_version() { local doca_root="${CLOUDAI_DOCA_ROOT:-/opt/mellanox/doca}" - local doca_info="$doca_root/tools/doca-info" - local package - local version - local version_file + local doca_info version - if command -v doca-info >/dev/null 2>&1; then - doca_info="$(command -v doca-info)" - fi + doca_info="$(command -v doca-info 2>/dev/null)" + doca_info="${doca_info:-$doca_root/tools/doca-info}" if [ -x "$doca_info" ]; then - version="$(bounded_command 10 "$doca_info" 2>/dev/null | awk ' - /^DOCA:/ {in_doca = 1; next} - in_doca && /^- / { - for (i = 1; i <= NF; i++) { - if ($i ~ /^[0-9]+\.[0-9]+/) { - print $i - exit - } - } - } - in_doca && /^[^ -]/ {exit} - ')" - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - fi - - if command -v dpkg-query >/dev/null 2>&1; then - for package in \ - doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ - doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do - if version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" && [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - done - fi - - if command -v rpm >/dev/null 2>&1; then - for package in \ - doca-host doca-all doca-networking doca-ofed doca-roce doca-host-basic \ - doca-runtime doca-sdk doca-tools doca-extra doca-cx-runtime doca-cx-sdk doca-cx-tools; do - if version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' "$package" 2>/dev/null)" && [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - done - fi - - for version_file in "$doca_root/version.json" "$doca_root/version.txt"; do - [ -r "$version_file" ] || continue - version="$(awk -F'"' '/"version"[[:space:]]*:/ {print $4; exit}' "$version_file" 2>/dev/null)" - if [ -z "$version" ]; then - version="$(awk 'NF {print; exit}' "$version_file" 2>/dev/null)" - fi - if [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - done - - return 1 -} - -mpi_type_probe() { - mpirun --version | awk 'tolower($0) ~ /open mpi/ {print "openmpi"; exit}' -} - -mpi_version_probe() { - mpirun --version | awk 'tolower($0) ~ /open mpi/ {print $NF; exit}' -} - -ofed_version_probe() { - command -v ofed_info >/dev/null 2>&1 || return 1 - ofed_info -s | sed 's/:$//' -} - -libfabric_version_probe() { - local version - - if command -v fi_info >/dev/null 2>&1; then - version="$(bounded_command 10 fi_info --version 2>/dev/null | awk ' - tolower($0) ~ /libfabric|fi_info/ { - for (i = 1; i <= NF; i++) { - if ($i ~ /^[0-9]+\.[0-9]+/) { - print $i - exit - } - } - } + version="$("$doca_info" 2>/dev/null | awk ' + /^DOCA:/ { found = 1; next } + found && /^- / { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9]+\.[0-9]+/) { print $i; exit } } ')" - [ -z "$version" ] || { printf '%s' "$version"; return 0; } fi - - if command -v pkg-config >/dev/null 2>&1; then - version="$(pkg-config --modversion libfabric 2>/dev/null)" - [ -z "$version" ] || { printf '%s' "$version"; return 0; } + if [ -z "$version" ] && command -v dpkg-query >/dev/null 2>&1; then + version="$(dpkg-query -W -f='${db:Status-Abbrev} ${Version}\n' 'doca-*' 2>/dev/null | + awk '$1 ~ /^ii/ { print $2; exit }')" fi - - if command -v dpkg-query >/dev/null 2>&1; then - for package in libfabric1 libfabric-dev; do - if version="$(dpkg-query -W -f='${Version}' "$package" 2>/dev/null)" && [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - done + if [ -z "$version" ] && command -v rpm >/dev/null 2>&1; then + version="$(rpm -qa --qf '%{VERSION}-%{RELEASE}\n' 'doca-*' 2>/dev/null | head -n1)" fi - - if command -v rpm >/dev/null 2>&1; then - if version="$(rpm -q --qf '%{VERSION}-%{RELEASE}' libfabric 2>/dev/null)" && [ -n "$version" ]; then - printf '%s' "$version" - return 0 - fi - fi - - return 1 -} - -os_release_file="${CLOUDAI_OS_RELEASE_FILE:-/etc/os-release}" -if [ -r "$os_release_file" ]; then - # shellcheck disable=SC1090 - . "$os_release_file" 2>/dev/null || true -fi - -user="${USER:-$(safe_probe whoami)}" -kernel_version="$(safe_probe uname -r)" -cpu_model="$(safe_probe lscpu_field "Model name")" -cpu_arch="$(safe_probe lscpu_field "Architecture")" -cpu_vendor="$(safe_probe lscpu_field "Vendor ID")" - -gpu_names="$(gpu_names_probe 2>/dev/null)" || gpu_names="" -gpu_arch_type="$(printf '%s\n' "$gpu_names" | awk 'NF {print; exit}' 2>/dev/null)" -gpu_arch_type="${gpu_arch_type:-$UNKNOWN}" -gpu_count="$(printf '%s\n' "$gpu_names" | nonempty_line_count 2>/dev/null)" || gpu_count=0 -gpu_inventory="$(printf '%s\n' "$gpu_names" | inventory_from_lines 2>/dev/null)" -gpu_inventory="${gpu_inventory:-$UNKNOWN}" - -nic_lines="$(nic_lines_probe 2>/dev/null)" || nic_lines="" -nic_models="$(printf '%s\n' "$nic_lines" | nic_models_from_lines 2>/dev/null)" -nics="$(printf '%s\n' "$nic_models" | awk 'NF {print; exit}' 2>/dev/null)" -nics="${nics:-$UNKNOWN}" -nic_count="$(printf '%s\n' "$nic_lines" | nonempty_line_count 2>/dev/null)" || nic_count=0 -nic_inventory="$(printf '%s\n' "$nic_models" | inventory_from_lines 2>/dev/null)" -nic_inventory="${nic_inventory:-$UNKNOWN}" - -emit_system_metadata() { - emit_string os_type "${ID:-$UNKNOWN}" - emit_string os_version "${VERSION:-${VERSION_ID:-$UNKNOWN}}" - emit_string linux_kernel_version "$kernel_version" - emit_string gpu_arch_type "$gpu_arch_type" - emit_integer gpu_count "$gpu_count" - emit_string gpu_inventory "$gpu_inventory" - emit_string cpu_model_name "$cpu_model" - emit_string cpu_arch_type "$cpu_arch" - emit_string cpu_vendor "$cpu_vendor" -} - -emit_physical_network_metadata() { - emit_string nics "$nics" - emit_integer nic_count "$nic_count" - emit_string nic_inventory "$nic_inventory" - emit_string hca_firmware_versions "$(safe_probe hca_firmware_probe)" - # These are deployment topology labels, not facts a compute node can infer - # reliably. Preserve the original explicit configuration contract. - emit_string switch_type "${SWITCH:-$UNKNOWN}" - emit_string network_name "${NETWORK:-$UNKNOWN}" - emit_string mofed_version "$(safe_probe ofed_version_probe)" - emit_string doca_host_version "$(safe_probe doca_host_version_probe)" -} - -mode="${1:-all}" -if [ "$mode" != "host" ]; then - emit_string user "$user" -fi - -if [ "$mode" = "all" ]; then - printf '\n[system]\n' - emit_system_metadata - - printf '\n[network]\n' - emit_physical_network_metadata - emit_string libfabric_version "$(safe_probe libfabric_version_probe)" -fi - -if [ "$mode" != "host" ]; then - printf '\n[mpi]\n' - emit_string mpi_type "$(safe_probe mpi_type_probe)" - emit_string mpi_version "$(safe_probe mpi_version_probe)" - hpcx_version=${HPCX_DIR##*/} - emit_string hpcx_version "${hpcx_version:-$UNKNOWN}" - - printf '\n[cuda]\n' - emit_string cuda_build_version "${CUDA_BUILD_VERSION:-$UNKNOWN}" - emit_string cuda_runtime_version "$(safe_probe driver_cuda_compat_probe)" - driver_version="$(safe_probe gpu_driver_probe)" - emit_string cuda_driver_version "$driver_version" - emit_string nvidia_driver_version "$driver_version" - emit_string cuda_toolkit_version "$(safe_probe cuda_toolkit_version_probe)" - - printf '\n[nccl]\n' - emit_string version "${NCCL_VERSION:-$UNKNOWN}" - emit_string commit_sha "${NCCL_COMMIT_SHA:-$UNKNOWN}" - - printf '\n[slurm]\n' - emit_string cluster_name "${SLURM_CLUSTER_NAME:-$UNKNOWN}" - emit_string node_list "${SLURM_NODELIST:-$UNKNOWN}" - emit_string num_nodes "${SLURM_NNODES:-$UNKNOWN}" - emit_string ntasks_per_node "${SLURM_NTASKS_PER_NODE:-$UNKNOWN}" - emit_string ntasks "${SLURM_NTASKS:-$UNKNOWN}" - emit_string job_id "${SLURM_JOBID:-$UNKNOWN}" - - if [ "$mode" = "runtime" ]; then - printf '\n[network]\n' - emit_string libfabric_version "$(safe_probe libfabric_version_probe)" + if [ -z "$version" ]; then + version="$(sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$doca_root/version.json" 2>/dev/null | head -n1)" fi -else - # The runtime collector leaves [network] open so the host can append the - # physical fabric fields without declaring the TOML table a second time. - printf '\n' - emit_physical_network_metadata - - printf '\n[system]\n' - emit_system_metadata -fi + [ -n "$version" ] || version="$(awk 'NF { print; exit }' "$doca_root/version.txt" 2>/dev/null)" + printf '%s' "${version:-null}" +} + +system_metadata() { + local gpu_names gpu_model gpu_count gpu_inventory + local cpu_model cpu_arch cpu_vendor + + gpu_names="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null)" + gpu_model="$(printf '%s\n' "$gpu_names" | awk 'NF { print; exit }')" + gpu_count="$(printf '%s\n' "$gpu_names" | awk 'NF { count++ } END { print count + 0 }')" + gpu_inventory="$(printf '%s\n' "$gpu_names" | inventory)" + cpu_model="$(lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -n1)" + cpu_arch="$(lscpu 2>/dev/null | sed -n 's/^Architecture:[[:space:]]*//p' | head -n1)" + cpu_vendor="$(lscpu 2>/dev/null | sed -n 's/^Vendor ID:[[:space:]]*//p' | head -n1)" + + cat < Date: Fri, 21 Aug 2026 23:03:25 +0200 Subject: [PATCH 8/8] more robust extraction of libfabric version --- src/cloudai/systems/slurm/slurm-metadata.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cloudai/systems/slurm/slurm-metadata.sh b/src/cloudai/systems/slurm/slurm-metadata.sh index 48cec31b3..911357b5e 100755 --- a/src/cloudai/systems/slurm/slurm-metadata.sh +++ b/src/cloudai/systems/slurm/slurm-metadata.sh @@ -19,7 +19,9 @@ ofed_version() { libfabric_version() { command -v fi_info >/dev/null || { echo null; return; } - fi_info --version 2>/dev/null | awk 'tolower($0) ~ /libfabric/ { print $2; exit }' | first_or_null + fi_info --version 2>/dev/null | + awk '{ for (i = 1; i <= NF; i++) if ($i ~ /^[0-9]+([.][0-9]+)+/) { print $i; exit } }' | + first_or_null } cuda_toolkit_version() {