From 83dae99c4f8ff107641087f4431580146623c3b1 Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 13:19:16 +0800 Subject: [PATCH 01/13] gpu bench --- README.md | 16 ++ gpu-bench/.env.example | 21 ++ gpu-bench/.gitignore | 4 + gpu-bench/README.md | 85 ++++++++ gpu-bench/init-node.sh | 16 ++ gpu-bench/record.sh | 426 +++++++++++++++++++++++++++++++++++++++++ gpu-bench/results.csv | 1 + gpu-bench/setup.sh | 291 ++++++++++++++++++++++++++++ 8 files changed, 860 insertions(+) create mode 100644 gpu-bench/.env.example create mode 100644 gpu-bench/.gitignore create mode 100644 gpu-bench/README.md create mode 100755 gpu-bench/init-node.sh create mode 100755 gpu-bench/record.sh create mode 100644 gpu-bench/results.csv create mode 100755 gpu-bench/setup.sh diff --git a/README.md b/README.md index ff54f48..419c8e4 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,22 @@ cargo build -p miner-cli --release - **Linux**: `nvidia-smi` (NVIDIA) or `radeontop` (AMD) - **Windows**: Task Manager GPU tab +### Cloud GPU bench + hardware spreadsheet + +Provider-agnostic tooling to run the miner on a rented NVIDIA GPU and append +hashrate / utilization / VRAM / efficiency rows to a CSV: + +```bash +cd gpu-bench +cp .env.example .env # set REWARDS_INNER_HASH for full stack +./setup.sh +./record.sh --provider vast.ai --cost-per-hour 0.35 +``` + +Or hardware-only (no node): `./record.sh --benchmark --provider runpod --cost-per-hour 0.42` + +See [`gpu-bench/README.md`](gpu-bench/README.md). + ## Examples ```bash diff --git a/gpu-bench/.env.example b/gpu-bench/.env.example new file mode 100644 index 0000000..dd41b8c --- /dev/null +++ b/gpu-bench/.env.example @@ -0,0 +1,21 @@ +# Wormhole inner hash (32-byte preimage, 0x-prefixed hex). +# Generate with: +# docker run --rm ghcr.io/quantus-network/quantus-node:latest \ +# key quantus --scheme wormhole +# Treat as sensitive ownership material — do not commit this file. +REWARDS_INNER_HASH=0xyour_inner_hash_here + +CHAIN=planck +NODE_NAME=gpu-bench-node + +# Optional overrides +# NODE_VERSION=latest +# GPU_DEVICES=1 +# METRICS_PORT=9900 +# HOST_MINER_LISTEN_PORT=9833 +# P2P_PORT=30333 +# RPC_PORT=9944 +# PROMETHEUS_PORT=9615 +# QUANTUS_MINER_DIR=/path/to/quantus-miner # default: parent of this directory +# MINER_BIN=/path/to/quantus-miner # skip cargo build if set +# MINER_LOG=info diff --git a/gpu-bench/.gitignore b/gpu-bench/.gitignore new file mode 100644 index 0000000..70eb89a --- /dev/null +++ b/gpu-bench/.gitignore @@ -0,0 +1,4 @@ +.run/ +*.log +.env +results.local.csv diff --git a/gpu-bench/README.md b/gpu-bench/README.md new file mode 100644 index 0000000..bc50b01 --- /dev/null +++ b/gpu-bench/README.md @@ -0,0 +1,85 @@ +# GPU miner bench (provider-agnostic) + +Spin up a Quantus node + native GPU miner on any rented NVIDIA GPU host (Vast.ai, +RunPod, Lambda, etc.), sample hashrate / utilization / GPU specs, and append a +row to [`results.csv`](results.csv) so we can answer “what hardware should we +recommend?” + +No cloud APIs. You supply `cloud_provider` and `cost_per_hour`. + +## Prerequisites + +On the GPU host: + +- NVIDIA driver + `nvidia-smi` +- Docker (for the node image) +- Rust toolchain (to build the miner), **or** a prebuilt binary via `MINER_BIN` +- This `quantus-miner` checkout (scripts live in `gpu-bench/`) + +## Quick start + +```bash +cd gpu-bench +cp .env.example .env +# Set REWARDS_INNER_HASH (from: docker run --rm ghcr.io/quantus-network/quantus-node:latest key quantus --scheme wormhole) + +./setup.sh +./record.sh --provider vast.ai --cost-per-hour 0.35 + +# When done +./setup.sh stop +``` + +### Hardware-only compare (no chain sync) + +If you only need hashrate vs cost for the spreadsheet, skip the full stack: + +```bash +./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 60 +``` + +This runs `quantus-miner benchmark` (builds the miner if needed) and samples +`nvidia-smi` utilization during the run. + +## Spreadsheet columns + +| Column | Source | +|--------|--------| +| `timestamp` | UTC ISO time | +| `cloud_provider` | `--provider` | +| `gpu_model` | `nvidia-smi` name | +| `vram_mb` | total VRAM | +| `sm_count` | CUDA SM / multiprocessor count | +| `driver_version` | NVIDIA driver | +| `hashrate` | avg GPU H/s (`miner_gpu_hash_rate` live, or benchmark average) | +| `gpu_utilization_pct` | avg `utilization.gpu` over the sample | +| `cost_per_hour` | `--cost-per-hour` (USD) | +| `efficiency` | `hashrate / cost_per_hour` | +| `sample_seconds` | duration used | +| `notes` | optional `--notes` | + +## Scripts + +| Script | Purpose | +|--------|---------| +| [`setup.sh`](setup.sh) | Start node (Docker) + native GPU miner; `./setup.sh stop` tears down | +| [`record.sh`](record.sh) | Sample metrics and append one CSV row (`--live` or `--benchmark`) | + +## Environment + +| Variable | Default | Meaning | +|----------|---------|---------| +| `REWARDS_INNER_HASH` | (required for setup) | Wormhole preimage for rewards | +| `QUANTUS_MINER_DIR` | repo root (parent of `gpu-bench`) | Miner source tree | +| `MINER_BIN` | built `target/release/quantus-miner` | Skip cargo build if set | +| `GPU_DEVICES` | `1` | GPUs for miner / benchmark | +| `METRICS_PORT` | `9900` | Miner Prometheus port | +| `NODE_VERSION` | `latest` | `ghcr.io/quantus-network/quantus-node` tag | + +## Notes + +- The public `quantus-miner` Docker image is CPU-only. This toolkit builds the + miner natively so WGPU can use host GPU drivers. +- Node and miner run on the **same host**; QUIC stays on `127.0.0.1:9833/udp` + (do not publish that port to the public internet on multi-host setups). +- Mining during live bench uses `--cpu-workers 0` so hashrate reflects the GPU. diff --git a/gpu-bench/init-node.sh b/gpu-bench/init-node.sh new file mode 100755 index 0000000..429a774 --- /dev/null +++ b/gpu-bench/init-node.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +NODE_KEY_PATH="/node-keys" +NODE_KEY_FILE="$NODE_KEY_PATH/key_node" + +# Generate node key if it doesn't exist +if [ ! -f "$NODE_KEY_FILE" ]; then + echo "Generating node key..." + mkdir -p "$NODE_KEY_PATH" + /usr/local/bin/quantus-node key generate-node-key --file "$NODE_KEY_FILE" + echo "Node key generated at: $NODE_KEY_FILE" +fi + +# Start node with original arguments +exec /usr/local/bin/quantus-node "$@" diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh new file mode 100755 index 0000000..9a1ee68 --- /dev/null +++ b/gpu-bench/record.sh @@ -0,0 +1,426 @@ +#!/usr/bin/env bash +# Sample GPU miner performance and append one row to results.csv. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="${SCRIPT_DIR}/.run" +ENV_FILE="${SCRIPT_DIR}/.env" +RESULTS_CSV="${SCRIPT_DIR}/results.csv" +CSV_HEADER="timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes" +DEFAULT_MINER_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +MODE="live" +PROVIDER="" +COST_PER_HOUR="" +DURATION=60 +NOTES="" +DRY_RUN=0 +GPU_DEVICES_FLAG="" + +usage() { + cat <<'EOF' +Usage: ./record.sh [options] + +Options: + --live Sample a running miner (default; requires setup.sh) + --benchmark Run quantus-miner benchmark (no node required) + --provider NAME Cloud provider label (e.g. vast.ai, runpod) + --cost-per-hour USD Hourly cost in USD (e.g. 0.35) + --duration SECONDS Sample / benchmark window (default: 60) + --gpu-devices N GPUs for --benchmark (default: GPU_DEVICES or 1) + --notes TEXT Optional notes column + --dry-run Print the CSV row but do not append + -h, --help Show this help + +Examples: + ./record.sh --provider vast.ai --cost-per-hour 0.35 + ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 60 +EOF +} + +load_env() { + if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + set -a + source "${ENV_FILE}" + set +a + fi +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +csv_escape() { + local s="${1:-}" + if [[ "${s}" == *","* || "${s}" == *"\""* || "${s}" == *$'\n'* ]]; then + s="${s//\"/\"\"}" + printf '"%s"' "${s}" + else + printf '%s' "${s}" + fi +} + +ensure_results_header() { + if [[ ! -f "${RESULTS_CSV}" ]]; then + printf '%s\n' "${CSV_HEADER}" >"${RESULTS_CSV}" + return + fi + if [[ ! -s "${RESULTS_CSV}" ]]; then + printf '%s\n' "${CSV_HEADER}" >"${RESULTS_CSV}" + fi +} + +query_gpu_static() { + # name, memory.total [MiB], multiprocessor_count, driver_version + local line + if ! line="$(nvidia-smi --query-gpu=name,memory.total,multiprocessor_count,driver_version \ + --format=csv,noheader,nounits 2>/dev/null | head -n 1)"; then + echo "error: failed to query GPU via nvidia-smi" >&2 + exit 1 + fi + if [[ -z "${line}" ]]; then + # Older drivers may lack multiprocessor_count + line="$(nvidia-smi --query-gpu=name,memory.total,driver_version \ + --format=csv,noheader,nounits 2>/dev/null | head -n 1)" + GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" + VRAM_MB="$(echo "${line}" | awk -F', ' '{print $2}' | tr -d ' ')" + SM_COUNT="" + DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" + return + fi + GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" + VRAM_MB="$(echo "${line}" | awk -F', ' '{print $2}' | tr -d ' ')" + SM_COUNT="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" + DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $4}' | tr -d ' ')" +} + +read_gpu_util() { + nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null \ + | head -n 1 | tr -d ' ' +} + +read_prometheus_hashrate() { + local port="$1" + local metrics + if ! metrics="$(curl -sf "http://127.0.0.1:${port}/metrics")"; then + echo "" + return + fi + local gpu_hr total_hr + gpu_hr="$(echo "${metrics}" | awk '/^miner_gpu_hash_rate[[:space:]]/{print $2; exit}')" + if [[ -n "${gpu_hr}" ]]; then + echo "${gpu_hr}" + return + fi + total_hr="$(echo "${metrics}" | awk '/^miner_hash_rate[[:space:]]/{print $2; exit}')" + echo "${total_hr}" +} + +average_list() { + # stdin: one number per line; prints average or empty + awk ' + NF && $1+0 == $1 { + sum += $1 + n++ + } + END { + if (n > 0) printf "%.6f", sum / n + } + ' +} + +resolve_miner_bin() { + if [[ -n "${MINER_BIN:-}" ]]; then + if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: MINER_BIN is not executable: ${MINER_BIN}" >&2 + exit 1 + fi + echo "${MINER_BIN}" + return + fi + + local miner_dir="${QUANTUS_MINER_DIR:-${DEFAULT_MINER_DIR}}" + local bin="${miner_dir}/target/release/quantus-miner" + if [[ -x "${bin}" ]]; then + echo "${bin}" + return + fi + + if [[ ! -d "${miner_dir}" ]]; then + echo "error: quantus-miner not found at ${miner_dir}" >&2 + exit 1 + fi + require_cmd cargo + echo "Building quantus-miner (release) in ${miner_dir} ..." >&2 + ( + cd "${miner_dir}" + cargo build -p miner-cli --release + ) >&2 + if [[ ! -x "${bin}" ]]; then + echo "error: expected binary missing: ${bin}" >&2 + exit 1 + fi + echo "${bin}" +} + +parse_benchmark_hashrate() { + # Prefer Total hashes / Total time for a numeric H/s. + local out="$1" + local total_hashes total_time + total_hashes="$(echo "${out}" | awk -F': ' '/^Total hashes:/{gsub(/[^0-9]/,"",$2); print $2; exit}')" + total_time="$(echo "${out}" | awk -F': ' '/^Total time:/{gsub(/s$/,"",$2); print $2; exit}')" + if [[ -n "${total_hashes}" && -n "${total_time}" ]]; then + awk -v h="${total_hashes}" -v t="${total_time}" 'BEGIN { + if (t+0 > 0) printf "%.6f", h / t + }' + return + fi + echo "" +} + +sample_util_during() { + # Background util sampler; writes samples to $1 for $2 seconds every ~2s + local out_file="$1" + local seconds="$2" + local end=$((SECONDS + seconds)) + : >"${out_file}" + while (( SECONDS < end )); do + local u + u="$(read_gpu_util || true)" + if [[ -n "${u}" ]]; then + echo "${u}" >>"${out_file}" + fi + sleep 2 + done +} + +prompt_if_empty() { + local var_name="$1" + local prompt="$2" + local current="${!var_name:-}" + if [[ -n "${current}" ]]; then + return + fi + if [[ ! -t 0 ]]; then + echo "error: ${var_name} required (pass flag; stdin is not a TTY)" >&2 + exit 1 + fi + local value + read -r -p "${prompt}: " value + printf -v "${var_name}" '%s' "${value}" +} + +emit_row() { + local timestamp="$1" + local hashrate="$2" + local util_avg="$3" + local efficiency="$4" + + local row + row="$(csv_escape "${timestamp}"),$(csv_escape "${PROVIDER}"),$(csv_escape "${GPU_MODEL}"),$(csv_escape "${VRAM_MB}"),$(csv_escape "${SM_COUNT}"),$(csv_escape "${DRIVER_VERSION}"),$(csv_escape "${hashrate}"),$(csv_escape "${util_avg}"),$(csv_escape "${COST_PER_HOUR}"),$(csv_escape "${efficiency}"),$(csv_escape "${DURATION}"),$(csv_escape "${NOTES}")" + + echo "${CSV_HEADER}" + echo "${row}" + + if [[ "${DRY_RUN}" -eq 1 ]]; then + echo "(dry-run: not written to ${RESULTS_CSV})" >&2 + return + fi + + ensure_results_header + echo "${row}" >>"${RESULTS_CSV}" + echo "Appended row to ${RESULTS_CSV}" >&2 +} + +run_live() { + require_cmd curl + local metrics_port="${METRICS_PORT:-9900}" + if [[ -f "${RUN_DIR}/metrics.port" ]]; then + metrics_port="$(cat "${RUN_DIR}/metrics.port")" + fi + + if ! curl -sf "http://127.0.0.1:${metrics_port}/metrics" >/dev/null; then + echo "error: miner metrics not reachable at http://127.0.0.1:${metrics_port}/metrics" >&2 + echo "Start the stack with ./setup.sh, or use --benchmark." >&2 + exit 1 + fi + + echo "Sampling live miner for ${DURATION}s (metrics :${metrics_port}) ..." >&2 + local hr_file util_file + hr_file="$(mktemp)" + util_file="$(mktemp)" + trap 'rm -f "${hr_file}" "${util_file}"' RETURN + + local end=$((SECONDS + DURATION)) + while (( SECONDS < end )); do + local hr util + hr="$(read_prometheus_hashrate "${metrics_port}")" + util="$(read_gpu_util || true)" + if [[ -n "${hr}" ]]; then + echo "${hr}" >>"${hr_file}" + fi + if [[ -n "${util}" ]]; then + echo "${util}" >>"${util_file}" + fi + sleep 2 + done + + local hashrate util_avg + hashrate="$(average_list <"${hr_file}")" + util_avg="$(average_list <"${util_file}")" + if [[ -z "${hashrate}" ]]; then + echo "error: no hashrate samples collected from :${metrics_port}" >&2 + exit 1 + fi + # Round util for spreadsheet readability + if [[ -n "${util_avg}" ]]; then + util_avg="$(awk -v u="${util_avg}" 'BEGIN { printf "%.2f", u }')" + fi + hashrate="$(awk -v h="${hashrate}" 'BEGIN { printf "%.6f", h }')" + + local efficiency + efficiency="$(awk -v h="${hashrate}" -v c="${COST_PER_HOUR}" 'BEGIN { + if (c+0 > 0) printf "%.6f", h / c + }')" + + local ts + ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${efficiency}" +} + +run_benchmark() { + local miner_bin + miner_bin="$(resolve_miner_bin)" + local gpu_devices="${GPU_DEVICES_FLAG:-${GPU_DEVICES:-1}}" + + local util_file bench_log + util_file="$(mktemp)" + bench_log="$(mktemp)" + trap 'rm -f "${util_file}" "${bench_log}"' RETURN + + echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}) ..." >&2 + sample_util_during "${util_file}" "${DURATION}" & + local sampler_pid=$! + + set +e + "${miner_bin}" benchmark \ + --cpu-workers 0 \ + --gpu-devices "${gpu_devices}" \ + --duration "${DURATION}" \ + >"${bench_log}" 2>&1 + local bench_rc=$? + set -e + + wait "${sampler_pid}" 2>/dev/null || true + + if [[ "${bench_rc}" -ne 0 ]]; then + echo "error: benchmark failed (exit ${bench_rc}). Output:" >&2 + cat "${bench_log}" >&2 + exit 1 + fi + + cat "${bench_log}" >&2 + + local hashrate util_avg + hashrate="$(parse_benchmark_hashrate "$(cat "${bench_log}")")" + if [[ -z "${hashrate}" ]]; then + echo "error: could not parse hashrate from benchmark output" >&2 + exit 1 + fi + util_avg="$(average_list <"${util_file}")" + if [[ -n "${util_avg}" ]]; then + util_avg="$(awk -v u="${util_avg}" 'BEGIN { printf "%.2f", u }')" + fi + + local efficiency + efficiency="$(awk -v h="${hashrate}" -v c="${COST_PER_HOUR}" 'BEGIN { + if (c+0 > 0) printf "%.6f", h / c + }')" + + local ts + ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${efficiency}" +} + +# --- args --- +while [[ $# -gt 0 ]]; do + case "$1" in + --live) + MODE="live" + shift + ;; + --benchmark) + MODE="benchmark" + shift + ;; + --provider) + PROVIDER="${2:-}" + shift 2 + ;; + --cost-per-hour) + COST_PER_HOUR="${2:-}" + shift 2 + ;; + --duration) + DURATION="${2:-}" + shift 2 + ;; + --gpu-devices) + GPU_DEVICES_FLAG="${2:-}" + shift 2 + ;; + --notes) + NOTES="${2:-}" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +load_env +require_cmd nvidia-smi +if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed — NVIDIA driver required" >&2 + exit 1 +fi + +if ! [[ "${DURATION}" =~ ^[0-9]+$ ]] || [[ "${DURATION}" -lt 1 ]]; then + echo "error: --duration must be a positive integer" >&2 + exit 1 +fi + +prompt_if_empty PROVIDER "Cloud provider (e.g. vast.ai)" +prompt_if_empty COST_PER_HOUR "Cost per hour USD (e.g. 0.35)" + +if ! awk -v c="${COST_PER_HOUR}" 'BEGIN { exit !(c+0 > 0) }'; then + echo "error: --cost-per-hour must be a positive number" >&2 + exit 1 +fi + +query_gpu_static +echo "GPU: ${GPU_MODEL} | VRAM: ${VRAM_MB} MiB | SMs: ${SM_COUNT:-n/a} | driver: ${DRIVER_VERSION}" >&2 + +case "${MODE}" in + live) run_live ;; + benchmark) run_benchmark ;; + *) + echo "error: unknown mode ${MODE}" >&2 + exit 1 + ;; +esac diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv new file mode 100644 index 0000000..0114764 --- /dev/null +++ b/gpu-bench/results.csv @@ -0,0 +1 @@ +timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes diff --git a/gpu-bench/setup.sh b/gpu-bench/setup.sh new file mode 100755 index 0000000..e9522ac --- /dev/null +++ b/gpu-bench/setup.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# Start (or stop) a Quantus node + native GPU miner on this host for benchmarking. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUN_DIR="${SCRIPT_DIR}/.run" +ENV_FILE="${SCRIPT_DIR}/.env" + +NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-quantus-gpu-bench-node}" +DEFAULT_MINER_DIR="${REPO_ROOT}" + +usage() { + cat <<'EOF' +Usage: ./setup.sh [start|stop|status] + + start Build/start native GPU miner + Docker node (default) + stop Stop miner and remove node container + status Show run state + +Environment (see .env.example): + REWARDS_INNER_HASH Required for start + QUANTUS_MINER_DIR Path to quantus-miner checkout (default: parent of gpu-bench) + MINER_BIN Prebuilt quantus-miner binary (skips cargo build) + GPU_DEVICES Number of GPUs (default: 1) +EOF +} + +load_env() { + if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + set -a + source "${ENV_FILE}" + set +a + fi +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +check_prereqs() { + require_cmd docker + require_cmd nvidia-smi + if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed — NVIDIA driver required for this toolkit" >&2 + exit 1 + fi +} + +resolve_miner_bin() { + if [[ -n "${MINER_BIN:-}" ]]; then + if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: MINER_BIN is not executable: ${MINER_BIN}" >&2 + exit 1 + fi + echo "${MINER_BIN}" + return + fi + + local miner_dir="${QUANTUS_MINER_DIR:-${DEFAULT_MINER_DIR}}" + if [[ ! -d "${miner_dir}" ]]; then + echo "error: quantus-miner not found at ${miner_dir}" >&2 + echo "Set QUANTUS_MINER_DIR or MINER_BIN." >&2 + exit 1 + fi + + require_cmd cargo + echo "Building quantus-miner (release) in ${miner_dir} ..." >&2 + ( + cd "${miner_dir}" + cargo build -p miner-cli --release + ) >&2 + local bin="${miner_dir}/target/release/quantus-miner" + if [[ ! -x "${bin}" ]]; then + echo "error: expected binary missing: ${bin}" >&2 + exit 1 + fi + echo "${bin}" +} + +start_node() { + local rewards="${REWARDS_INNER_HASH:-}" + if [[ -z "${rewards}" || "${rewards}" == "0xyour_inner_hash_here" ]]; then + echo "error: set REWARDS_INNER_HASH in ${ENV_FILE} or the environment" >&2 + echo "Generate with: docker run --rm ghcr.io/quantus-network/quantus-node:latest key quantus --scheme wormhole" >&2 + exit 1 + fi + + if docker ps -a --format '{{.Names}}' | grep -qx "${NODE_CONTAINER_NAME}"; then + echo "Removing existing container ${NODE_CONTAINER_NAME} ..." + docker rm -f "${NODE_CONTAINER_NAME}" >/dev/null + fi + + local node_version="${NODE_VERSION:-latest}" + local chain="${CHAIN:-planck}" + local node_name="${NODE_NAME:-gpu-bench-node}" + local p2p_port="${P2P_PORT:-30333}" + local rpc_port="${RPC_PORT:-9944}" + local prom_port="${PROMETHEUS_PORT:-9615}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + + mkdir -p "${RUN_DIR}/node-keys" "${RUN_DIR}/node-data" + + echo "Starting quantus-node (${NODE_CONTAINER_NAME}) ..." + docker run -d \ + --name "${NODE_CONTAINER_NAME}" \ + --restart unless-stopped \ + --platform linux/amd64 \ + -v "${SCRIPT_DIR}/init-node.sh:/init-node.sh:ro" \ + -v "${RUN_DIR}/node-keys:/node-keys" \ + -v "${RUN_DIR}/node-data:/var/lib/quantus" \ + -p "${p2p_port}:30333" \ + -p "${rpc_port}:9944" \ + -p "${prom_port}:9615" \ + -p "${miner_listen_port}:9833/udp" \ + --entrypoint /init-node.sh \ + "ghcr.io/quantus-network/quantus-node:${node_version}" \ + --validator \ + --base-path /var/lib/quantus \ + --chain "${chain}" \ + --node-key-file /node-keys/key_node \ + --rewards-inner-hash "${rewards}" \ + --name "${node_name}" \ + --wasm-execution compiled \ + --db-cache 2048 \ + --rpc-cors all \ + --prometheus-external \ + --miner-listen-port 9833 \ + >/dev/null + + echo "${NODE_CONTAINER_NAME}" >"${RUN_DIR}/node.container" + echo "Node listening for miners on 127.0.0.1:${miner_listen_port}/udp" +} + +wait_for_miner_port() { + local port="$1" + local attempts=30 + local i + for ((i = 1; i <= attempts; i++)); do + if curl -sf "http://127.0.0.1:${port}/metrics" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "warning: miner metrics not ready on :${port} after ${attempts}s (continuing)" >&2 +} + +start_miner() { + local miner_bin + miner_bin="$(resolve_miner_bin)" + + local gpu_devices="${GPU_DEVICES:-1}" + local metrics_port="${METRICS_PORT:-9900}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + local miner_log="${MINER_LOG:-info}" + + if [[ -f "${RUN_DIR}/miner.pid" ]]; then + local old_pid + old_pid="$(cat "${RUN_DIR}/miner.pid")" + if kill -0 "${old_pid}" 2>/dev/null; then + echo "Stopping previous miner (pid ${old_pid}) ..." + kill "${old_pid}" 2>/dev/null || true + local i + for ((i = 0; i < 10; i++)); do + kill -0 "${old_pid}" 2>/dev/null || break + sleep 0.5 + done + if kill -0 "${old_pid}" 2>/dev/null; then + kill -9 "${old_pid}" 2>/dev/null || true + fi + fi + fi + + mkdir -p "${RUN_DIR}" + local log_file="${RUN_DIR}/miner.log" + echo "Starting GPU miner: ${miner_bin}" + echo " --node-addr 127.0.0.1:${miner_listen_port} --gpu-devices ${gpu_devices} --cpu-workers 0" + + RUST_LOG="${miner_log}" nohup "${miner_bin}" serve \ + --node-addr "127.0.0.1:${miner_listen_port}" \ + --gpu-devices "${gpu_devices}" \ + --cpu-workers 0 \ + --metrics-port "${metrics_port}" \ + >"${log_file}" 2>&1 & + + local pid=$! + echo "${pid}" >"${RUN_DIR}/miner.pid" + echo "${miner_bin}" >"${RUN_DIR}/miner.bin" + echo "${metrics_port}" >"${RUN_DIR}/metrics.port" + + wait_for_miner_port "${metrics_port}" + echo "Miner pid ${pid}; metrics http://127.0.0.1:${metrics_port}/metrics" + echo "Logs: ${log_file}" +} + +do_start() { + load_env + check_prereqs + mkdir -p "${RUN_DIR}" + start_node + # Brief pause so UDP listener is up before miner connects + sleep 2 + start_miner + echo + echo "Stack is up. Record a spreadsheet row with:" + echo " ./record.sh --provider --cost-per-hour " +} + +do_stop() { + load_env + if [[ -f "${RUN_DIR}/miner.pid" ]]; then + local pid + pid="$(cat "${RUN_DIR}/miner.pid")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Stopping miner (pid ${pid}) ..." + kill "${pid}" 2>/dev/null || true + # Give it a moment; force if needed + sleep 1 + if kill -0 "${pid}" 2>/dev/null; then + kill -9 "${pid}" 2>/dev/null || true + fi + fi + rm -f "${RUN_DIR}/miner.pid" + fi + + local container="${NODE_CONTAINER_NAME}" + if [[ -f "${RUN_DIR}/node.container" ]]; then + container="$(cat "${RUN_DIR}/node.container")" + fi + if docker ps -a --format '{{.Names}}' | grep -qx "${container}"; then + echo "Removing node container ${container} ..." + docker rm -f "${container}" >/dev/null + fi + rm -f "${RUN_DIR}/node.container" + echo "Stopped." +} + +do_status() { + load_env + echo "Run dir: ${RUN_DIR}" + if [[ -f "${RUN_DIR}/node.container" ]]; then + local c + c="$(cat "${RUN_DIR}/node.container")" + if docker ps --format '{{.Names}}' | grep -qx "${c}"; then + echo "Node: running (${c})" + else + echo "Node: not running (recorded ${c})" + fi + else + echo "Node: not started via setup.sh" + fi + + if [[ -f "${RUN_DIR}/miner.pid" ]]; then + local pid + pid="$(cat "${RUN_DIR}/miner.pid")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Miner: running (pid ${pid})" + else + echo "Miner: not running (stale pid ${pid})" + fi + else + echo "Miner: not started via setup.sh" + fi + + local metrics_port="${METRICS_PORT:-9900}" + if [[ -f "${RUN_DIR}/metrics.port" ]]; then + metrics_port="$(cat "${RUN_DIR}/metrics.port")" + fi + if curl -sf "http://127.0.0.1:${metrics_port}/metrics" >/dev/null 2>&1; then + echo "Metrics: ok on :${metrics_port}" + else + echo "Metrics: not reachable on :${metrics_port}" + fi +} + +cmd="${1:-start}" +case "${cmd}" in + start) do_start ;; + stop) do_stop ;; + status) do_status ;; + -h | --help | help) usage ;; + *) + echo "error: unknown command: ${cmd}" >&2 + usage >&2 + exit 1 + ;; +esac From 008e7f8662c879a598dbaf9f33375dd7fc61e885 Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 13:40:23 +0800 Subject: [PATCH 02/13] setup script --- README.md | 5 +- gpu-bench/.env.example | 11 +- gpu-bench/README.md | 53 +++--- gpu-bench/setup.sh | 360 ++++++++++++++++++++++++++++++++++------- 4 files changed, 341 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 419c8e4..76d0feb 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,9 @@ hashrate / utilization / VRAM / efficiency rows to a CSV: ```bash cd gpu-bench -cp .env.example .env # set REWARDS_INNER_HASH for full stack -./setup.sh +cp .env.example .env +./setup.sh wormhole # save inner_hash into .env +./setup.sh # native node + GPU miner (same host) ./record.sh --provider vast.ai --cost-per-hour 0.35 ``` diff --git a/gpu-bench/.env.example b/gpu-bench/.env.example index dd41b8c..1e344a8 100644 --- a/gpu-bench/.env.example +++ b/gpu-bench/.env.example @@ -1,7 +1,8 @@ # Wormhole inner hash (32-byte preimage, 0x-prefixed hex). # Generate with: -# docker run --rm ghcr.io/quantus-network/quantus-node:latest \ -# key quantus --scheme wormhole +# ./setup.sh # downloads node if needed, then Ctrl+C before mining — or: +# quantus-node key quantus --scheme wormhole +# # or: ./.run/bin/quantus-node key quantus --scheme wormhole # Treat as sensitive ownership material — do not commit this file. REWARDS_INNER_HASH=0xyour_inner_hash_here @@ -9,13 +10,15 @@ CHAIN=planck NODE_NAME=gpu-bench-node # Optional overrides -# NODE_VERSION=latest # GPU_DEVICES=1 # METRICS_PORT=9900 # HOST_MINER_LISTEN_PORT=9833 # P2P_PORT=30333 # RPC_PORT=9944 # PROMETHEUS_PORT=9615 +# QUANTUS_NODE_BIN=/path/to/quantus-node # skip GitHub download if set +# QUANTUS_CHAIN_DIR=/path/to/chain # use local release build if present # QUANTUS_MINER_DIR=/path/to/quantus-miner # default: parent of this directory -# MINER_BIN=/path/to/quantus-miner # skip cargo build if set +# MINER_BIN=/path/to/quantus-miner # skip cargo build if set # MINER_LOG=info +# NODE_VERSION=latest # only used with: ./setup.sh start --docker diff --git a/gpu-bench/README.md b/gpu-bench/README.md index bc50b01..2b2a13e 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -1,46 +1,53 @@ # GPU miner bench (provider-agnostic) Spin up a Quantus node + native GPU miner on any rented NVIDIA GPU host (Vast.ai, -RunPod, Lambda, etc.), sample hashrate / utilization / GPU specs, and append a -row to [`results.csv`](results.csv) so we can answer “what hardware should we -recommend?” +RunPod, etc.), sample hashrate / utilization / GPU specs, and append a row to +[`results.csv`](results.csv) so we can answer “what hardware should we recommend?” No cloud APIs. You supply `cloud_provider` and `cost_per_hour`. +**Default path is native binaries on the same machine** (no Docker / miner-stack). +That works on RunPod, Vast, and similar GPU rentals. + ## Prerequisites On the GPU host: - NVIDIA driver + `nvidia-smi` -- Docker (for the node image) +- `curl` + `tar` (to fetch `quantus-node` from GitHub Releases) - Rust toolchain (to build the miner), **or** a prebuilt binary via `MINER_BIN` -- This `quantus-miner` checkout (scripts live in `gpu-bench/`) +- This `quantus-miner` checkout -## Quick start +## Quick start (same-host mining) ```bash cd gpu-bench cp .env.example .env -# Set REWARDS_INNER_HASH (from: docker run --rm ghcr.io/quantus-network/quantus-node:latest key quantus --scheme wormhole) -./setup.sh -./record.sh --provider vast.ai --cost-per-hour 0.35 +./setup.sh wormhole # downloads node if needed; save inner_hash +# paste inner_hash into .env as REWARDS_INNER_HASH + +./setup.sh # native node + GPU miner on this host +./record.sh --provider runpod --cost-per-hour 0.35 -# When done ./setup.sh stop ``` -### Hardware-only compare (no chain sync) +`setup.sh` will: + +1. Resolve `quantus-node` (`QUANTUS_NODE_BIN` → `PATH` → sibling `chain` build → GitHub release download into `.run/bin/`) +2. Generate a node key under `.run/node-keys/` +3. Start the node with `--miner-listen-port 9833` +4. Build/start the GPU miner against `127.0.0.1:9833` (`--cpu-workers 0`) -If you only need hashrate vs cost for the spreadsheet, skip the full stack: +Optional Docker node (not recommended on RunPod): `./setup.sh start --docker` + +### Hardware-only compare (no chain sync) ```bash ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 60 ``` -This runs `quantus-miner benchmark` (builds the miner if needed) and samples -`nvidia-smi` utilization during the run. - ## Spreadsheet columns | Column | Source | @@ -62,7 +69,7 @@ This runs `quantus-miner benchmark` (builds the miner if needed) and samples | Script | Purpose | |--------|---------| -| [`setup.sh`](setup.sh) | Start node (Docker) + native GPU miner; `./setup.sh stop` tears down | +| [`setup.sh`](setup.sh) | Native node + GPU miner on this host; `stop` / `status` | | [`record.sh`](record.sh) | Sample metrics and append one CSV row (`--live` or `--benchmark`) | ## Environment @@ -70,16 +77,16 @@ This runs `quantus-miner benchmark` (builds the miner if needed) and samples | Variable | Default | Meaning | |----------|---------|---------| | `REWARDS_INNER_HASH` | (required for setup) | Wormhole preimage for rewards | -| `QUANTUS_MINER_DIR` | repo root (parent of `gpu-bench`) | Miner source tree | -| `MINER_BIN` | built `target/release/quantus-miner` | Skip cargo build if set | +| `QUANTUS_NODE_BIN` | download to `.run/bin/quantus-node` | Skip release download | +| `QUANTUS_MINER_DIR` | repo root | Miner source tree | +| `MINER_BIN` | built `target/release/quantus-miner` | Skip cargo build | | `GPU_DEVICES` | `1` | GPUs for miner / benchmark | | `METRICS_PORT` | `9900` | Miner Prometheus port | -| `NODE_VERSION` | `latest` | `ghcr.io/quantus-network/quantus-node` tag | ## Notes -- The public `quantus-miner` Docker image is CPU-only. This toolkit builds the - miner natively so WGPU can use host GPU drivers. -- Node and miner run on the **same host**; QUIC stays on `127.0.0.1:9833/udp` - (do not publish that port to the public internet on multi-host setups). +- Prefer this over `chain/miner-stack` on cloud GPUs: Compose + the public miner + image are CPU-oriented and awkward on nested-Docker hosts. +- Node and miner share the host; QUIC is `127.0.0.1:9833/udp`. +- Give the volume enough disk for chain sync (tens of GB). - Mining during live bench uses `--cpu-workers 0` so hashrate reflects the GPU. diff --git a/gpu-bench/setup.sh b/gpu-bench/setup.sh index e9522ac..1244a2b 100755 --- a/gpu-bench/setup.sh +++ b/gpu-bench/setup.sh @@ -1,27 +1,37 @@ #!/usr/bin/env bash -# Start (or stop) a Quantus node + native GPU miner on this host for benchmarking. +# Start (or stop) a native Quantus node + GPU miner on this host. +# Same-machine layout for cloud GPU rentals (RunPod, Vast, etc.). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" RUN_DIR="${SCRIPT_DIR}/.run" ENV_FILE="${SCRIPT_DIR}/.env" +BIN_DIR="${RUN_DIR}/bin" -NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-quantus-gpu-bench-node}" DEFAULT_MINER_DIR="${REPO_ROOT}" +DEFAULT_CHAIN_DIR="$(cd "${REPO_ROOT}/.." && pwd)/chain" +NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-quantus-gpu-bench-node}" + +NODE_MODE="native" # native | docker usage() { cat <<'EOF' -Usage: ./setup.sh [start|stop|status] +Usage: ./setup.sh [start|stop|status|fetch-node|wormhole] [--docker] - start Build/start native GPU miner + Docker node (default) - stop Stop miner and remove node container - status Show run state + start Download/build binaries, start native node + GPU miner (default) + stop Stop miner and node + status Show run state + fetch-node Download quantus-node into .run/bin/ (no start) + wormhole Print a new wormhole address + inner_hash (for REWARDS_INNER_HASH) + --docker Use Docker for the node instead of a native binary + (not recommended on RunPod / nested-Docker hosts) Environment (see .env.example): REWARDS_INNER_HASH Required for start + QUANTUS_NODE_BIN Prebuilt quantus-node (skips download) QUANTUS_MINER_DIR Path to quantus-miner checkout (default: parent of gpu-bench) - MINER_BIN Prebuilt quantus-miner binary (skips cargo build) + MINER_BIN Prebuilt quantus-miner (skips cargo build) GPU_DEVICES Number of GPUs (default: 1) EOF } @@ -43,12 +53,40 @@ require_cmd() { } check_prereqs() { - require_cmd docker require_cmd nvidia-smi if ! nvidia-smi >/dev/null 2>&1; then echo "error: nvidia-smi failed — NVIDIA driver required for this toolkit" >&2 exit 1 fi + if [[ "${NODE_MODE}" == "docker" ]]; then + require_cmd docker + else + require_cmd curl + require_cmd tar + fi +} + +stop_pidfile() { + local pidfile="$1" + local label="$2" + if [[ ! -f "${pidfile}" ]]; then + return 0 + fi + local pid + pid="$(cat "${pidfile}")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Stopping ${label} (pid ${pid}) ..." + kill "${pid}" 2>/dev/null || true + local i + for ((i = 0; i < 20; i++)); do + kill -0 "${pid}" 2>/dev/null || break + sleep 0.5 + done + if kill -0 "${pid}" 2>/dev/null; then + kill -9 "${pid}" 2>/dev/null || true + fi + fi + rm -f "${pidfile}" } resolve_miner_bin() { @@ -68,13 +106,18 @@ resolve_miner_bin() { exit 1 fi + local bin="${miner_dir}/target/release/quantus-miner" + if [[ -x "${bin}" ]]; then + echo "${bin}" + return + fi + require_cmd cargo echo "Building quantus-miner (release) in ${miner_dir} ..." >&2 ( cd "${miner_dir}" cargo build -p miner-cli --release ) >&2 - local bin="${miner_dir}/target/release/quantus-miner" if [[ ! -x "${bin}" ]]; then echo "error: expected binary missing: ${bin}" >&2 exit 1 @@ -82,13 +125,176 @@ resolve_miner_bin() { echo "${bin}" } -start_node() { +detect_node_target() { + local os arch + case "$(uname -s)" in + Linux) os="linux" ;; + Darwin) os="macos" ;; + *) + echo "error: unsupported OS for node download: $(uname -s)" >&2 + exit 1 + ;; + esac + case "$(uname -m)" in + x86_64 | amd64) + if [[ "${os}" == "linux" ]]; then + arch="x86_64-unknown-linux-gnu" + else + arch="x86_64-apple-darwin" + fi + ;; + arm64 | aarch64) + if [[ "${os}" == "linux" ]]; then + arch="aarch64-unknown-linux-gnu" + else + arch="aarch64-apple-darwin" + fi + ;; + *) + echo "error: unsupported arch for node download: $(uname -m)" >&2 + exit 1 + ;; + esac + echo "${arch}" +} + +download_node_binary() { + local dest="$1" + local target + target="$(detect_node_target)" + echo "Fetching latest quantus-node release (${target}) ..." >&2 + + local release_json tag asset_url tmp + release_json="$(curl -fsSL https://api.github.com/repos/Quantus-Network/chain/releases/latest)" + tag="$(echo "${release_json}" | grep -o '"tag_name": "[^"]*"' | head -n 1 | cut -d'"' -f4)" + if [[ -z "${tag}" ]]; then + echo "error: could not determine latest chain release tag" >&2 + exit 1 + fi + + asset_url="https://github.com/Quantus-Network/chain/releases/download/${tag}/quantus-node-${tag}-${target}.tar.gz" + echo "Downloading ${asset_url} ..." >&2 + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" RETURN + if ! curl -fL "${asset_url}" -o "${tmp}/node.tar.gz"; then + echo "error: failed to download quantus-node for ${target}" >&2 + echo "Set QUANTUS_NODE_BIN to a local binary, or build from the chain repo." >&2 + exit 1 + fi + tar -xzf "${tmp}/node.tar.gz" -C "${tmp}" + if [[ ! -f "${tmp}/quantus-node" ]]; then + echo "error: archive did not contain quantus-node" >&2 + exit 1 + fi + mkdir -p "$(dirname "${dest}")" + mv "${tmp}/quantus-node" "${dest}" + chmod +x "${dest}" + echo "Installed quantus-node -> ${dest}" >&2 +} + +resolve_node_bin() { + if [[ -n "${QUANTUS_NODE_BIN:-}" ]]; then + if [[ ! -x "${QUANTUS_NODE_BIN}" ]]; then + echo "error: QUANTUS_NODE_BIN is not executable: ${QUANTUS_NODE_BIN}" >&2 + exit 1 + fi + echo "${QUANTUS_NODE_BIN}" + return + fi + + if command -v quantus-node >/dev/null 2>&1; then + command -v quantus-node + return + fi + + local chain_dir="${QUANTUS_CHAIN_DIR:-${DEFAULT_CHAIN_DIR}}" + if [[ -x "${chain_dir}/target/release/quantus-node" ]]; then + echo "${chain_dir}/target/release/quantus-node" + return + fi + + local cached="${BIN_DIR}/quantus-node" + if [[ -x "${cached}" ]]; then + echo "${cached}" + return + fi + + download_node_binary "${cached}" + echo "${cached}" +} + +ensure_rewards() { local rewards="${REWARDS_INNER_HASH:-}" if [[ -z "${rewards}" || "${rewards}" == "0xyour_inner_hash_here" ]]; then echo "error: set REWARDS_INNER_HASH in ${ENV_FILE} or the environment" >&2 - echo "Generate with: docker run --rm ghcr.io/quantus-network/quantus-node:latest key quantus --scheme wormhole" >&2 + echo "Generate with (after node binary is available):" >&2 + echo " quantus-node key quantus --scheme wormhole" >&2 exit 1 fi +} + +ensure_node_key() { + local node_bin="$1" + local key_file="${RUN_DIR}/node-keys/key_node" + mkdir -p "$(dirname "${key_file}")" + if [[ ! -f "${key_file}" ]]; then + echo "Generating node key at ${key_file} ..." + "${node_bin}" key generate-node-key --file "${key_file}" + fi + echo "${key_file}" +} + +start_node_native() { + ensure_rewards + local node_bin + node_bin="$(resolve_node_bin)" + local key_file + key_file="$(ensure_node_key "${node_bin}")" + + local chain="${CHAIN:-planck}" + local node_name="${NODE_NAME:-gpu-bench-node}" + local p2p_port="${P2P_PORT:-30333}" + local rpc_port="${RPC_PORT:-9944}" + local prom_port="${PROMETHEUS_PORT:-9615}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + local base_path="${RUN_DIR}/node-data" + mkdir -p "${base_path}" + + stop_pidfile "${RUN_DIR}/node.pid" "node" + + local log_file="${RUN_DIR}/node.log" + echo "Starting native quantus-node: ${node_bin}" + echo " --miner-listen-port ${miner_listen_port} --chain ${chain}" + + nohup "${node_bin}" \ + --validator \ + --base-path "${base_path}" \ + --chain "${chain}" \ + --node-key-file "${key_file}" \ + --rewards-inner-hash "${REWARDS_INNER_HASH}" \ + --name "${node_name}" \ + --port "${p2p_port}" \ + --rpc-port "${rpc_port}" \ + --prometheus-port "${prom_port}" \ + --prometheus-external \ + --miner-listen-port "${miner_listen_port}" \ + --wasm-execution compiled \ + --db-cache 2048 \ + --rpc-cors all \ + --max-blocks-per-request 64 \ + >"${log_file}" 2>&1 & + + local pid=$! + echo "${pid}" >"${RUN_DIR}/node.pid" + echo "native" >"${RUN_DIR}/node.mode" + echo "${node_bin}" >"${RUN_DIR}/node.bin" + echo "Node pid ${pid}; miner QUIC on 127.0.0.1:${miner_listen_port}/udp" + echo "Logs: ${log_file}" +} + +start_node_docker() { + ensure_rewards if docker ps -a --format '{{.Names}}' | grep -qx "${NODE_CONTAINER_NAME}"; then echo "Removing existing container ${NODE_CONTAINER_NAME} ..." @@ -105,7 +311,7 @@ start_node() { mkdir -p "${RUN_DIR}/node-keys" "${RUN_DIR}/node-data" - echo "Starting quantus-node (${NODE_CONTAINER_NAME}) ..." + echo "Starting quantus-node via Docker (${NODE_CONTAINER_NAME}) ..." docker run -d \ --name "${NODE_CONTAINER_NAME}" \ --restart unless-stopped \ @@ -123,7 +329,7 @@ start_node() { --base-path /var/lib/quantus \ --chain "${chain}" \ --node-key-file /node-keys/key_node \ - --rewards-inner-hash "${rewards}" \ + --rewards-inner-hash "${REWARDS_INNER_HASH}" \ --name "${node_name}" \ --wasm-execution compiled \ --db-cache 2048 \ @@ -133,6 +339,7 @@ start_node() { >/dev/null echo "${NODE_CONTAINER_NAME}" >"${RUN_DIR}/node.container" + echo "docker" >"${RUN_DIR}/node.mode" echo "Node listening for miners on 127.0.0.1:${miner_listen_port}/udp" } @@ -158,22 +365,7 @@ start_miner() { local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" local miner_log="${MINER_LOG:-info}" - if [[ -f "${RUN_DIR}/miner.pid" ]]; then - local old_pid - old_pid="$(cat "${RUN_DIR}/miner.pid")" - if kill -0 "${old_pid}" 2>/dev/null; then - echo "Stopping previous miner (pid ${old_pid}) ..." - kill "${old_pid}" 2>/dev/null || true - local i - for ((i = 0; i < 10; i++)); do - kill -0 "${old_pid}" 2>/dev/null || break - sleep 0.5 - done - if kill -0 "${old_pid}" 2>/dev/null; then - kill -9 "${old_pid}" 2>/dev/null || true - fi - fi - fi + stop_pidfile "${RUN_DIR}/miner.pid" "miner" mkdir -p "${RUN_DIR}" local log_file="${RUN_DIR}/miner.log" @@ -197,58 +389,90 @@ start_miner() { echo "Logs: ${log_file}" } +do_fetch_node() { + load_env + require_cmd curl + require_cmd tar + mkdir -p "${BIN_DIR}" + local bin + bin="$(resolve_node_bin)" + echo "quantus-node ready: ${bin}" + echo "Generate rewards hash with: ./setup.sh wormhole" +} + +do_wormhole() { + load_env + require_cmd curl + require_cmd tar + mkdir -p "${BIN_DIR}" + local bin + bin="$(resolve_node_bin)" + echo "Using ${bin}" >&2 + echo "Save the inner_hash into .env as REWARDS_INNER_HASH" >&2 + echo >&2 + "${bin}" key quantus --scheme wormhole +} + do_start() { load_env check_prereqs mkdir -p "${RUN_DIR}" - start_node - # Brief pause so UDP listener is up before miner connects + + if [[ "${NODE_MODE}" == "docker" ]]; then + start_node_docker + else + start_node_native + fi + sleep 2 start_miner echo - echo "Stack is up. Record a spreadsheet row with:" + echo "Stack is up (same host). Next:" echo " ./record.sh --provider --cost-per-hour " + echo "Or hardware-only (no node): ./record.sh --benchmark --provider --cost-per-hour " } do_stop() { load_env - if [[ -f "${RUN_DIR}/miner.pid" ]]; then - local pid - pid="$(cat "${RUN_DIR}/miner.pid")" - if kill -0 "${pid}" 2>/dev/null; then - echo "Stopping miner (pid ${pid}) ..." - kill "${pid}" 2>/dev/null || true - # Give it a moment; force if needed - sleep 1 - if kill -0 "${pid}" 2>/dev/null; then - kill -9 "${pid}" 2>/dev/null || true - fi - fi - rm -f "${RUN_DIR}/miner.pid" - fi + stop_pidfile "${RUN_DIR}/miner.pid" "miner" + stop_pidfile "${RUN_DIR}/node.pid" "node" local container="${NODE_CONTAINER_NAME}" if [[ -f "${RUN_DIR}/node.container" ]]; then container="$(cat "${RUN_DIR}/node.container")" fi - if docker ps -a --format '{{.Names}}' | grep -qx "${container}"; then + if command -v docker >/dev/null 2>&1 && docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "${container}"; then echo "Removing node container ${container} ..." docker rm -f "${container}" >/dev/null fi - rm -f "${RUN_DIR}/node.container" + rm -f "${RUN_DIR}/node.container" "${RUN_DIR}/node.mode" echo "Stopped." } do_status() { load_env echo "Run dir: ${RUN_DIR}" - if [[ -f "${RUN_DIR}/node.container" ]]; then + local mode="unknown" + if [[ -f "${RUN_DIR}/node.mode" ]]; then + mode="$(cat "${RUN_DIR}/node.mode")" + fi + echo "Node mode: ${mode}" + + if [[ -f "${RUN_DIR}/node.pid" ]]; then + local pid + pid="$(cat "${RUN_DIR}/node.pid")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Node: running (pid ${pid})" + else + echo "Node: not running (stale pid ${pid})" + fi + elif [[ -f "${RUN_DIR}/node.container" ]]; then local c c="$(cat "${RUN_DIR}/node.container")" - if docker ps --format '{{.Names}}' | grep -qx "${c}"; then - echo "Node: running (${c})" + if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "${c}"; then + echo "Node: running (docker ${c})" else - echo "Node: not running (recorded ${c})" + echo "Node: not running (recorded docker ${c})" fi else echo "Node: not started via setup.sh" @@ -277,15 +501,33 @@ do_status() { fi } -cmd="${1:-start}" +cmd="start" +while [[ $# -gt 0 ]]; do + case "$1" in + start | stop | status | fetch-node | wormhole) + cmd="$1" + shift + ;; + --docker) + NODE_MODE="docker" + shift + ;; + -h | --help | help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + case "${cmd}" in start) do_start ;; stop) do_stop ;; status) do_status ;; - -h | --help | help) usage ;; - *) - echo "error: unknown command: ${cmd}" >&2 - usage >&2 - exit 1 - ;; + fetch-node) do_fetch_node ;; + wormhole) do_wormhole ;; esac From 38da9e4b77514c73e17a13fb4a914aaf7948c44a Mon Sep 17 00:00:00 2001 From: Jangle <30788925+Jfangle@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:04:51 +0800 Subject: [PATCH 03/13] updated gpu-bench/README.md --- gpu-bench/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gpu-bench/README.md b/gpu-bench/README.md index 2b2a13e..9db02bd 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -1,7 +1,7 @@ # GPU miner bench (provider-agnostic) -Spin up a Quantus node + native GPU miner on any rented NVIDIA GPU host (Vast.ai, -RunPod, etc.), sample hashrate / utilization / GPU specs, and append a row to +Spin up a Quantus node + native GPU miner on any rented NVIDIA GPU host (Clore, +RunPod, Vast.ai, etc.), sample hashrate / utilization / GPU specs, and append a row to [`results.csv`](results.csv) so we can answer “what hardware should we recommend?” No cloud APIs. You supply `cloud_provider` and `cost_per_hour`. @@ -28,11 +28,15 @@ cp .env.example .env # paste inner_hash into .env as REWARDS_INNER_HASH ./setup.sh # native node + GPU miner on this host -./record.sh --provider runpod --cost-per-hour 0.35 +./record.sh --provider clore --cost-per-hour 0.35 # or --provider runpod, vast, ... ./setup.sh stop ``` +Pass whatever marketplace you rented from as `--provider` (e.g. `clore`, +`runpod`, `vast`) plus the machine's total `--cost-per-hour` in USD — the +scripts have no provider-specific logic. + `setup.sh` will: 1. Resolve `quantus-node` (`QUANTUS_NODE_BIN` → `PATH` → sibling `chain` build → GitHub release download into `.run/bin/`) From fcfa887c24b3a6b5fd4aab798a070f9ae890500b Mon Sep 17 00:00:00 2001 From: Jangle <30788925+Jfangle@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:36:00 +0800 Subject: [PATCH 04/13] fix gpu-bench: nvidia-smi sm-count fallback and node-key path capture - record.sh: driver 580+ removed multiprocessor_count from nvidia-smi; the failed query exited under pipefail before the fallback could run. Treat failure as empty and fall through (sm_count stays blank). - setup.sh: ensure_node_key echoed its progress message to stdout, which the caller captured into the node-key file path, so first runs failed with NetworkKeyNotFound. Send the message to stderr. Found on a Clore GTX 1080 Ti rental during Phase 0 validation. --- gpu-bench/record.sh | 5 ++--- gpu-bench/setup.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh index 9a1ee68..a19c7c1 100755 --- a/gpu-bench/record.sh +++ b/gpu-bench/record.sh @@ -79,11 +79,10 @@ query_gpu_static() { local line if ! line="$(nvidia-smi --query-gpu=name,memory.total,multiprocessor_count,driver_version \ --format=csv,noheader,nounits 2>/dev/null | head -n 1)"; then - echo "error: failed to query GPU via nvidia-smi" >&2 - exit 1 + line="" fi if [[ -z "${line}" ]]; then - # Older drivers may lack multiprocessor_count + # Some drivers (e.g. 580.x) do not support multiprocessor_count line="$(nvidia-smi --query-gpu=name,memory.total,driver_version \ --format=csv,noheader,nounits 2>/dev/null | head -n 1)" GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" diff --git a/gpu-bench/setup.sh b/gpu-bench/setup.sh index 1244a2b..758c98c 100755 --- a/gpu-bench/setup.sh +++ b/gpu-bench/setup.sh @@ -239,7 +239,7 @@ ensure_node_key() { local key_file="${RUN_DIR}/node-keys/key_node" mkdir -p "$(dirname "${key_file}")" if [[ ! -f "${key_file}" ]]; then - echo "Generating node key at ${key_file} ..." + echo "Generating node key at ${key_file} ..." >&2 "${node_bin}" key generate-node-key --file "${key_file}" fi echo "${key_file}" From 21de287b5bf20c24ca2a112492610f9d772d840b Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 17:45:14 +0800 Subject: [PATCH 05/13] Add RunPod GPU bench sweep with Vulkan-ready remote runner. Supports --dev node + release binaries over SSH, interactive runpod-shell debugging, SM-count fallbacks, and installing NVIDIA GL userspace on compute-only Community hosts. Co-authored-by: Cursor --- README.md | 13 +- gpu-bench/.env.example | 9 +- gpu-bench/.gitignore | 1 + gpu-bench/README.md | 182 +++++++---- gpu-bench/gpus.all.txt | 51 +++ gpu-bench/gpus.example.txt | 7 + gpu-bench/record.sh | 91 +++++- gpu-bench/remote-run.sh | 467 ++++++++++++++++++++++++++ gpu-bench/runpod-shell.sh | 248 ++++++++++++++ gpu-bench/runpod-sweep.sh | 654 +++++++++++++++++++++++++++++++++++++ gpu-bench/setup.sh | 72 ++-- 11 files changed, 1683 insertions(+), 112 deletions(-) create mode 100644 gpu-bench/gpus.all.txt create mode 100644 gpu-bench/gpus.example.txt create mode 100755 gpu-bench/remote-run.sh create mode 100755 gpu-bench/runpod-shell.sh create mode 100755 gpu-bench/runpod-sweep.sh diff --git a/README.md b/README.md index 76d0feb..f0591d7 100644 --- a/README.md +++ b/README.md @@ -75,13 +75,16 @@ hashrate / utilization / VRAM / efficiency rows to a CSV: ```bash cd gpu-bench -cp .env.example .env -./setup.sh wormhole # save inner_hash into .env -./setup.sh # native node + GPU miner (same host) -./record.sh --provider vast.ai --cost-per-hour 0.35 +./setup.sh --dev # local --dev node + GPU miner (no sync / no rewards hash) +./record.sh --provider runpod --cost-per-hour 0.69 ``` -Or hardware-only (no node): `./record.sh --benchmark --provider runpod --cost-per-hour 0.42` +RunPod multi-GPU sweep (REST API + SSH, release binaries, Prometheus → CSV): + +```bash +export RUNPOD_API_KEY=... +./runpod-sweep.sh --gpus-file gpus.example.txt +``` See [`gpu-bench/README.md`](gpu-bench/README.md). diff --git a/gpu-bench/.env.example b/gpu-bench/.env.example index 1e344a8..35f43c6 100644 --- a/gpu-bench/.env.example +++ b/gpu-bench/.env.example @@ -1,9 +1,6 @@ -# Wormhole inner hash (32-byte preimage, 0x-prefixed hex). -# Generate with: -# ./setup.sh # downloads node if needed, then Ctrl+C before mining — or: -# quantus-node key quantus --scheme wormhole -# # or: ./.run/bin/quantus-node key quantus --scheme wormhole -# Treat as sensitive ownership material — do not commit this file. +# Not required for ./setup.sh --dev or remote-run.sh / runpod-sweep.sh. +# Required only for Planck (non-dev) mining. +# Generate with: ./setup.sh wormhole REWARDS_INNER_HASH=0xyour_inner_hash_here CHAIN=planck diff --git a/gpu-bench/.gitignore b/gpu-bench/.gitignore index 70eb89a..c268006 100644 --- a/gpu-bench/.gitignore +++ b/gpu-bench/.gitignore @@ -2,3 +2,4 @@ *.log .env results.local.csv +sweep-out/ diff --git a/gpu-bench/README.md b/gpu-bench/README.md index 2b2a13e..7c39222 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -1,92 +1,156 @@ # GPU miner bench (provider-agnostic) -Spin up a Quantus node + native GPU miner on any rented NVIDIA GPU host (Vast.ai, -RunPod, etc.), sample hashrate / utilization / GPU specs, and append a row to -[`results.csv`](results.csv) so we can answer “what hardware should we recommend?” +Spin up a Quantus `--dev` node + GPU miner on a rented NVIDIA host, scrape +Prometheus hashrate into [`results.csv`](results.csv), and compare hardware. -No cloud APIs. You supply `cloud_provider` and `cost_per_hour`. +## Release binaries -**Default path is native binaries on the same machine** (no Docker / miner-stack). -That works on RunPod, Vast, and similar GPU rentals. +The Linux miner release includes WGPU (same as a normal `cargo build -p miner-cli --release`): -## Prerequisites +`https://github.com/Quantus-Network/quantus-miner/releases/download/v3.3.1/quantus-miner-linux-x86_64` -On the GPU host: +`remote-run.sh` / the RunPod sweep download that plus the latest +`quantus-node` linux x86_64 tarball from +[chain releases](https://github.com/Quantus-Network/chain/releases/latest). -- NVIDIA driver + `nvidia-smi` -- `curl` + `tar` (to fetch `quantus-node` from GitHub Releases) -- Rust toolchain (to build the miner), **or** a prebuilt binary via `MINER_BIN` -- This `quantus-miner` checkout - -## Quick start (same-host mining) +## Same-host (manual) ```bash cd gpu-bench -cp .env.example .env +./setup.sh --dev # native --dev node + GPU miner (no rewards hash) +./record.sh --provider runpod --cost-per-hour 0.69 +./setup.sh stop +``` -./setup.sh wormhole # downloads node if needed; save inner_hash -# paste inner_hash into .env as REWARDS_INNER_HASH +On-pod one-shot (downloads binaries itself): -./setup.sh # native node + GPU miner on this host -./record.sh --provider runpod --cost-per-hour 0.35 +```bash +./remote-run.sh --provider runpod --cost-per-hour 0.69 --duration 60 +``` -./setup.sh stop +## RunPod API sweep + +Uses the **REST API** (not MCP): create Pod → SSH → `remote-run.sh` → scp CSV → delete. + +### 1. Prerequisites (laptop) + +- `RUNPOD_API_KEY` from [RunPod settings](https://www.runpod.io/console/user/settings) +- SSH public key added in RunPod **Settings → SSH Public Keys** +- `curl`, `ssh`, `scp`, `python3` + +### 2. How to make a RunPod template (optional but nice) + +You don’t *need* a custom template — the sweep defaults to a slim RunPod CUDA +base image (`runpod/base:…-cuda…`) with port `22/tcp`. **Don’t use PyTorch** +images unless you need Torch; they’re much larger and unused here. NVIDIA +drivers are on the host; the image just needs CUDA userspace + RunPod’s +`/start.sh` (SSH). + +**Console** + +1. [Templates → New Template](https://www.console.runpod.io/user/templates) +2. Image: e.g. `runpod/base:1.1.0-cuda1281-ubuntu2404` (use **24.04** — node needs GLIBC ≥ 2.38) +3. Env: `NVIDIA_DRIVER_CAPABILITIES=all` (required for Vulkan/WGPU; compute-only → llvmpipe only) +4. Container disk ~20GB (keep small on Community — large disks often 500), volume 0 unless you need persistence +5. Expose **TCP 22** (SSH). `runpod/base` / `runpod/pytorch` already start `sshd`. +6. Save → copy the template id → `export TEMPLATE_ID=...` + +**API** + +```bash +curl -sS -X POST https://rest.runpod.io/v1/templates \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "quantus-gpu-bench", + "imageName": "runpod/base:1.1.0-cuda1281-ubuntu2404", + "category": "NVIDIA", + "containerDiskInGb": 20, + "volumeInGb": 0, + "volumeMountPath": "/workspace", + "ports": ["22/tcp", "9900/http"], + "env": { "NVIDIA_DRIVER_CAPABILITIES": "all" }, + "isPublic": false + }' ``` -`setup.sh` will: +### 3. Debug one Pod interactively (recommended first) -1. Resolve `quantus-node` (`QUANTUS_NODE_BIN` → `PATH` → sibling `chain` build → GitHub release download into `.run/bin/`) -2. Generate a node key under `.run/node-keys/` -3. Start the node with `--miner-listen-port 9833` -4. Build/start the GPU miner against `127.0.0.1:9833` (`--cpu-workers 0`) +Creates a Pod, uploads scripts, prints SSH, and **leaves it running** so you +can fix Vulkan / miner commands without pay-per-recreate: + +```bash +export RUNPOD_API_KEY=... +chmod +x runpod-shell.sh remote-run.sh record.sh runpod-sweep.sh -Optional Docker node (not recommended on RunPod): `./setup.sh start --docker` +./runpod-shell.sh "NVIDIA L4" +# ssh … (printed), or: ./runpod-shell.sh --ssh + +# edit remote-run.sh locally, then: +./runpod-shell.sh --upload +# on pod: bash -x ./remote-run.sh --cost-per-hour … --duration 30 + +./runpod-shell.sh --delete # stop billing when done +``` -### Hardware-only compare (no chain sync) +### 4. Run the sweep ```bash -./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 60 +export RUNPOD_API_KEY=... +# optional: export TEMPLATE_ID=... +# optional: export CLOUD_TYPE=SECURE # default COMMUNITY +# optional: export SSH_KEY=~/.ssh/id_ed25519 + +chmod +x remote-run.sh record.sh runpod-sweep.sh setup.sh + +./runpod-sweep.sh --gpus-file gpus.example.txt +# full NVIDIA catalog: +./runpod-sweep.sh --gpus-file gpus.all.txt +# or: +./runpod-sweep.sh "NVIDIA GeForce RTX 4090" "NVIDIA GeForce RTX 3090" ``` +Merged rows land in `sweep-out/results.csv`. + +On failure the Pod is deleted unless `KEEP_ON_FAILURE=1`. + +If Community has no capacity for a GPU, the sweep retries on **Secure** +(`FALLBACK_SECURE=1` by default). If a Pod is RUNNING but never gets a public +IP (common on Community), it falls back to proxy SSH +(`podId@ssh.runpod.io`) and pipes files instead of `scp`. + +### What each Pod does + +1. Download `quantus-node` + `quantus-miner` release binaries +2. `quantus-node --dev --miner-listen-port 9833` (local chain, no Planck sync) +3. `quantus-miner serve --gpu-devices 1 --cpu-workers 0` → `:9900` +4. Sample Prometheus (`miner_gpu_hash_rate`) + `nvidia-smi` into CSV +5. Tear down + ## Spreadsheet columns | Column | Source | |--------|--------| -| `timestamp` | UTC ISO time | -| `cloud_provider` | `--provider` | -| `gpu_model` | `nvidia-smi` name | -| `vram_mb` | total VRAM | -| `sm_count` | CUDA SM / multiprocessor count | -| `driver_version` | NVIDIA driver | -| `hashrate` | avg GPU H/s (`miner_gpu_hash_rate` live, or benchmark average) | -| `gpu_utilization_pct` | avg `utilization.gpu` over the sample | -| `cost_per_hour` | `--cost-per-hour` (USD) | +| `cloud_provider` | `runpod` / flag | +| `gpu_model`, `vram_mb`, `sm_count` | `nvidia-smi` (not the RunPod catalog) | +| `hashrate` | avg `miner_gpu_hash_rate` from `:9900/metrics` | +| `gpu_utilization_pct` | avg `utilization.gpu` | +| `cost_per_hour` | Pod `costPerHr` from API (sweep) or your flag | | `efficiency` | `hashrate / cost_per_hour` | -| `sample_seconds` | duration used | -| `notes` | optional `--notes` | ## Scripts -| Script | Purpose | -|--------|---------| -| [`setup.sh`](setup.sh) | Native node + GPU miner on this host; `stop` / `status` | -| [`record.sh`](record.sh) | Sample metrics and append one CSV row (`--live` or `--benchmark`) | - -## Environment - -| Variable | Default | Meaning | -|----------|---------|---------| -| `REWARDS_INNER_HASH` | (required for setup) | Wormhole preimage for rewards | -| `QUANTUS_NODE_BIN` | download to `.run/bin/quantus-node` | Skip release download | -| `QUANTUS_MINER_DIR` | repo root | Miner source tree | -| `MINER_BIN` | built `target/release/quantus-miner` | Skip cargo build | -| `GPU_DEVICES` | `1` | GPUs for miner / benchmark | -| `METRICS_PORT` | `9900` | Miner Prometheus port | +| Script | Role | +|--------|------| +| [`setup.sh`](setup.sh) | Local native node+miner (`--dev` or Planck) | +| [`remote-run.sh`](remote-run.sh) | On-box: fetch bins, `--dev`, mine, `record.sh` | +| [`record.sh`](record.sh) | Prometheus / nvidia-smi → CSV row | +| [`runpod-sweep.sh`](runpod-sweep.sh) | RunPod REST API multi-GPU loop | +| [`runpod-shell.sh`](runpod-shell.sh) | One Pod + SSH; keep alive for manual debug | +| [`gpus.all.txt`](gpus.all.txt) | Full RunPod NVIDIA `gpuTypeId` list | ## Notes -- Prefer this over `chain/miner-stack` on cloud GPUs: Compose + the public miner - image are CPU-oriented and awkward on nested-Docker hosts. -- Node and miner share the host; QUIC is `127.0.0.1:9833/udp`. -- Give the volume enough disk for chain sync (tens of GB). -- Mining during live bench uses `--cpu-workers 0` so hashrate reflects the GPU. +- `--dev` is for **hardware comparison**, not mainnet rewards. +- Prefer this over `chain/miner-stack` on cloud GPUs (Compose / nested Docker). +- Catalog “vCPU” is host CPU, not CUDA SMs — SM count comes from inside the Pod. diff --git a/gpu-bench/gpus.all.txt b/gpu-bench/gpus.all.txt new file mode 100644 index 0000000..aafd337 --- /dev/null +++ b/gpu-bench/gpus.all.txt @@ -0,0 +1,51 @@ +# Full RunPod NVIDIA gpuTypeId list (AMD omitted — WGPU Vulkan path is NVIDIA). +# Source: https://docs.runpod.io/references/gpu-types +# Usage: ./runpod-sweep.sh --gpus-file gpus.all.txt + +# Consumer / GeForce +NVIDIA GeForce RTX 3070 +NVIDIA GeForce RTX 3080 +NVIDIA GeForce RTX 3080 Ti +NVIDIA GeForce RTX 3090 +NVIDIA GeForce RTX 3090 Ti +NVIDIA GeForce RTX 4070 Ti +NVIDIA GeForce RTX 4080 +NVIDIA GeForce RTX 4080 SUPER +NVIDIA GeForce RTX 4090 +NVIDIA GeForce RTX 5080 +NVIDIA GeForce RTX 5090 + +# Pro / Ada / Ampere workstation +NVIDIA RTX A2000 +NVIDIA RTX A4000 +NVIDIA RTX A4500 +NVIDIA RTX A5000 +NVIDIA RTX A6000 +NVIDIA RTX 2000 Ada Generation +NVIDIA RTX 4000 Ada Generation +NVIDIA RTX 4000 SFF Ada Generation +NVIDIA RTX 5000 Ada Generation +NVIDIA RTX 6000 Ada Generation +NVIDIA RTX PRO 4500 Blackwell +NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition +NVIDIA RTX PRO 6000 Blackwell Server Edition +NVIDIA RTX PRO 6000 Blackwell Workstation Edition + +# Data center +NVIDIA L4 +NVIDIA L40 +NVIDIA L40S +NVIDIA A30 +NVIDIA A40 +NVIDIA A100 80GB PCIe +NVIDIA A100-SXM4-80GB +NVIDIA H100 PCIe +NVIDIA H100 NVL +NVIDIA H100 80GB HBM3 +NVIDIA H200 +NVIDIA H200 NVL +NVIDIA B200 +NVIDIA B300 SXM6 AC +Tesla V100-PCIE-16GB +Tesla V100-SXM2-16GB +Tesla V100-SXM2-32GB diff --git a/gpu-bench/gpus.example.txt b/gpu-bench/gpus.example.txt new file mode 100644 index 0000000..21b8b94 --- /dev/null +++ b/gpu-bench/gpus.example.txt @@ -0,0 +1,7 @@ +# One RunPod gpuTypeId per line (from console / API). +# Lines starting with # are ignored. +#NVIDIA GeForce RTX 4090 +NVIDIA GeForce RTX 3090 +#NVIDIA RTX A5000 +NVIDIA L4 +NVIDIA RTX A4000 diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh index 9a1ee68..700991c 100755 --- a/gpu-bench/record.sh +++ b/gpu-bench/record.sh @@ -74,28 +74,89 @@ ensure_results_header() { fi } +# Best-effort SM / multiprocessor count. Many cloud drivers omit +# --query-gpu=multiprocessor_count (fails the whole CSV row), so we try +# several sources after name/VRAM are already known. +query_sm_count() { + local name="${1:-}" + local sm="" + + sm="$(nvidia-smi --query-gpu=multiprocessor_count --format=csv,noheader,nounits 2>/dev/null \ + | head -n 1 | tr -d '[:space:]')" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + sm="$(nvidia-smi -q 2>/dev/null \ + | awk -F: 'tolower($0) ~ /multiprocessor count/ { + gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit + }')" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + sm="$(nvidia-smi -q -x 2>/dev/null \ + | sed -n 's/.*\([0-9][0-9]*\)<\/multiprocessor_count>.*/\1/p' \ + | head -n 1)" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + # Static fallback for common RunPod SKUs (architecture SM counts). + case "${name}" in + *"RTX 3070"*) echo 46 ;; + *"RTX 3080 Ti"*) echo 80 ;; + *"RTX 3080"*) echo 68 ;; + *"RTX 3090 Ti"*) echo 84 ;; + *"RTX 3090"*) echo 82 ;; + *"RTX 4070 Ti"*) echo 60 ;; + *"RTX 4080 SUPER"*) echo 80 ;; + *"RTX 4080"*) echo 76 ;; + *"RTX 4090"*) echo 128 ;; + *"RTX 5080"*) echo 84 ;; + *"RTX 5090"*) echo 170 ;; + *"RTX A2000"*) echo 26 ;; + *"RTX A4000"*) echo 48 ;; + *"RTX A4500"*) echo 56 ;; + *"RTX A5000"*) echo 64 ;; + *"RTX A6000"*) echo 84 ;; + *"RTX 2000 Ada"*) echo 22 ;; + *"RTX 4000 Ada"*|*"RTX 4000 SFF Ada"*) echo 48 ;; + *"RTX 5000 Ada"*) echo 100 ;; + *"RTX 6000 Ada"*) echo 142 ;; + *"RTX PRO 4500"*) echo 80 ;; + *"RTX PRO 6000"*) echo 188 ;; + *"NVIDIA L4"|*" L4") echo 60 ;; + *"L40S"*) echo 142 ;; + *"L40"*) echo 142 ;; + *"NVIDIA A30"|*" A30") echo 56 ;; + *"NVIDIA A40"|*" A40") echo 84 ;; + *"A100"*) echo 108 ;; + *"H100 NVL"*) echo 132 ;; + *"H100"*) echo 132 ;; + *"H200"*) echo 132 ;; + *"B200"*) echo 160 ;; + *"B300"*) echo 160 ;; + *"V100"*) echo 80 ;; + *) echo "" ;; + esac +} + query_gpu_static() { - # name, memory.total [MiB], multiprocessor_count, driver_version local line - if ! line="$(nvidia-smi --query-gpu=name,memory.total,multiprocessor_count,driver_version \ - --format=csv,noheader,nounits 2>/dev/null | head -n 1)"; then + SM_COUNT="" + if ! line="$(nvidia-smi --query-gpu=name,memory.total,driver_version \ + --format=csv,noheader,nounits 2>/dev/null | head -n 1)" || [[ -z "${line}" ]]; then echo "error: failed to query GPU via nvidia-smi" >&2 exit 1 fi - if [[ -z "${line}" ]]; then - # Older drivers may lack multiprocessor_count - line="$(nvidia-smi --query-gpu=name,memory.total,driver_version \ - --format=csv,noheader,nounits 2>/dev/null | head -n 1)" - GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" - VRAM_MB="$(echo "${line}" | awk -F', ' '{print $2}' | tr -d ' ')" - SM_COUNT="" - DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" - return - fi GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" VRAM_MB="$(echo "${line}" | awk -F', ' '{print $2}' | tr -d ' ')" - SM_COUNT="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" - DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $4}' | tr -d ' ')" + DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" + SM_COUNT="$(query_sm_count "${GPU_MODEL}")" } read_gpu_util() { diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh new file mode 100755 index 0000000..0e5422c --- /dev/null +++ b/gpu-bench/remote-run.sh @@ -0,0 +1,467 @@ +#!/usr/bin/env bash +# Run on a GPU host (e.g. RunPod): fetch release binaries, start --dev node + +# GPU miner, sample Prometheus into results.csv. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${WORK_DIR:-/workspace/quantus-gpu-bench}" +BIN_DIR="${WORK_DIR}/bin" +RUN_DIR="${WORK_DIR}/.run" +RESULTS_CSV="${WORK_DIR}/results.csv" + +MINER_VERSION="${MINER_VERSION:-v3.3.1}" +MINER_URL="${MINER_URL:-https://github.com/Quantus-Network/quantus-miner/releases/download/${MINER_VERSION}/quantus-miner-linux-x86_64}" +NODE_URL="${NODE_URL:-}" # optional override; default = latest chain release + +PROVIDER="${PROVIDER:-runpod}" +COST_PER_HOUR="${COST_PER_HOUR:-}" +DURATION="${DURATION:-60}" +GPU_DEVICES="${GPU_DEVICES:-1}" +METRICS_PORT="${METRICS_PORT:-9900}" +MINER_LISTEN_PORT="${MINER_LISTEN_PORT:-9833}" +WARMUP_SECONDS="${WARMUP_SECONDS:-30}" +NOTES="${NOTES:-}" + +usage() { + cat <<'EOF' +Usage: ./remote-run.sh [options] + + --provider NAME cloud_provider column (default: runpod) + --cost-per-hour USD required for efficiency column + --duration SECONDS Prometheus sample window (default: 60) + --gpu-devices N default 1 + --warmup SECONDS wait after miner start before sampling (default: 30) + --notes TEXT + --miner-url URL override miner binary URL + --help + +Downloads linux x86_64 release binaries, runs quantus-node --dev with +--miner-listen-port, runs quantus-miner against 127.0.0.1, then samples +:9900/metrics into results.csv (via record.sh --live). +EOF +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --provider) PROVIDER="${2:-}"; shift 2 ;; + --cost-per-hour) COST_PER_HOUR="${2:-}"; shift 2 ;; + --duration) DURATION="${2:-}"; shift 2 ;; + --gpu-devices) GPU_DEVICES="${2:-}"; shift 2 ;; + --warmup) WARMUP_SECONDS="${2:-}"; shift 2 ;; + --notes) NOTES="${2:-}"; shift 2 ;; + --miner-url) MINER_URL="${2:-}"; shift 2 ;; + --node-url) NODE_URL="${2:-}"; shift 2 ;; + -h | --help) usage; exit 0 ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${COST_PER_HOUR}" ]]; then + echo "error: --cost-per-hour is required" >&2 + exit 1 +fi + +require_cmd curl +require_cmd tar +require_cmd nvidia-smi +if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed" >&2 + exit 1 +fi + +mkdir -p "${BIN_DIR}" "${RUN_DIR}" +cd "${WORK_DIR}" + +# Copy record.sh next to us if we're invoked from gpu-bench/ +if [[ -f "${SCRIPT_DIR}/record.sh" && ! -f "${WORK_DIR}/record.sh" ]]; then + cp "${SCRIPT_DIR}/record.sh" "${WORK_DIR}/record.sh" + chmod +x "${WORK_DIR}/record.sh" +fi +if [[ ! -x "${WORK_DIR}/record.sh" ]]; then + echo "error: record.sh not found next to remote-run.sh" >&2 + exit 1 +fi + +# Exit code for sweep: host has CUDA but no usable NVIDIA Vulkan (retry new pod). +EXIT_COMPUTE_ONLY=42 + +find_libglx_nvidia() { + local candidate + for candidate in \ + /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0 \ + /usr/lib64/libGLX_nvidia.so.0 \ + /usr/lib/libGLX_nvidia.so.0 \ + /usr/local/nvidia/lib64/libGLX_nvidia.so.0 \ + "${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0"; do + if [[ -e "${candidate}" ]]; then + echo "${candidate}" + return 0 + fi + done + ldconfig -p 2>/dev/null | awk '/libGLX_nvidia\.so\.0/ { print $NF; exit }' || true +} + +# Many Community hosts only mount compute libs. Install matching graphics +# userspace via apt, then (if needed) extract from the NVIDIA .run installer. +install_nvidia_gl_userspace() { + local ver major dest + ver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \ + | head -n 1 | tr -d '[:space:]')" + if [[ -z "${ver}" ]]; then + return 1 + fi + major="${ver%%.*}" + dest="${WORK_DIR}/nvidia-gl" + mkdir -p "${dest}" + + if command -v apt-get >/dev/null 2>&1; then + echo "Trying apt libnvidia-gl-${major} (host driver ${ver}) ..." >&2 + apt-get install -y -qq "libnvidia-gl-${major}" >/dev/null 2>&1 || true + if [[ -n "$(find_libglx_nvidia)" ]]; then + return 0 + fi + fi + + if [[ -e "${dest}/libGLX_nvidia.so.0" ]]; then + export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + return 0 + fi + + echo "Extracting NVIDIA ${ver} userspace libs (compute-only host) ..." >&2 + local tmp runfile url + tmp="$(mktemp -d)" + runfile="${tmp}/NVIDIA.run" + # GeForce/pro first; Tesla URL as fallback for datacenter hosts. + for url in \ + "https://download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ + "https://us.download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ + "https://us.download.nvidia.com/tesla/${ver}/NVIDIA-Linux-x86_64-${ver}.run"; do + if curl -fL --connect-timeout 20 --max-time 600 "${url}" -o "${runfile}"; then + break + fi + rm -f "${runfile}" + done + if [[ ! -f "${runfile}" ]]; then + echo "error: could not download NVIDIA ${ver} .run installer" >&2 + rm -rf "${tmp}" + return 1 + fi + + chmod +x "${runfile}" + # Extract only — do not install kernel modules into the container. + if ! sh "${runfile}" --extract-only --target "${tmp}/extract" >/tmp/nvidia-extract.log 2>&1; then + echo "error: NVIDIA .run extract failed; log:" >&2 + tail -n 40 /tmp/nvidia-extract.log >&2 || true + rm -rf "${tmp}" + return 1 + fi + + local f + for f in \ + libGLX_nvidia.so.* \ + libEGL_nvidia.so.* \ + libnvidia-glcore.so.* \ + libnvidia-glsi.so.* \ + libnvidia-tls.so.* \ + libnvidia-glvkspirv.so.* \ + libnvidia-gpucomp.so.* \ + libnvidia-rtcore.so.* \ + libnvoptix.so.*; do + # shellcheck disable=SC2086 + cp -n ${tmp}/extract/${f} "${dest}/" 2>/dev/null || true + done + # Stable SONAME links expected by the ICD. + local so + for so in libGLX_nvidia.so libEGL_nvidia.so; do + if [[ ! -e "${dest}/${so}.0" ]]; then + local real + real="$(ls -1 "${dest}/${so}".* 2>/dev/null | grep -v '\.so$' | sort -V | tail -n 1 || true)" + if [[ -n "${real}" ]]; then + ln -sfn "$(basename "${real}")" "${dest}/${so}.0" + fi + fi + done + + if [[ -f "${tmp}/extract/nvidia_icd.json" ]]; then + mkdir -p /usr/share/vulkan/icd.d + # Rewrite library_path to our extracted lib if needed later. + cp -f "${tmp}/extract/nvidia_icd.json" /usr/share/vulkan/icd.d/nvidia_icd.json || true + fi + + echo "${dest}" >"${dest}/.ld_path" + export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + echo "${dest}" >>/etc/ld.so.conf.d/nvidia-gl-bench.conf 2>/dev/null || true + ldconfig 2>/dev/null || true + + rm -rf "${tmp}" + if [[ -e "${dest}/libGLX_nvidia.so.0" ]] || [[ -n "$(find_libglx_nvidia)" ]]; then + echo "NVIDIA GL userspace ready in ${dest}" >&2 + return 0 + fi + return 1 +} + +# WGPU uses Vulkan. RunPod/CUDA images often only expose compute; Mesa's +# llvmpipe then becomes the only adapter and the miner refuses it. +ensure_nvidia_vulkan() { + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/dev/null 2>&1 || true + # libegl1/libxext6 are required for libGLX_nvidia to export vk_icd* in + # headless containers (without them: ERROR_INCOMPATIBLE_DRIVER). + # Do NOT install mesa-vulkan-drivers (llvmpipe distracts / can win). + apt-get install -y -qq \ + curl ca-certificates \ + libvulkan1 vulkan-tools \ + libegl1 libxext6 libgl1 \ + >/dev/null 2>&1 || \ + apt-get install -y -qq curl ca-certificates libvulkan1 libegl1 libxext6 >/dev/null 2>&1 || true + fi + mkdir -p /tmp/runtime-root + chmod 700 /tmp/runtime-root 2>/dev/null || true + export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" + + local lib + lib="$(find_libglx_nvidia)" + if [[ -z "${lib}" || ! -e "${lib}" ]]; then + echo "libGLX_nvidia.so.0 not mounted (compute-only host); installing userspace ..." >&2 + if ! install_nvidia_gl_userspace; then + echo "error: NVIDIA Vulkan/GL library missing (libGLX_nvidia.so.0)." >&2 + echo "exit ${EXIT_COMPUTE_ONLY}: sweep should retry on another host" >&2 + exit "${EXIT_COMPUTE_ONLY}" + fi + lib="$(find_libglx_nvidia)" + fi + + if [[ -z "${lib}" || ! -e "${lib}" ]]; then + echo "error: still no libGLX_nvidia.so.0 after userspace install" >&2 + exit "${EXIT_COMPUTE_ONLY}" + fi + + # Prefer extracted dir on LD_LIBRARY_PATH when we installed locally. + if [[ -d "${WORK_DIR}/nvidia-gl" ]]; then + export LD_LIBRARY_PATH="${WORK_DIR}/nvidia-gl${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + fi + + local icd_dir="/usr/share/vulkan/icd.d" + local icd="${icd_dir}/nvidia_icd.json" + mkdir -p "${icd_dir}" /etc/vulkan/icd.d + cat >"${icd}" </dev/null || true + echo "NVIDIA Vulkan ICD: ${icd} -> ${lib}" >&2 + + # Prefer NVIDIA ICD only (ignore any Mesa CPU ICDs that may already exist). + export VK_ICD_FILENAMES="${icd}" + export NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" + export __GLX_VENDOR_LIBRARY_NAME=nvidia + + if command -v vulkaninfo >/dev/null 2>&1; then + if ! vulkaninfo --summary 2>/dev/null | grep -qiE 'NVIDIA|GeForce|Tesla|Quadro|RTX|A100|L4|L40'; then + echo "error: vulkaninfo sees no NVIDIA GPU (only software/CPU adapters?)." >&2 + vulkaninfo --summary 2>&1 | tail -n 40 >&2 || true + echo "exit ${EXIT_COMPUTE_ONLY}: bad Vulkan host — sweep should retry" >&2 + exit "${EXIT_COMPUTE_ONLY}" + fi + echo "Vulkan NVIDIA adapter OK" >&2 + else + echo "warning: vulkan-tools not installed; skipping vulkaninfo check" >&2 + fi +} + +download_miner() { + local dest="${BIN_DIR}/quantus-miner" + if [[ -x "${dest}" ]]; then + echo "Using existing ${dest}" >&2 + echo "${dest}" + return + fi + echo "Downloading miner: ${MINER_URL}" >&2 + curl -fL "${MINER_URL}" -o "${dest}" + chmod +x "${dest}" + echo "${dest}" +} + +download_node() { + local dest="${BIN_DIR}/quantus-node" + if [[ -x "${dest}" ]]; then + echo "Using existing ${dest}" >&2 + echo "${dest}" + return + fi + + local url="${NODE_URL}" + if [[ -z "${url}" ]]; then + echo "Resolving latest quantus-node release ..." >&2 + local release_json tag + release_json="$(curl -fsSL https://api.github.com/repos/Quantus-Network/chain/releases/latest)" + tag="$(echo "${release_json}" | grep -o '"tag_name": "[^"]*"' | head -n 1 | cut -d'"' -f4)" + url="https://github.com/Quantus-Network/chain/releases/download/${tag}/quantus-node-${tag}-x86_64-unknown-linux-gnu.tar.gz" + fi + + echo "Downloading node: ${url}" >&2 + local tmp + tmp="$(mktemp -d)" + curl -fL "${url}" -o "${tmp}/node.tar.gz" + tar -xzf "${tmp}/node.tar.gz" -C "${tmp}" + if [[ ! -f "${tmp}/quantus-node" ]]; then + echo "error: archive missing quantus-node" >&2 + exit 1 + fi + mv "${tmp}/quantus-node" "${dest}" + chmod +x "${dest}" + rm -rf "${tmp}" + echo "${dest}" +} + +stop_pidfile() { + local pidfile="$1" + if [[ ! -f "${pidfile}" ]]; then + return 0 + fi + local pid + pid="$(cat "${pidfile}")" + if kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + sleep 1 + kill -9 "${pid}" 2>/dev/null || true + fi + rm -f "${pidfile}" +} + +cleanup() { + stop_pidfile "${RUN_DIR}/miner.pid" + stop_pidfile "${RUN_DIR}/node.pid" +} +trap cleanup EXIT + +ensure_nvidia_vulkan +NODE_BIN="$(download_node)" +MINER_BIN="$(download_miner)" + +# Fail fast with a clear hint if the host glibc is too old for the release binary. +if ! "${NODE_BIN}" --version >/tmp/qnode-ver.txt 2>/tmp/qnode-ver.err; then + if grep -q 'GLIBC_' /tmp/qnode-ver.err 2>/dev/null; then + echo "error: quantus-node needs a newer glibc than this image provides:" >&2 + cat /tmp/qnode-ver.err >&2 + echo "Use an Ubuntu 24.04+ image, e.g. IMAGE_NAME=runpod/base:1.1.0-cuda1281-ubuntu2404" >&2 + exit 1 + fi +fi + +echo "GPU:" +nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv || nvidia-smi || true + +mkdir -p "${RUN_DIR}/node-data" +echo "Starting quantus-node --dev ..." +nohup "${NODE_BIN}" \ + --dev \ + --base-path "${RUN_DIR}/node-data" \ + --rpc-port 9944 \ + --prometheus-port 9615 \ + --prometheus-external \ + --miner-listen-port "${MINER_LISTEN_PORT}" \ + --rpc-cors all \ + >"${RUN_DIR}/node.log" 2>&1 & +echo $! >"${RUN_DIR}/node.pid" +sleep 3 + +if ! kill -0 "$(cat "${RUN_DIR}/node.pid")" 2>/dev/null; then + echo "error: node failed to start; log:" >&2 + tail -n 80 "${RUN_DIR}/node.log" >&2 || true + if grep -q 'GLIBC_' "${RUN_DIR}/node.log" 2>/dev/null; then + echo "hint: IMAGE_NAME=runpod/base:1.1.0-cuda1281-ubuntu2404 (GLIBC 2.38+)" >&2 + fi + exit 1 +fi + +echo "Starting quantus-miner (gpu-devices=${GPU_DEVICES}) ..." +# Keep VK_ICD_FILENAMES / NVIDIA caps in the miner environment. +nohup env \ + VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ + NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" \ + __GLX_VENDOR_LIBRARY_NAME=nvidia \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" \ + XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" \ + "${MINER_BIN}" serve \ + --node-addr "127.0.0.1:${MINER_LISTEN_PORT}" \ + --gpu-devices "${GPU_DEVICES}" \ + --cpu-workers 0 \ + --metrics-port "${METRICS_PORT}" \ + >"${RUN_DIR}/miner.log" 2>&1 & +echo $! >"${RUN_DIR}/miner.pid" +sleep 3 + +if ! kill -0 "$(cat "${RUN_DIR}/miner.pid")" 2>/dev/null; then + echo "error: miner failed to start; log:" >&2 + tail -n 80 "${RUN_DIR}/miner.log" >&2 || true + if grep -qiE 'No usable GPU|llvmpipe|Vulkan' "${RUN_DIR}/miner.log" 2>/dev/null; then + echo "hint: need NVIDIA Vulkan ICD + NVIDIA_DRIVER_CAPABILITIES=all (not Mesa llvmpipe)" >&2 + fi + exit 1 +fi + +# Wait for metrics + non-zero hashrate (jobs from --dev) +echo "Warming up ${WARMUP_SECONDS}s / waiting for hashrate ..." +ready=0 +for ((i = 0; i < WARMUP_SECONDS + 60; i++)); do + if ! kill -0 "$(cat "${RUN_DIR}/miner.pid")" 2>/dev/null; then + echo "error: miner exited during warmup; log:" >&2 + tail -n 80 "${RUN_DIR}/miner.log" >&2 || true + exit 1 + fi + if curl -sf "http://127.0.0.1:${METRICS_PORT}/metrics" \ + | awk '/^miner_gpu_hash_rate[[:space:]]/ { if ($2+0 > 0) found=1 } END { exit !found }'; then + ready=1 + break + fi + sleep 1 +done + +if [[ "${ready}" -ne 1 ]]; then + echo "warning: no positive miner_gpu_hash_rate yet; sampling anyway" >&2 + echo "--- miner log ---" >&2 + tail -n 40 "${RUN_DIR}/miner.log" >&2 || true + echo "--- node log ---" >&2 + tail -n 40 "${RUN_DIR}/node.log" >&2 || true +fi + +# Point record.sh at our run dir / metrics +export METRICS_PORT +mkdir -p "${SCRIPT_DIR}/.run" 2>/dev/null || true +# record.sh reads .run/metrics.port relative to its own dir — use WORK_DIR copy +echo "${METRICS_PORT}" >"${WORK_DIR}/.run/metrics.port" + +NOTE_ARGS=() +if [[ -n "${NOTES}" ]]; then + NOTE_ARGS=(--notes "${NOTES}") +fi + +cd "${WORK_DIR}" +./record.sh --live \ + --provider "${PROVIDER}" \ + --cost-per-hour "${COST_PER_HOUR}" \ + --duration "${DURATION}" \ + "${NOTE_ARGS[@]}" + +echo "Done. Results: ${RESULTS_CSV}" +cat "${RESULTS_CSV}" +# Leave processes up until EXIT trap on script end — stop after record so CSV is final diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh new file mode 100755 index 0000000..73e14bf --- /dev/null +++ b/gpu-bench/runpod-shell.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# Spin up one RunPod GPU, upload bench scripts, print SSH — leave it running +# so you can iterate on Vulkan / miner commands without recreate-delete cycles. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=runpod-sweep.sh +source "${SCRIPT_DIR}/runpod-sweep.sh" + +STATE_FILE="${STATE_FILE:-${SCRIPT_DIR}/sweep-out/last-shell-pod.env}" + +usage() { + cat <<'EOF' +Usage: + ./runpod-shell.sh "NVIDIA L4" Create pod, upload scripts, print SSH + ./runpod-shell.sh --delete [POD_ID] Terminate pod (default: last created) + ./runpod-shell.sh --ssh ssh into last pod + ./runpod-shell.sh --status Show last pod id / SSH line + +Same env as runpod-sweep.sh (RUNPOD_API_KEY, IMAGE_NAME, CLOUD_TYPE, SSH_KEY, …). + +On the pod, typical debug loop: + cd /workspace/quantus-gpu-bench + bash -x ./remote-run.sh --cost-per-hour 0.39 --duration 30 --warmup 20 + + # or step through (see printed hints) +EOF +} + +write_state() { + local pod_id="$1" mode="$2" host="$3" port="$4" cost="$5" gpu_type="$6" + mkdir -p "$(dirname "${STATE_FILE}")" + # Quote values so GPU names with spaces survive `source`. + { + printf "POD_ID=%q\n" "${pod_id}" + printf "SSH_MODE=%q\n" "${mode}" + printf "SSH_HOST=%q\n" "${host}" + printf "SSH_PORT=%q\n" "${port}" + printf "COST_PER_HOUR=%q\n" "${cost}" + printf "GPU_TYPE=%q\n" "${gpu_type}" + printf "SSH_KEY=%q\n" "${SSH_KEY}" + printf "SSH_USER=%q\n" "${SSH_USER}" + printf "REMOTE_DIR=%q\n" "${REMOTE_DIR}" + } >"${STATE_FILE}" +} + +load_state() { + if [[ ! -f "${STATE_FILE}" ]]; then + echo "error: no saved pod state at ${STATE_FILE}" >&2 + echo "Create one with: ./runpod-shell.sh \"NVIDIA L4\"" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "${STATE_FILE}" +} + +print_ssh_hint() { + local mode="$1" host="$2" port="$3" cost="$4" pod_id="$5" + echo + echo "======== interactive pod ready ========" + echo "Pod id: ${pod_id}" + echo "costPerHr: ${cost}" + echo "state file: ${STATE_FILE}" + echo + if [[ "${mode}" == "direct" ]]; then + echo "ssh -i ${SSH_KEY} -p ${port} ${SSH_USER}@${host}" + else + echo "ssh -i ${SSH_KEY} ${host}@ssh.runpod.io" + fi + echo + echo "Re-attach: ./runpod-shell.sh --ssh" + echo "Tear down: ./runpod-shell.sh --delete" + echo + echo "On the pod:" + cat </dev/null || true + vulkaninfo --summary 2>&1 | head -n 60 + + # After editing scripts locally, re-upload from your laptop: + # ./runpod-shell.sh --upload +EOF + echo "=======================================" +} + +cmd_create() { + local gpu_type="$1" + + require_cmd curl + require_cmd ssh + require_cmd python3 + + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + if [[ ! -f "${SSH_KEY}" ]]; then + echo "error: SSH private key not found: ${SSH_KEY}" >&2 + exit 1 + fi + + echo "======== shell: ${gpu_type} ========" >&2 + local create_json pod_id create_rc=0 + set +e + create_json="$(create_pod "${gpu_type}")" + create_rc=$? + set -e + if [[ "${create_rc}" -eq 2 ]]; then + echo "error: no instances available for ${gpu_type} (Community + Secure)." >&2 + echo "tip: try another GPU, or CLOUD_TYPE=SECURE ./runpod-shell.sh \"${gpu_type}\"" >&2 + exit 2 + fi + if [[ "${create_rc}" -ne 0 ]]; then + echo "error: failed to create Pod for ${gpu_type}" >&2 + exit 1 + fi + # create_pod already printed JSON to stdout via create_pod_with_cloud; capture it. + # (create_json from $(create_pod) only has stdout — errors went to stderr.) + pod_id="$(json_pod_field "${create_json}" id)" + if [[ -z "${pod_id}" ]]; then + echo "error: create Pod response missing id:" >&2 + echo "${create_json}" >&2 + exit 1 + fi + echo "Pod id: ${pod_id}" >&2 + + local ready_line tag mode host port cost + if ! ready_line="$(wait_ssh "${pod_id}")"; then + echo "error: SSH never came up — deleting ${pod_id}" >&2 + delete_pod "${pod_id}" + exit 1 + fi + IFS='|' read -r tag mode host port cost <<<"${ready_line}" + if [[ "${tag}" != "ready" || -z "${host}" || -z "${port}" ]]; then + echo "error: bad wait_ssh result: ${ready_line}" >&2 + delete_pod "${pod_id}" + exit 1 + fi + [[ -n "${cost}" ]] || cost="0" + + ssh_upload_files "${mode}" "${host}" "${port}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" + ssh_cmd "${mode}" "${host}" "${port}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh'" + + write_state "${pod_id}" "${mode}" "${host}" "${port}" "${cost}" "${gpu_type}" + print_ssh_hint "${mode}" "${host}" "${port}" "${cost}" "${pod_id}" +} + +cmd_upload() { + load_state + echo "Uploading scripts to ${POD_ID} (${REMOTE_DIR}) ..." >&2 + ssh_upload_files "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" + ssh_cmd "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh'" + echo "Done." >&2 +} + +cmd_ssh() { + load_state + local ssh_opts=( + -i "${SSH_KEY}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o PreferredAuthentications=publickey + ) + if [[ "${SSH_MODE}" == "direct" ]]; then + exec ssh "${ssh_opts[@]}" -p "${SSH_PORT}" "${SSH_USER}@${SSH_HOST}" + else + exec ssh "${ssh_opts[@]}" "${SSH_HOST}@ssh.runpod.io" + fi +} + +cmd_status() { + load_state + echo "POD_ID=${POD_ID}" + echo "GPU_TYPE=${GPU_TYPE:-}" + echo "COST_PER_HOUR=${COST_PER_HOUR}" + if [[ "${SSH_MODE}" == "direct" ]]; then + echo "ssh -i ${SSH_KEY} -p ${SSH_PORT} ${SSH_USER}@${SSH_HOST}" + else + echo "ssh -i ${SSH_KEY} ${SSH_HOST}@ssh.runpod.io" + fi + if [[ -n "${RUNPOD_API_KEY:-}" ]]; then + local info status + if info="$(api GET "/pods/${POD_ID}?includeMachine=true" 2>/dev/null)"; then + status="$(json_pod_field "${info}" desiredStatus || true)" + echo "desiredStatus=${status}" + fi + fi +} + +cmd_delete() { + local pod_id="${1:-}" + if [[ -z "${pod_id}" ]]; then + load_state + pod_id="${POD_ID}" + fi + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + delete_pod "${pod_id}" + if [[ -f "${STATE_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${STATE_FILE}" + if [[ "${POD_ID:-}" == "${pod_id}" ]]; then + rm -f "${STATE_FILE}" + fi + fi +} + +case "${1:-}" in + -h | --help | "") + usage + [[ -n "${1:-}" ]] || exit 1 + exit 0 + ;; + --delete) + shift + cmd_delete "${1:-}" + ;; + --ssh) + cmd_ssh + ;; + --upload) + cmd_upload + ;; + --status) + cmd_status + ;; + --*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + cmd_create "$1" + ;; +esac diff --git a/gpu-bench/runpod-sweep.sh b/gpu-bench/runpod-sweep.sh new file mode 100755 index 0000000..30fbac7 --- /dev/null +++ b/gpu-bench/runpod-sweep.sh @@ -0,0 +1,654 @@ +#!/usr/bin/env bash +# Orchestrate RunPod Pods via REST API: for each GPU type, create → SSH → +# remote-run.sh (--dev node + miner + Prometheus CSV) → scp results → delete. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_BASE="${RUNPOD_API_BASE:-https://rest.runpod.io/v1}" +OUT_DIR="${OUT_DIR:-${SCRIPT_DIR}/sweep-out}" +SSH_KEY="${SSH_KEY:-${HOME}/.ssh/id_ed25519}" +SSH_USER="${SSH_USER:-root}" +CLOUD_TYPE="${CLOUD_TYPE:-COMMUNITY}" +# Prefer runpod/base (CUDA + SSH, no PyTorch). Drivers come from the host. +# Use a current Hub tag — stale tags leave Pods RUNNING with runtime=null forever. +# Ubuntu 24.04 required: recent quantus-node releases need GLIBC_2.38+ +# (22.04 ships 2.35 and will fail with "GLIBC_2.38 not found"). +IMAGE_NAME="${IMAGE_NAME:-runpod/base:1.1.0-cuda1281-ubuntu2404}" +TEMPLATE_ID="${TEMPLATE_ID:-}" +# Community hosts often reject large disks ("machine does not have the resources"). +# Binaries + --dev chain fit in a small container disk; volume defaults to 0. +CONTAINER_DISK_GB="${CONTAINER_DISK_GB:-20}" +VOLUME_GB="${VOLUME_GB:-0}" +DURATION="${DURATION:-60}" +WARMUP_SECONDS="${WARMUP_SECONDS:-45}" +GPU_DEVICES="${GPU_DEVICES:-1}" +REMOTE_DIR="${REMOTE_DIR:-/workspace/quantus-gpu-bench}" +KEEP_ON_FAILURE="${KEEP_ON_FAILURE:-0}" +CREATE_RETRIES="${CREATE_RETRIES:-5}" +CREATE_RETRY_SLEEP="${CREATE_RETRY_SLEEP:-15}" +# Retry a new Pod when remote-run exits 42 (compute-only / no Vulkan host). +HOST_RETRIES="${HOST_RETRIES:-3}" +EXIT_COMPUTE_ONLY=42 +# If Community create keeps failing capacity, retry on Secure Cloud. +FALLBACK_SECURE="${FALLBACK_SECURE:-1}" +# Proxy SSH (podId@ssh.runpod.io) is off by default — needs a RunPod-account +# SSH key and is easy to false-trigger while the image is still extracting. +# Direct TCP (after GraphQL shows port 22) is the reliable path. +ALLOW_PROXY_SSH="${ALLOW_PROXY_SSH:-0}" +# Image pull/extract on Community can take several minutes. +SSH_WAIT_SECONDS="${SSH_WAIT_SECONDS:-900}" + +usage() { + cat <<'EOF' +Usage: ./runpod-sweep.sh [gpuTypeId ...] + +Environment: + RUNPOD_API_KEY Required (https://www.runpod.io/console/user/settings) + SSH_KEY Default ~/.ssh/id_ed25519 (public key must be in RunPod) + CLOUD_TYPE COMMUNITY (default) or SECURE + IMAGE_NAME Docker image (default: runpod/base … ubuntu2404 for GLIBC) + TEMPLATE_ID Optional RunPod template id (skips IMAGE_NAME) + DURATION Sample seconds per GPU (default 60) + OUT_DIR Where to write merged results (default ./sweep-out) + KEEP_ON_FAILURE=1 Do not delete Pod if remote-run fails + HOST_RETRIES New Pods to try if host lacks Vulkan libs (default 3) + +GPU type ids are RunPod strings, e.g.: + "NVIDIA GeForce RTX 4090" + "NVIDIA GeForce RTX 3090" + "NVIDIA RTX A5000" + +Or pass a file via: + ./runpod-sweep.sh --gpus-file gpus.txt + +See README for creating a RunPod template. +EOF +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +# Last failed API response body (for callers to classify errors). +API_LAST_ERROR_BODY="" +API_LAST_HTTP_CODE="" + +# Prints response body to stdout. Exits non-zero on HTTP >= 400. +api() { + local method="$1" + local path="$2" + local data="${3:-}" + local tmp code + API_LAST_ERROR_BODY="" + API_LAST_HTTP_CODE="" + tmp="$(mktemp)" + local args=( + -sS + -X "${method}" + -H "Authorization: Bearer ${RUNPOD_API_KEY}" + -H "Content-Type: application/json" + -o "${tmp}" + -w "%{http_code}" + ) + if [[ -n "${data}" ]]; then + args+=(-d "${data}") + fi + code="$(curl "${args[@]}" "${API_BASE}${path}" || true)" + API_LAST_HTTP_CODE="${code}" + if [[ -z "${code}" || "${code}" == "000" ]]; then + API_LAST_ERROR_BODY="$(cat "${tmp}" 2>/dev/null || true)" + echo "error: RunPod API request failed (network): ${method} ${path}" >&2 + echo "${API_LAST_ERROR_BODY}" >&2 || true + rm -f "${tmp}" + return 1 + fi + if [[ "${code}" -ge 400 ]]; then + API_LAST_ERROR_BODY="$(cat "${tmp}" 2>/dev/null || true)" + echo "error: RunPod API ${method} ${path} -> HTTP ${code}" >&2 + if [[ "${VERBOSE:-0}" == "1" && -n "${data}" ]]; then + echo "request body:" >&2 + echo "${data}" >&2 + fi + echo "response: ${API_LAST_ERROR_BODY}" >&2 + echo >&2 + rm -f "${tmp}" + return 1 + fi + cat "${tmp}" + rm -f "${tmp}" +} + +# True if last API error means "no stock for this GPU" — skip the model. +is_no_capacity_error() { + echo "${API_LAST_ERROR_BODY}" | grep -qiE \ + 'no instances currently available|no .*available|out of capacity|insufficient.*capacity' +} + +# Extract a field from a Pod JSON object. Fails clearly if body is an error list/object. +json_pod_field() { + local body="$1" + local field="$2" + BODY="${body}" FIELD="${field}" python3 - <<'PY' +import json, os, sys +raw = os.environ["BODY"] +field = os.environ["FIELD"] +try: + data = json.loads(raw) +except json.JSONDecodeError as e: + print(f"error: invalid JSON from RunPod: {e}", file=sys.stderr) + print(raw[:2000], file=sys.stderr) + sys.exit(1) +if isinstance(data, list): + print("error: RunPod returned a list (usually an API error), not a Pod object:", file=sys.stderr) + print(json.dumps(data, indent=2)[:4000], file=sys.stderr) + sys.exit(1) +if not isinstance(data, dict): + print(f"error: unexpected RunPod JSON type: {type(data).__name__}", file=sys.stderr) + print(raw[:2000], file=sys.stderr) + sys.exit(1) +if field == "ssh_port": + m = data.get("portMappings") or {} + val = m.get("22") or m.get(22) or "" + if not val: + for p in (data.get("runtime") or {}).get("ports") or []: + if str(p.get("privatePort")) == "22" and p.get("publicPort"): + val = p.get("publicPort") + break +elif field == "publicIp": + val = data.get("publicIp") or "" + if not val: + for p in (data.get("runtime") or {}).get("ports") or []: + if str(p.get("privatePort")) == "22" and p.get("ip"): + val = p.get("ip") + break +elif field == "cost": + val = data.get("costPerHr") or data.get("adjustedCostPerHr") or "" +else: + val = data.get(field) or "" +print(val if val is not None else "") +PY +} + +# SSH helpers. MODE=direct|proxy. For proxy, HOST is pod id and PORT is unused. +ssh_cmd() { + local mode="$1" + local host="$2" + local port="$3" + shift 3 + if [[ -z "${host}" ]]; then + echo "error: ssh_cmd called with empty host (mode=${mode})" >&2 + return 2 + fi + local ssh_opts=( + -i "${SSH_KEY}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o GlobalKnownHostsFile=/dev/null + -o ConnectTimeout=20 + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o LogLevel=ERROR + ) + if [[ "${mode}" == "direct" ]]; then + if [[ -z "${port}" ]]; then + echo "error: direct ssh_cmd missing port" >&2 + return 2 + fi + ssh "${ssh_opts[@]}" -p "${port}" "${SSH_USER}@${host}" "$@" + else + # Basic/proxy SSH — no scp, but command + stdin/stdout pipes work. + ssh "${ssh_opts[@]}" "${host}@ssh.runpod.io" "$@" + fi +} + +# Upload local files into REMOTE_DIR (works for direct and proxy SSH). +ssh_upload_files() { + local mode="$1" + local host="$2" + local port="$3" + shift 3 + ssh_cmd "${mode}" "${host}" "${port}" "mkdir -p '${REMOTE_DIR}'" + local f + for f in "$@"; do + local base + base="$(basename "${f}")" + ssh_cmd "${mode}" "${host}" "${port}" "cat > '${REMOTE_DIR}/${base}'" <"${f}" + done +} + +ssh_download() { + local mode="$1" + local host="$2" + local port="$3" + local remote_path="$4" + local local_path="$5" + ssh_cmd "${mode}" "${host}" "${port}" "cat '${remote_path}'" >"${local_path}" +} + +# When sourced by runpod-shell.sh, only helpers below are used. +build_pod_payload() { + local gpu_type="$1" + local name="$2" + local disk_gb="$3" + local volume_gb="$4" + local cloud_type="$5" + GPU_TYPE="${gpu_type}" \ + CLOUD_TYPE="${cloud_type}" \ + IMAGE_NAME="${IMAGE_NAME}" \ + TEMPLATE_ID="${TEMPLATE_ID}" \ + POD_NAME="${name}" \ + CONTAINER_DISK_GB="${disk_gb}" \ + VOLUME_GB="${volume_gb}" \ + python3 - <<'PY' +import json, os +volume = int(os.environ["VOLUME_GB"]) +# graphics+utility so NVIDIA Container Toolkit mounts Vulkan/GL libs +# (compute-only pods only get CUDA → WGPU sees llvmpipe and fails). +body = { + "name": os.environ["POD_NAME"], + "cloudType": os.environ["CLOUD_TYPE"], + "computeType": "GPU", + "gpuTypeIds": [os.environ["GPU_TYPE"]], + "gpuCount": 1, + "gpuTypePriority": "availability", + "supportPublicIp": True, + "containerDiskInGb": int(os.environ["CONTAINER_DISK_GB"]), + "volumeMountPath": "/workspace", + "ports": ["22/tcp"], + "volumeInGb": volume, + "env": { + "NVIDIA_DRIVER_CAPABILITIES": "all", + "NVIDIA_VISIBLE_DEVICES": "all", + }, +} +template = os.environ.get("TEMPLATE_ID") or "" +if template: + body["templateId"] = template +else: + body["imageName"] = os.environ["IMAGE_NAME"] +print(json.dumps(body)) +PY +} + +create_pod_with_cloud() { + local gpu_type="$1" + local cloud_type="$2" + local name="qbench-$(echo "${gpu_type}" | tr -c 'A-Za-z0-9' '-' | cut -c1-40)-$(date +%s)" + local disk="${CONTAINER_DISK_GB}" + local volume="${VOLUME_GB}" + local attempt payload body_file api_rc + + body_file="$(mktemp)" + for ((attempt = 1; attempt <= CREATE_RETRIES; attempt++)); do + payload="$(build_pod_payload "${gpu_type}" "${name}-${attempt}" "${disk}" "${volume}" "${cloud_type}")" + echo "Creating Pod for ${gpu_type} on ${cloud_type} (attempt ${attempt}/${CREATE_RETRIES}, disk=${disk}G volume=${volume}G) ..." >&2 + if [[ "${VERBOSE:-0}" == "1" ]]; then + echo "payload: ${payload}" >&2 + fi + # Call api in this shell (not $(api …)) so API_LAST_ERROR_BODY is preserved. + set +e + api POST /pods "${payload}" >"${body_file}" + api_rc=$? + set -e + if [[ "${api_rc}" -eq 0 ]]; then + cat "${body_file}" + rm -f "${body_file}" + return 0 + fi + # No stock — shrinking disk will not help; stop this cloud tier. + if is_no_capacity_error; then + echo "no capacity for ${gpu_type} on ${cloud_type}" >&2 + rm -f "${body_file}" + return 2 + fi + if [[ "${volume}" -gt 0 ]]; then + volume=0 + elif [[ "${disk}" -gt 15 ]]; then + disk=15 + elif [[ "${disk}" -gt 10 ]]; then + disk=10 + fi + if [[ "${attempt}" -lt "${CREATE_RETRIES}" ]]; then + echo "retrying in ${CREATE_RETRY_SLEEP}s (next disk=${disk}G volume=${volume}G) ..." >&2 + sleep "${CREATE_RETRY_SLEEP}" + fi + done + rm -f "${body_file}" + return 1 +} + +# 0 = created (JSON on stdout), 2 = skip (no capacity on any tried tier), 1 = other failure +create_pod() { + local gpu_type="$1" + local rc=0 + + set +e + create_pod_with_cloud "${gpu_type}" "${CLOUD_TYPE}" + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + return 0 + fi + + # Community out of stock → try Secure (same as resource exhaustion). + if [[ "${FALLBACK_SECURE}" == "1" && "${CLOUD_TYPE}" == "COMMUNITY" ]]; then + echo "Community unavailable for ${gpu_type}; trying SECURE cloud ..." >&2 + set +e + create_pod_with_cloud "${gpu_type}" "SECURE" + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + return 0 + fi + fi + + if [[ "${rc}" -eq 2 ]]; then + return 2 + fi + echo "error: exhausted create retries for ${gpu_type}" >&2 + return 1 +} + +graphql_pod_ssh() { + # Prints one line to stdout: "IP PORT" when TCP/22 is mapped, else nothing. + # Quiet unless VERBOSE=1. + local pod_id="$1" + local query payload + query="$(printf 'query { pod(input: {podId: "%s"}) { id desiredStatus costPerHr runtime { ports { ip isIpPublic privatePort publicPort type } } } }' "${pod_id}")" + payload="$(POD_QUERY="${query}" python3 - <<'PY' +import json, os +print(json.dumps({"query": os.environ["POD_QUERY"]})) +PY +)" + local resp + resp="$(curl -sS -X POST \ + -H "content-type: application/json" \ + --url "https://api.runpod.io/graphql?api_key=${RUNPOD_API_KEY}" \ + -d "${payload}" || true)" + BODY="${resp}" VERBOSE="${VERBOSE:-0}" python3 - <<'PY' +import json, os, sys +raw = os.environ["BODY"] +verbose = os.environ.get("VERBOSE") == "1" +try: + data = json.loads(raw) +except Exception as e: + if verbose: + print(f"graphql parse error: {e}", file=sys.stderr) + sys.exit(0) +if data.get("errors"): + if verbose: + print("graphql errors:", json.dumps(data["errors"])[:500], file=sys.stderr) + sys.exit(0) +pod = ((data.get("data") or {}).get("pod")) or {} +ports = ((pod.get("runtime") or {}).get("ports")) or [] +best = None +for p in ports: + if int(p.get("privatePort") or 0) != 22: + continue + if not p.get("ip") or not p.get("publicPort"): + continue + if p.get("isIpPublic"): + best = p + break + if best is None: + best = p +if best: + print(f"{best['ip']} {best['publicPort']}") +elif verbose: + print(f" graphql: status={pod.get('desiredStatus')} ports={len(ports)}", file=sys.stderr) +PY +} + +# Prints one stdout line: ready|||| +# Do not print anything else to stdout (stderr is OK). +wait_ssh() { + local pod_id="$1" + if [[ -z "${pod_id}" ]]; then + echo "error: wait_ssh: empty pod id" >&2 + return 1 + fi + local deadline=$((SECONDS + SSH_WAIT_SECONDS)) + local last_heartbeat=0 + local spun_at=$SECONDS + + echo " waiting for image extract + SSH (up to ${SSH_WAIT_SECONDS}s) ..." >&2 + + while (( SECONDS < deadline )); do + local info ip port status cost g_ip g_port elapsed + elapsed=$((SECONDS - spun_at)) + if ! info="$(api GET "/pods/${pod_id}?includeMachine=true")"; then + sleep 10 + continue + fi + ip="$(json_pod_field "${info}" publicIp)" || return 1 + port="$(json_pod_field "${info}" ssh_port)" || return 1 + status="$(json_pod_field "${info}" desiredStatus)" || return 1 + cost="$(json_pod_field "${info}" cost)" || return 1 + [[ -n "${cost}" ]] || cost="0" + + g_ip="" + g_port="" + if read -r g_ip g_port <<<"$(graphql_pod_ssh "${pod_id}")"; then + if [[ -n "${g_ip}" && -n "${g_port}" ]]; then + ip="${g_ip}" + port="${g_port}" + fi + fi + + # Only attempt SSH after port mappings exist (container finished starting). + if [[ "${status}" == "RUNNING" && -n "${ip}" && -n "${port}" ]]; then + echo " ports up (${elapsed}s) — probing ${SSH_USER}@${ip}:${port}" >&2 + if ssh_cmd direct "${ip}" "${port}" nvidia-smi >/dev/null 2>&1; then + printf 'ready|direct|%s|%s|%s\n' "${ip}" "${port}" "${cost}" + return 0 + fi + echo " SSH daemon not ready yet, retrying ..." >&2 + elif (( elapsed - last_heartbeat >= 30 )); then + last_heartbeat=$elapsed + if [[ -n "${ip}" && -n "${port}" ]]; then + echo " still waiting ${elapsed}s (tcp=${ip}:${port}, ssh not ready)" >&2 + else + echo " still waiting ${elapsed}s (image extract / ports pending)" >&2 + fi + fi + + # Optional proxy — only after ports exist, and only if explicitly enabled. + if [[ "${ALLOW_PROXY_SSH}" == "1" && "${status}" == "RUNNING" && -n "${ip}" && -n "${port}" ]]; then + if ssh_cmd proxy "${pod_id}" "22" nvidia-smi >/dev/null 2>&1; then + printf 'ready|proxy|%s|22|%s\n' "${pod_id}" "${cost}" + return 0 + fi + fi + + sleep 10 + done + echo "error: timed out after ${SSH_WAIT_SECONDS}s waiting for Pod ${pod_id}" >&2 + echo "tip: console logs still extracting? increase SSH_WAIT_SECONDS=1200" >&2 + return 1 +} + +delete_pod() { + local pod_id="$1" + echo "Deleting Pod ${pod_id} ..." >&2 + api DELETE "/pods/${pod_id}" >/dev/null || true +} + +run_one_attempt() { + local gpu_type="$1" + local create_json pod_id create_rc=0 + set +e + create_json="$(create_pod "${gpu_type}")" + create_rc=$? + set -e + if [[ "${create_rc}" -eq 2 ]]; then + echo "skip: no instances available for ${gpu_type}" >&2 + return 2 + fi + if [[ "${create_rc}" -ne 0 ]]; then + echo "error: failed to create Pod for ${gpu_type}" >&2 + return 1 + fi + if ! pod_id="$(json_pod_field "${create_json}" id)"; then + echo "error: create Pod response missing id:" >&2 + echo "${create_json}" >&2 + return 1 + fi + if [[ -z "${pod_id}" ]]; then + echo "error: empty Pod id. Full response:" >&2 + echo "${create_json}" >&2 + return 1 + fi + echo "Pod id: ${pod_id}" >&2 + + local ready_line tag mode host port cost + if ! ready_line="$(wait_ssh "${pod_id}")"; then + delete_pod "${pod_id}" + return 1 + fi + IFS='|' read -r tag mode host port cost <<<"${ready_line}" + if [[ "${tag}" != "ready" || -z "${mode}" || -z "${host}" || -z "${port}" ]]; then + echo "error: bad wait_ssh result: ${ready_line}" >&2 + delete_pod "${pod_id}" + return 1 + fi + if [[ -z "${cost}" ]]; then + cost="0" + fi + if [[ "${mode}" == "direct" ]]; then + echo "SSH ${SSH_USER}@${host} -p ${port} (costPerHr=${cost})" >&2 + else + echo "SSH ${host}@ssh.runpod.io (costPerHr=${cost})" >&2 + fi + + ssh_upload_files "${mode}" "${host}" "${port}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" + + local remote_rc=0 + set +e + ssh_cmd "${mode}" "${host}" "${port}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' && \ + WORK_DIR='${REMOTE_DIR}' '${REMOTE_DIR}/remote-run.sh' \ + --provider runpod \ + --cost-per-hour '${cost}' \ + --duration '${DURATION}' \ + --warmup '${WARMUP_SECONDS}' \ + --gpu-devices '${GPU_DEVICES}' \ + --notes 'gpuTypeId=${gpu_type};pod=${pod_id};ssh=${mode}'" + remote_rc=$? + set -e + + if [[ "${remote_rc}" -eq 0 ]]; then + local local_row="${OUT_DIR}/row-${pod_id}.csv" + ssh_download "${mode}" "${host}" "${port}" \ + "${REMOTE_DIR}/results.csv" "${local_row}" + tail -n +2 "${local_row}" >>"${MERGED}" + echo "Appended results to ${MERGED}" >&2 + delete_pod "${pod_id}" + return 0 + fi + + echo "error: remote-run failed on ${gpu_type} (pod ${pod_id}, rc=${remote_rc})" >&2 + if [[ "${KEEP_ON_FAILURE}" == "1" ]]; then + echo "KEEP_ON_FAILURE=1 — leaving Pod ${pod_id} up" >&2 + if [[ "${mode}" == "direct" ]]; then + echo " ssh -i ${SSH_KEY} -p ${port} ${SSH_USER}@${host}" >&2 + else + echo " ssh -i ${SSH_KEY} ${host}@ssh.runpod.io" >&2 + fi + return "${remote_rc}" + fi + delete_pod "${pod_id}" + # Propagate compute-only so caller can try another host. + return "${remote_rc}" +} + +run_one() { + local gpu_type="$1" + local attempt rc=1 + for ((attempt = 1; attempt <= HOST_RETRIES; attempt++)); do + if [[ "${attempt}" -gt 1 ]]; then + echo "host retry ${attempt}/${HOST_RETRIES} for ${gpu_type} ..." >&2 + fi + set +e + run_one_attempt "${gpu_type}" + rc=$? + set -e + if [[ "${rc}" -eq 0 || "${rc}" -eq 2 ]]; then + return "${rc}" + fi + if [[ "${rc}" -eq "${EXIT_COMPUTE_ONLY}" && "${attempt}" -lt "${HOST_RETRIES}" ]]; then + echo "compute-only / bad Vulkan host — trying another Pod ..." >&2 + continue + fi + return "${rc}" + done + return "${rc}" +} + +runpod_sweep_main() { + local GPUS=() + while [[ $# -gt 0 ]]; do + case "$1" in + --gpus-file) + while IFS= read -r line || [[ -n "${line}" ]]; do + [[ -z "${line}" || "${line}" =~ ^# ]] && continue + GPUS+=("${line}") + done <"${2:?}" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + GPUS+=("$1") + shift + ;; + esac + done + + if [[ "${#GPUS[@]}" -eq 0 ]]; then + echo "error: pass at least one GPU type id" >&2 + usage >&2 + exit 1 + fi + + require_cmd curl + require_cmd ssh + require_cmd python3 + + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + if [[ ! -f "${SSH_KEY}" ]]; then + echo "error: SSH private key not found: ${SSH_KEY}" >&2 + exit 1 + fi + + mkdir -p "${OUT_DIR}" + MERGED="${OUT_DIR}/results.csv" + if [[ ! -f "${MERGED}" ]]; then + head -n 1 "${SCRIPT_DIR}/results.csv" >"${MERGED}" + fi + + local failed=0 + local gpu + for gpu in "${GPUS[@]}"; do + echo "======== ${gpu} ========" >&2 + if ! run_one "${gpu}"; then + failed=1 + fi + done + + echo "Merged CSV: ${MERGED}" >&2 + exit "${failed}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + runpod_sweep_main "$@" +fi diff --git a/gpu-bench/setup.sh b/gpu-bench/setup.sh index 1244a2b..bad694b 100755 --- a/gpu-bench/setup.sh +++ b/gpu-bench/setup.sh @@ -14,23 +14,24 @@ DEFAULT_CHAIN_DIR="$(cd "${REPO_ROOT}/.." && pwd)/chain" NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-quantus-gpu-bench-node}" NODE_MODE="native" # native | docker +DEV_MODE=0 # 1 = --dev local chain (no rewards hash / no sync) usage() { cat <<'EOF' -Usage: ./setup.sh [start|stop|status|fetch-node|wormhole] [--docker] +Usage: ./setup.sh [start|stop|status|fetch-node|wormhole] [--docker] [--dev] start Download/build binaries, start native node + GPU miner (default) stop Stop miner and node status Show run state fetch-node Download quantus-node into .run/bin/ (no start) wormhole Print a new wormhole address + inner_hash (for REWARDS_INNER_HASH) + --dev Local --dev chain (no Planck sync, rewards hash optional) --docker Use Docker for the node instead of a native binary (not recommended on RunPod / nested-Docker hosts) Environment (see .env.example): - REWARDS_INNER_HASH Required for start + REWARDS_INNER_HASH Required for start unless --dev QUANTUS_NODE_BIN Prebuilt quantus-node (skips download) - QUANTUS_MINER_DIR Path to quantus-miner checkout (default: parent of gpu-bench) MINER_BIN Prebuilt quantus-miner (skips cargo build) GPU_DEVICES Number of GPUs (default: 1) EOF @@ -246,15 +247,9 @@ ensure_node_key() { } start_node_native() { - ensure_rewards local node_bin node_bin="$(resolve_node_bin)" - local key_file - key_file="$(ensure_node_key "${node_bin}")" - local chain="${CHAIN:-planck}" - local node_name="${NODE_NAME:-gpu-bench-node}" - local p2p_port="${P2P_PORT:-30333}" local rpc_port="${RPC_PORT:-9944}" local prom_port="${PROMETHEUS_PORT:-9615}" local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" @@ -265,25 +260,44 @@ start_node_native() { local log_file="${RUN_DIR}/node.log" echo "Starting native quantus-node: ${node_bin}" - echo " --miner-listen-port ${miner_listen_port} --chain ${chain}" - nohup "${node_bin}" \ - --validator \ - --base-path "${base_path}" \ - --chain "${chain}" \ - --node-key-file "${key_file}" \ - --rewards-inner-hash "${REWARDS_INNER_HASH}" \ - --name "${node_name}" \ - --port "${p2p_port}" \ - --rpc-port "${rpc_port}" \ - --prometheus-port "${prom_port}" \ - --prometheus-external \ - --miner-listen-port "${miner_listen_port}" \ - --wasm-execution compiled \ - --db-cache 2048 \ - --rpc-cors all \ - --max-blocks-per-request 64 \ - >"${log_file}" 2>&1 & + if [[ "${DEV_MODE}" -eq 1 ]]; then + echo " --dev --miner-listen-port ${miner_listen_port}" + nohup "${node_bin}" \ + --dev \ + --base-path "${base_path}" \ + --rpc-port "${rpc_port}" \ + --prometheus-port "${prom_port}" \ + --prometheus-external \ + --miner-listen-port "${miner_listen_port}" \ + --rpc-cors all \ + >"${log_file}" 2>&1 & + else + ensure_rewards + local key_file + key_file="$(ensure_node_key "${node_bin}")" + local chain="${CHAIN:-planck}" + local node_name="${NODE_NAME:-gpu-bench-node}" + local p2p_port="${P2P_PORT:-30333}" + echo " --miner-listen-port ${miner_listen_port} --chain ${chain}" + nohup "${node_bin}" \ + --validator \ + --base-path "${base_path}" \ + --chain "${chain}" \ + --node-key-file "${key_file}" \ + --rewards-inner-hash "${REWARDS_INNER_HASH}" \ + --name "${node_name}" \ + --port "${p2p_port}" \ + --rpc-port "${rpc_port}" \ + --prometheus-port "${prom_port}" \ + --prometheus-external \ + --miner-listen-port "${miner_listen_port}" \ + --wasm-execution compiled \ + --db-cache 2048 \ + --rpc-cors all \ + --max-blocks-per-request 64 \ + >"${log_file}" 2>&1 & + fi local pid=$! echo "${pid}" >"${RUN_DIR}/node.pid" @@ -512,6 +526,10 @@ while [[ $# -gt 0 ]]; do NODE_MODE="docker" shift ;; + --dev) + DEV_MODE=1 + shift + ;; -h | --help | help) usage exit 0 From 38186dff4ef93b544e75135b583b13f66d9fbf41 Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 17:46:22 +0800 Subject: [PATCH 06/13] Track gpu-bench results.csv as a collaborative hardware dataset. Seed six RunPod sweep rows and point the sweep at results.csv so new samples append to the shared table. Co-authored-by: Cursor --- gpu-bench/README.md | 9 ++++++++- gpu-bench/results.csv | 6 ++++++ gpu-bench/runpod-sweep.sh | 18 +++++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/gpu-bench/README.md b/gpu-bench/README.md index 7c39222..ca198f4 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -110,7 +110,9 @@ chmod +x remote-run.sh record.sh runpod-sweep.sh setup.sh ./runpod-sweep.sh "NVIDIA GeForce RTX 4090" "NVIDIA GeForce RTX 3090" ``` -Merged rows land in `sweep-out/results.csv`. +Successful rows append to the shared dataset [`results.csv`](results.csv) +(commit new rows so others can reuse them). Per-pod temps go under +`sweep-out/` (gitignored). On failure the Pod is deleted unless `KEEP_ON_FAILURE=1`. @@ -127,6 +129,11 @@ IP (common on Community), it falls back to proxy SSH 4. Sample Prometheus (`miner_gpu_hash_rate`) + `nvidia-smi` into CSV 5. Tear down +## Collaborative dataset + +[`results.csv`](results.csv) is the shared hardware comparison table. After a +sweep or `record.sh` run, commit any new rows you want others to see. + ## Spreadsheet columns | Column | Source | diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv index 0114764..238b887 100644 --- a/gpu-bench/results.csv +++ b/gpu-bench/results.csv @@ -1 +1,7 @@ timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes +2026-08-01T08:16:22Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,7237290.633333,71.30,0.18,40207170.185183,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=4exbkxuw781hte;ssh=direct +2026-08-01T08:18:38Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.126.09,6613452.233333,61.40,0.22,30061146.515150,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=w40modnpa8t6na;ssh=direct +2026-08-01T08:21:18Z,runpod,NVIDIA GeForce RTX 3090 Ti,24564,84,580.65.06,7323359.689655,73.28,0.27,27123554.406130,60,gpuTypeId=NVIDIA GeForce RTX 3090 Ti;pod=lflvz4kxqkxsol;ssh=direct +2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,26346855.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct +2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,30352894.210526,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct +2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,39002828.965515,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct diff --git a/gpu-bench/runpod-sweep.sh b/gpu-bench/runpod-sweep.sh index 30fbac7..d63d924 100755 --- a/gpu-bench/runpod-sweep.sh +++ b/gpu-bench/runpod-sweep.sh @@ -6,6 +6,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" API_BASE="${RUNPOD_API_BASE:-https://rest.runpod.io/v1}" OUT_DIR="${OUT_DIR:-${SCRIPT_DIR}/sweep-out}" +# Shared collaborative dataset (tracked in git). Per-pod temps stay in OUT_DIR. +RESULTS_CSV="${RESULTS_CSV:-${SCRIPT_DIR}/results.csv}" SSH_KEY="${SSH_KEY:-${HOME}/.ssh/id_ed25519}" SSH_USER="${SSH_USER:-root}" CLOUD_TYPE="${CLOUD_TYPE:-COMMUNITY}" @@ -49,7 +51,8 @@ Environment: IMAGE_NAME Docker image (default: runpod/base … ubuntu2404 for GLIBC) TEMPLATE_ID Optional RunPod template id (skips IMAGE_NAME) DURATION Sample seconds per GPU (default 60) - OUT_DIR Where to write merged results (default ./sweep-out) + RESULTS_CSV Collaborative dataset to append (default ./results.csv) + OUT_DIR Per-pod temp rows (default ./sweep-out, gitignored) KEEP_ON_FAILURE=1 Do not delete Pod if remote-run fails HOST_RETRIES New Pods to try if host lacks Vulkan libs (default 3) @@ -545,8 +548,8 @@ run_one_attempt() { local local_row="${OUT_DIR}/row-${pod_id}.csv" ssh_download "${mode}" "${host}" "${port}" \ "${REMOTE_DIR}/results.csv" "${local_row}" - tail -n +2 "${local_row}" >>"${MERGED}" - echo "Appended results to ${MERGED}" >&2 + tail -n +2 "${local_row}" >>"${RESULTS_CSV}" + echo "Appended results to ${RESULTS_CSV}" >&2 delete_pod "${pod_id}" return 0 fi @@ -631,9 +634,10 @@ runpod_sweep_main() { fi mkdir -p "${OUT_DIR}" - MERGED="${OUT_DIR}/results.csv" - if [[ ! -f "${MERGED}" ]]; then - head -n 1 "${SCRIPT_DIR}/results.csv" >"${MERGED}" + if [[ ! -f "${RESULTS_CSV}" ]]; then + printf '%s\n' \ + "timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes" \ + >"${RESULTS_CSV}" fi local failed=0 @@ -645,7 +649,7 @@ runpod_sweep_main() { fi done - echo "Merged CSV: ${MERGED}" >&2 + echo "Dataset CSV: ${RESULTS_CSV}" >&2 exit "${failed}" } From 2bb4d0e4efa044f9fbd92096c7ce952725924bae Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 18:15:19 +0800 Subject: [PATCH 07/13] Improve GPU tier matching and build miner from git in gpu-bench. Split NVIDIA workstation/datacenter dispatch tiers, prefer clone+cargo build from MINER_BRANCH for iteration, and track cost_per_sec / hash_per_dollar in results.csv. Co-authored-by: Cursor --- README.md | 2 +- crates/engine-gpu/src/gpu_tiers.rs | 228 +++++++++++++++++++++++++++-- gpu-bench/README.md | 25 ++-- gpu-bench/gpus.all.txt | 11 +- gpu-bench/record.sh | 46 ++++-- gpu-bench/remote-run.sh | 132 +++++++++++++++-- gpu-bench/results.csv | 14 +- gpu-bench/runpod-shell.sh | 3 +- gpu-bench/runpod-sweep.sh | 37 ++++- 9 files changed, 433 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index f0591d7..7fba24e 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ cargo build -p miner-cli --release ### Cloud GPU bench + hardware spreadsheet Provider-agnostic tooling to run the miner on a rented NVIDIA GPU and append -hashrate / utilization / VRAM / efficiency rows to a CSV: +hashrate / utilization / VRAM / hash_per_dollar rows to a CSV: ```bash cd gpu-bench diff --git a/crates/engine-gpu/src/gpu_tiers.rs b/crates/engine-gpu/src/gpu_tiers.rs index 3089d73..c5e897a 100644 --- a/crates/engine-gpu/src/gpu_tiers.rs +++ b/crates/engine-gpu/src/gpu_tiers.rs @@ -41,35 +41,185 @@ impl CompiledGpuTier { // Use word boundaries (\b) to avoid substring issues like "550" matching "5500" const NVIDIA_TIERS: &[GpuTier] = &[ - // Blackwell (RTX 50 series) + // --- Workstation / datacenter first (avoid "rtx 40" / "rtx 50" substring traps) --- + // Blackwell PRO (e.g. "NVIDIA RTX PRO 6000 Blackwell Server Edition") GpuTier { - pattern: r"\b50[89]0\b", + pattern: r"rtx pro 6000|pro 6000 blackwell", + name: "NVIDIA RTX PRO 6000 (Blackwell)", + workgroup_divisor: 6, + min_workgroups: 5120, + }, + GpuTier { + pattern: r"rtx pro 5000|pro 5000 blackwell", + name: "NVIDIA RTX PRO 5000 (Blackwell)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"rtx pro 4500|pro 4500 blackwell", + name: "NVIDIA RTX PRO 4500 (Blackwell)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"rtx pro 4000|pro 4000 blackwell|rtx pro", + name: "NVIDIA RTX PRO (Blackwell)", + workgroup_divisor: 9, + min_workgroups: 3584, + }, + // Ada Lovelace workstation (must precede consumer `\brtx 50xx` / `\brtx 40xx`) + GpuTier { + pattern: r"rtx 6000 ada|6000 ada generation", + name: "NVIDIA RTX 6000 Ada (Workstation)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"rtx 5000 ada|5000 ada generation", + name: "NVIDIA RTX 5000 Ada (Workstation)", + workgroup_divisor: 9, + min_workgroups: 3584, + }, + GpuTier { + pattern: r"rtx 4000 ada|4000 ada generation|4000 sff ada", + name: "NVIDIA RTX 4000 Ada (Workstation)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx 2000 ada|2000 ada generation", + name: "NVIDIA RTX 2000 Ada (Workstation)", + workgroup_divisor: 14, + min_workgroups: 1536, + }, + // Datacenter — largest / newest first. L40S before L40 before L4. + GpuTier { + pattern: r"\bb300\b|\bb200\b", + name: "NVIDIA B200/B300 (Blackwell DC)", + workgroup_divisor: 6, + min_workgroups: 5120, + }, + GpuTier { + pattern: r"\bh200\b", + name: "NVIDIA H200 (Hopper)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"\bh100\b", + name: "NVIDIA H100 (Hopper)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"\ba100\b", + name: "NVIDIA A100 (Ampere DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\bl40s\b", + name: "NVIDIA L40S (Ada DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\bl40\b", + name: "NVIDIA L40 (Ada DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\ba40\b", + name: "NVIDIA A40 (Ampere DC)", + workgroup_divisor: 10, + min_workgroups: 3072, + }, + GpuTier { + pattern: r"\bl4\b", + name: "NVIDIA L4 (Ada DC)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"\ba30\b", + name: "NVIDIA A30 (Ampere DC)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + // Ampere workstation (RTX Axxxx) — split by class; was one shared pro bucket + GpuTier { + pattern: r"rtx a6000|\ba6000\b", + name: "NVIDIA RTX A6000 (Ampere Pro)", + workgroup_divisor: 10, + min_workgroups: 3072, + }, + GpuTier { + pattern: r"rtx a5000|\ba5000\b", + name: "NVIDIA RTX A5000 (Ampere Pro)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx a4500|\ba4500\b", + name: "NVIDIA RTX A4500 (Ampere Pro)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx a4000|\ba4000\b", + name: "NVIDIA RTX A4000 (Ampere Pro)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + GpuTier { + pattern: r"rtx a2000|\ba2000\b", + name: "NVIDIA RTX A2000 (Ampere Pro)", + workgroup_divisor: 16, + min_workgroups: 1024, + }, + GpuTier { + pattern: r"rtx a\d{4}", + name: "NVIDIA RTX A-series (Ampere Pro)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + GpuTier { + pattern: r"tesla|\bv100\b|quadro", + name: "NVIDIA Tesla/Quadro (Legacy Pro)", + workgroup_divisor: 14, + min_workgroups: 1536, + }, + // --- GeForce consumer --- + // Blackwell (RTX 50 series). Do NOT use bare "rtx 50" (matches "RTX 5000 Ada"). + GpuTier { + pattern: r"\b50[89]0\b|\brtx 50[89]0\b", name: "NVIDIA RTX 50 Flagship (Blackwell)", workgroup_divisor: 6, min_workgroups: 5120, }, GpuTier { - pattern: r"\b50[67]0\b|rtx 50", + pattern: r"\b50[67]0\b|\brtx 50[67]0\b", name: "NVIDIA RTX 50 (Blackwell)", workgroup_divisor: 7, min_workgroups: 4608, }, - // Ada Lovelace (RTX 40 series) + // Ada Lovelace (RTX 40 series). Do NOT use bare "rtx 40" (matches "RTX 4000 Ada"). GpuTier { - pattern: r"\b40[89]0\b", + pattern: r"\b40[89]0\b|\brtx 40[89]0\b", name: "NVIDIA RTX 40 Flagship (Ada)", workgroup_divisor: 8, min_workgroups: 4096, }, GpuTier { - pattern: r"\b40[67]0\b|rtx 40", + pattern: r"\b40[67]0\b|\brtx 40[67]0\b", name: "NVIDIA RTX 40 (Ada)", workgroup_divisor: 10, min_workgroups: 3072, }, // Ampere/Turing (RTX 30/20 series) GpuTier { - pattern: r"\b30[5-9]0\b|\b20[6-8]0\b|rtx 30|rtx 20", + pattern: r"\b30[5-9]0\b|\b20[6-8]0\b|\brtx 30[5-9]0\b|\brtx 20[6-8]0\b", name: "NVIDIA RTX 30/20 (Ampere/Turing)", workgroup_divisor: 12, min_workgroups: 2048, @@ -121,13 +271,6 @@ const NVIDIA_TIERS: &[GpuTier] = &[ workgroup_divisor: 28, min_workgroups: 256, }, - // Professional - GpuTier { - pattern: r"quadro|rtx a\d|tesla|\ba100\b|\bh100\b|\bl4\b", - name: "NVIDIA Quadro/Professional", - workgroup_divisor: 10, - min_workgroups: 2560, - }, ]; const AMD_TIERS: &[GpuTier] = &[ @@ -648,6 +791,63 @@ mod tests { assert_eq!(tier.name, "NVIDIA GTX 16/10 (Turing/Pascal)"); } + #[test] + fn test_nvidia_ada_workstation_not_confused_with_geforce() { + // Must not match bare "rtx 40" / "rtx 50" GeForce tiers. + let tier = detect_gpu_tier("NVIDIA RTX 4000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 4000 Ada (Workstation)"); + assert!(!tier.is_fallback); + + let tier = detect_gpu_tier("NVIDIA RTX 5000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 5000 Ada (Workstation)"); + assert!(!tier.is_fallback); + + let tier = detect_gpu_tier("NVIDIA RTX 6000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 6000 Ada (Workstation)"); + assert!(!tier.is_fallback); + } + + #[test] + fn test_nvidia_ampere_pro_split_by_class() { + let a2000 = detect_gpu_tier("NVIDIA RTX A2000", 0x10DE, false); + assert_eq!(a2000.name, "NVIDIA RTX A2000 (Ampere Pro)"); + assert_eq!(a2000.workgroup_divisor, 16); + + let a4500 = detect_gpu_tier("NVIDIA RTX A4500", 0x10DE, false); + assert_eq!(a4500.name, "NVIDIA RTX A4500 (Ampere Pro)"); + assert_eq!(a4500.workgroup_divisor, 11); + + let a6000 = detect_gpu_tier("NVIDIA RTX A6000", 0x10DE, false); + assert_eq!(a6000.name, "NVIDIA RTX A6000 (Ampere Pro)"); + assert_eq!(a6000.workgroup_divisor, 10); + + // Smaller cards should use a more conservative (higher) divisor. + assert!(a2000.workgroup_divisor > a4500.workgroup_divisor); + assert!(a4500.workgroup_divisor >= a6000.workgroup_divisor); + } + + #[test] + fn test_nvidia_datacenter_l40_before_l4() { + let l4 = detect_gpu_tier("NVIDIA L4", 0x10DE, false); + assert_eq!(l4.name, "NVIDIA L4 (Ada DC)"); + + let l40 = detect_gpu_tier("NVIDIA L40", 0x10DE, false); + assert_eq!(l40.name, "NVIDIA L40 (Ada DC)"); + + let l40s = detect_gpu_tier("NVIDIA L40S", 0x10DE, false); + assert_eq!(l40s.name, "NVIDIA L40S (Ada DC)"); + + let h100 = detect_gpu_tier("NVIDIA H100 80GB HBM3", 0x10DE, false); + assert_eq!(h100.name, "NVIDIA H100 (Hopper)"); + + let pro = detect_gpu_tier( + "NVIDIA RTX PRO 6000 Blackwell Server Edition", + 0x10DE, + false, + ); + assert_eq!(pro.name, "NVIDIA RTX PRO 6000 (Blackwell)"); + } + #[test] fn test_amd_rdna_vs_polaris() { // RDNA 1 - should NOT match Polaris diff --git a/gpu-bench/README.md b/gpu-bench/README.md index ca198f4..4b00bd1 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -3,15 +3,21 @@ Spin up a Quantus `--dev` node + GPU miner on a rented NVIDIA host, scrape Prometheus hashrate into [`results.csv`](results.csv), and compare hardware. -## Release binaries +## Miner binary (git build by default) -The Linux miner release includes WGPU (same as a normal `cargo build -p miner-cli --release`): +`remote-run.sh` / the RunPod sweep **clone + `cargo build -p miner-cli --release`** +from a git branch (default `illuzen/gpu-bench`) so you can iterate without +cutting a GitHub release. Push the branch before sweeping. -`https://github.com/Quantus-Network/quantus-miner/releases/download/v3.3.1/quantus-miner-linux-x86_64` +```bash +export MINER_BRANCH=illuzen/gpu-bench # default +# escape hatch: +# export MINER_SOURCE=release +``` -`remote-run.sh` / the RunPod sweep download that plus the latest -`quantus-node` linux x86_64 tarball from -[chain releases](https://github.com/Quantus-Network/chain/releases/latest). +`quantus-node` still comes from the latest +[chain release](https://github.com/Quantus-Network/chain/releases/latest) +linux x86_64 tarball. Container disk defaults to **50GB** for cargo `target/`. ## Same-host (manual) @@ -51,7 +57,7 @@ drivers are on the host; the image just needs CUDA userspace + RunPod’s 1. [Templates → New Template](https://www.console.runpod.io/user/templates) 2. Image: e.g. `runpod/base:1.1.0-cuda1281-ubuntu2404` (use **24.04** — node needs GLIBC ≥ 2.38) 3. Env: `NVIDIA_DRIVER_CAPABILITIES=all` (required for Vulkan/WGPU; compute-only → llvmpipe only) -4. Container disk ~20GB (keep small on Community — large disks often 500), volume 0 unless you need persistence +4. Container disk ~50GB if building miner from git (cargo `target/`); volume 0 unless you need persistence 5. Expose **TCP 22** (SSH). `runpod/base` / `runpod/pytorch` already start `sshd`. 6. Save → copy the template id → `export TEMPLATE_ID=...` @@ -65,7 +71,7 @@ curl -sS -X POST https://rest.runpod.io/v1/templates \ "name": "quantus-gpu-bench", "imageName": "runpod/base:1.1.0-cuda1281-ubuntu2404", "category": "NVIDIA", - "containerDiskInGb": 20, + "containerDiskInGb": 50, "volumeInGb": 0, "volumeMountPath": "/workspace", "ports": ["22/tcp", "9900/http"], @@ -143,7 +149,8 @@ sweep or `record.sh` run, commit any new rows you want others to see. | `hashrate` | avg `miner_gpu_hash_rate` from `:9900/metrics` | | `gpu_utilization_pct` | avg `utilization.gpu` | | `cost_per_hour` | Pod `costPerHr` from API (sweep) or your flag | -| `efficiency` | `hashrate / cost_per_hour` | +| `cost_per_sec` | `cost_per_hour / 3600` | +| `hash_per_dollar` | `hashrate / cost_per_sec` (hashes per $) | ## Scripts diff --git a/gpu-bench/gpus.all.txt b/gpu-bench/gpus.all.txt index aafd337..eebb01a 100644 --- a/gpu-bench/gpus.all.txt +++ b/gpu-bench/gpus.all.txt @@ -1,5 +1,6 @@ -# Full RunPod NVIDIA gpuTypeId list (AMD omitted — WGPU Vulkan path is NVIDIA). -# Source: https://docs.runpod.io/references/gpu-types +# RunPod NVIDIA gpuTypeId list (must match REST POST /pods enum). +# AMD omitted — WGPU Vulkan path is NVIDIA. +# Refresh from API errors or https://docs.runpod.io/references/gpu-types # Usage: ./runpod-sweep.sh --gpus-file gpus.all.txt # Consumer / GeForce @@ -26,7 +27,9 @@ NVIDIA RTX 4000 Ada Generation NVIDIA RTX 4000 SFF Ada Generation NVIDIA RTX 5000 Ada Generation NVIDIA RTX 6000 Ada Generation +NVIDIA RTX PRO 4000 Blackwell NVIDIA RTX PRO 4500 Blackwell +NVIDIA RTX PRO 5000 Blackwell NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition NVIDIA RTX PRO 6000 Blackwell Server Edition NVIDIA RTX PRO 6000 Blackwell Workstation Edition @@ -35,9 +38,9 @@ NVIDIA RTX PRO 6000 Blackwell Workstation Edition NVIDIA L4 NVIDIA L40 NVIDIA L40S -NVIDIA A30 NVIDIA A40 NVIDIA A100 80GB PCIe +NVIDIA A100-SXM4-40GB NVIDIA A100-SXM4-80GB NVIDIA H100 PCIe NVIDIA H100 NVL @@ -46,6 +49,6 @@ NVIDIA H200 NVIDIA H200 NVL NVIDIA B200 NVIDIA B300 SXM6 AC +NVIDIA B300 SXM6 AC MIG 1g.34gb Tesla V100-PCIE-16GB Tesla V100-SXM2-16GB -Tesla V100-SXM2-32GB diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh index 700991c..1cb39ab 100755 --- a/gpu-bench/record.sh +++ b/gpu-bench/record.sh @@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RUN_DIR="${SCRIPT_DIR}/.run" ENV_FILE="${SCRIPT_DIR}/.env" RESULTS_CSV="${SCRIPT_DIR}/results.csv" -CSV_HEADER="timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes" +CSV_HEADER="timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes" DEFAULT_MINER_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" MODE="live" @@ -127,12 +127,13 @@ query_sm_count() { *"RTX 4000 Ada"*|*"RTX 4000 SFF Ada"*) echo 48 ;; *"RTX 5000 Ada"*) echo 100 ;; *"RTX 6000 Ada"*) echo 142 ;; + *"RTX PRO 4000"*) echo 48 ;; *"RTX PRO 4500"*) echo 80 ;; + *"RTX PRO 5000"*) echo 140 ;; *"RTX PRO 6000"*) echo 188 ;; *"NVIDIA L4"|*" L4") echo 60 ;; *"L40S"*) echo 142 ;; *"L40"*) echo 142 ;; - *"NVIDIA A30"|*" A30") echo 56 ;; *"NVIDIA A40"|*" A40") echo 84 ;; *"A100"*) echo 108 ;; *"H100 NVL"*) echo 132 ;; @@ -275,14 +276,29 @@ prompt_if_empty() { printf -v "${var_name}" '%s' "${value}" } +# From cost_per_hour ($/hr) and hashrate (H/s): +# cost_per_sec = cost_per_hour / 3600 +# hash_per_dollar = hashrate / cost_per_sec (hashes per $) +compute_cost_metrics() { + local hashrate="$1" + local cost_per_hour="$2" + awk -v h="${hashrate}" -v c="${cost_per_hour}" 'BEGIN { + if (c+0 <= 0) { print ""; print ""; exit } + cps = c / 3600 + hpd = h / cps + printf "%.10f\n%.6f\n", cps, hpd + }' +} + emit_row() { local timestamp="$1" local hashrate="$2" local util_avg="$3" - local efficiency="$4" + local cost_per_sec="$4" + local hash_per_dollar="$5" local row - row="$(csv_escape "${timestamp}"),$(csv_escape "${PROVIDER}"),$(csv_escape "${GPU_MODEL}"),$(csv_escape "${VRAM_MB}"),$(csv_escape "${SM_COUNT}"),$(csv_escape "${DRIVER_VERSION}"),$(csv_escape "${hashrate}"),$(csv_escape "${util_avg}"),$(csv_escape "${COST_PER_HOUR}"),$(csv_escape "${efficiency}"),$(csv_escape "${DURATION}"),$(csv_escape "${NOTES}")" + row="$(csv_escape "${timestamp}"),$(csv_escape "${PROVIDER}"),$(csv_escape "${GPU_MODEL}"),$(csv_escape "${VRAM_MB}"),$(csv_escape "${SM_COUNT}"),$(csv_escape "${DRIVER_VERSION}"),$(csv_escape "${hashrate}"),$(csv_escape "${util_avg}"),$(csv_escape "${COST_PER_HOUR}"),$(csv_escape "${cost_per_sec}"),$(csv_escape "${hash_per_dollar}"),$(csv_escape "${DURATION}"),$(csv_escape "${NOTES}")" echo "${CSV_HEADER}" echo "${row}" @@ -343,14 +359,15 @@ run_live() { fi hashrate="$(awk -v h="${hashrate}" 'BEGIN { printf "%.6f", h }')" - local efficiency - efficiency="$(awk -v h="${hashrate}" -v c="${COST_PER_HOUR}" 'BEGIN { - if (c+0 > 0) printf "%.6f", h / c - }')" + local cost_per_sec hash_per_dollar + { + read -r cost_per_sec + read -r hash_per_dollar + } < <(compute_cost_metrics "${hashrate}" "${COST_PER_HOUR}") local ts ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" - emit_row "${ts}" "${hashrate}" "${util_avg}" "${efficiency}" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" } run_benchmark() { @@ -397,14 +414,15 @@ run_benchmark() { util_avg="$(awk -v u="${util_avg}" 'BEGIN { printf "%.2f", u }')" fi - local efficiency - efficiency="$(awk -v h="${hashrate}" -v c="${COST_PER_HOUR}" 'BEGIN { - if (c+0 > 0) printf "%.6f", h / c - }')" + local cost_per_sec hash_per_dollar + { + read -r cost_per_sec + read -r hash_per_dollar + } < <(compute_cost_metrics "${hashrate}" "${COST_PER_HOUR}") local ts ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" - emit_row "${ts}" "${hashrate}" "${util_avg}" "${efficiency}" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" } # --- args --- diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh index 0e5422c..bb76917 100755 --- a/gpu-bench/remote-run.sh +++ b/gpu-bench/remote-run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run on a GPU host (e.g. RunPod): fetch release binaries, start --dev node + -# GPU miner, sample Prometheus into results.csv. +# Run on a GPU host (e.g. RunPod): fetch node release + build miner from git, +# start --dev node + GPU miner, sample Prometheus into results.csv. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -9,8 +9,14 @@ BIN_DIR="${WORK_DIR}/bin" RUN_DIR="${WORK_DIR}/.run" RESULTS_CSV="${WORK_DIR}/results.csv" +# Miner: default builds from git (iterate without re-releasing). +# MINER_SOURCE=release still downloads a GitHub release binary. +MINER_SOURCE="${MINER_SOURCE:-git}" +MINER_REPO="${MINER_REPO:-https://github.com/Quantus-Network/quantus-miner.git}" +MINER_BRANCH="${MINER_BRANCH:-illuzen/gpu-bench}" MINER_VERSION="${MINER_VERSION:-v3.3.1}" MINER_URL="${MINER_URL:-https://github.com/Quantus-Network/quantus-miner/releases/download/${MINER_VERSION}/quantus-miner-linux-x86_64}" +FORCE_MINER_BUILD="${FORCE_MINER_BUILD:-0}" NODE_URL="${NODE_URL:-}" # optional override; default = latest chain release PROVIDER="${PROVIDER:-runpod}" @@ -27,17 +33,23 @@ usage() { Usage: ./remote-run.sh [options] --provider NAME cloud_provider column (default: runpod) - --cost-per-hour USD required for efficiency column + --cost-per-hour USD required for hash_per_dollar column --duration SECONDS Prometheus sample window (default: 60) --gpu-devices N default 1 --warmup SECONDS wait after miner start before sampling (default: 30) --notes TEXT - --miner-url URL override miner binary URL + --miner-branch REF git branch/tag/commit to build (default: illuzen/gpu-bench) + --miner-repo URL git remote (default: Quantus-Network/quantus-miner) + --miner-source git|release + git = clone+cargo build (default); release = binary URL + --miner-url URL release binary URL (with --miner-source release) + --force-miner-build rebuild even if binary exists --help -Downloads linux x86_64 release binaries, runs quantus-node --dev with ---miner-listen-port, runs quantus-miner against 127.0.0.1, then samples -:9900/metrics into results.csv (via record.sh --live). +Builds quantus-miner from git (or downloads a release), downloads quantus-node, +runs --dev + GPU miner, samples :9900/metrics into results.csv. + +Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, NODE_URL EOF } @@ -56,7 +68,11 @@ while [[ $# -gt 0 ]]; do --gpu-devices) GPU_DEVICES="${2:-}"; shift 2 ;; --warmup) WARMUP_SECONDS="${2:-}"; shift 2 ;; --notes) NOTES="${2:-}"; shift 2 ;; - --miner-url) MINER_URL="${2:-}"; shift 2 ;; + --miner-branch) MINER_BRANCH="${2:-}"; shift 2 ;; + --miner-repo) MINER_REPO="${2:-}"; shift 2 ;; + --miner-source) MINER_SOURCE="${2:-}"; shift 2 ;; + --miner-url) MINER_URL="${2:-}"; MINER_SOURCE="release"; shift 2 ;; + --force-miner-build) FORCE_MINER_BUILD=1; shift ;; --node-url) NODE_URL="${2:-}"; shift 2 ;; -h | --help) usage; exit 0 ;; *) @@ -287,19 +303,113 @@ EOF fi } -download_miner() { +ensure_rust() { + if command -v cargo >/dev/null 2>&1; then + return 0 + fi + echo "Installing Rust toolchain (rustup) ..." >&2 + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + # shellcheck disable=SC1091 + source "${HOME}/.cargo/env" + if ! command -v cargo >/dev/null 2>&1; then + echo "error: cargo not available after rustup install" >&2 + exit 1 + fi +} + +ensure_build_deps() { + if ! command -v apt-get >/dev/null 2>&1; then + return 0 + fi + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/dev/null 2>&1 || true + apt-get install -y -qq \ + git curl ca-certificates build-essential pkg-config libssl-dev \ + >/dev/null 2>&1 || \ + apt-get install -y -qq git curl build-essential pkg-config libssl-dev || true +} + +download_miner_release() { local dest="${BIN_DIR}/quantus-miner" - if [[ -x "${dest}" ]]; then + if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" ]]; then echo "Using existing ${dest}" >&2 echo "${dest}" return fi - echo "Downloading miner: ${MINER_URL}" >&2 + echo "Downloading miner release: ${MINER_URL}" >&2 curl -fL "${MINER_URL}" -o "${dest}" chmod +x "${dest}" echo "${dest}" } +# Clone MINER_BRANCH from MINER_REPO and cargo build -p miner-cli --release. +build_miner_from_git() { + local dest="${BIN_DIR}/quantus-miner" + local src="${WORK_DIR}/quantus-miner-src" + local rev_file="${BIN_DIR}/quantus-miner.rev" + local built_rev="" + + ensure_build_deps + require_cmd git + ensure_rust + # shellcheck disable=SC1091 + [[ -f "${HOME}/.cargo/env" ]] && source "${HOME}/.cargo/env" + + echo "Miner source: ${MINER_REPO} @ ${MINER_BRANCH}" >&2 + if [[ -d "${src}/.git" ]]; then + git -C "${src}" remote set-url origin "${MINER_REPO}" + git -C "${src}" fetch --depth 1 origin "${MINER_BRANCH}" + git -C "${src}" checkout -f FETCH_HEAD + else + rm -rf "${src}" + if ! git clone --depth 1 --branch "${MINER_BRANCH}" "${MINER_REPO}" "${src}"; then + echo "error: git clone failed for ${MINER_REPO} branch ${MINER_BRANCH}" >&2 + echo "Push the branch to origin, or set MINER_BRANCH / MINER_REPO." >&2 + exit 1 + fi + fi + + built_rev="$(git -C "${src}" rev-parse HEAD)" + if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" && -f "${rev_file}" ]]; then + if [[ "$(cat "${rev_file}")" == "${built_rev}" ]]; then + echo "Using existing ${dest} (rev ${built_rev})" >&2 + echo "${dest}" + return + fi + fi + + echo "Building quantus-miner (release, rev ${built_rev}) ..." >&2 + ( + cd "${src}" + cargo build -p miner-cli --release + ) + if [[ ! -x "${src}/target/release/quantus-miner" ]]; then + echo "error: cargo build did not produce target/release/quantus-miner" >&2 + exit 1 + fi + mkdir -p "${BIN_DIR}" + cp -f "${src}/target/release/quantus-miner" "${dest}" + chmod +x "${dest}" + echo "${built_rev}" >"${rev_file}" + echo "Built ${dest}" >&2 + echo "${dest}" +} + +download_miner() { + case "${MINER_SOURCE}" in + git | build | source) + build_miner_from_git + ;; + release | binary) + download_miner_release + ;; + *) + echo "error: unknown MINER_SOURCE=${MINER_SOURCE} (use git or release)" >&2 + exit 1 + ;; + esac +} + download_node() { local dest="${BIN_DIR}/quantus-node" if [[ -x "${dest}" ]]; then diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv index 238b887..2ac4f97 100644 --- a/gpu-bench/results.csv +++ b/gpu-bench/results.csv @@ -1,7 +1,7 @@ -timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes -2026-08-01T08:16:22Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,7237290.633333,71.30,0.18,40207170.185183,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=4exbkxuw781hte;ssh=direct -2026-08-01T08:18:38Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.126.09,6613452.233333,61.40,0.22,30061146.515150,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=w40modnpa8t6na;ssh=direct -2026-08-01T08:21:18Z,runpod,NVIDIA GeForce RTX 3090 Ti,24564,84,580.65.06,7323359.689655,73.28,0.27,27123554.406130,60,gpuTypeId=NVIDIA GeForce RTX 3090 Ti;pod=lflvz4kxqkxsol;ssh=direct -2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,26346855.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct -2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,30352894.210526,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct -2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,39002828.965515,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct +timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes +2026-08-01T08:16:22Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,7237290.633333,71.30,0.18,0.0000500000,144745812666.660034,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=4exbkxuw781hte;ssh=direct +2026-08-01T08:18:38Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.126.09,6613452.233333,61.40,0.22,0.0000611111,108220127454.540009,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=w40modnpa8t6na;ssh=direct +2026-08-01T08:21:18Z,runpod,NVIDIA GeForce RTX 3090 Ti,24564,84,580.65.06,7323359.689655,73.28,0.27,0.0000750000,97644795862.066666,60,gpuTypeId=NVIDIA GeForce RTX 3090 Ti;pod=lflvz4kxqkxsol;ssh=direct +2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,0.0000333333,94848678000.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct +2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,0.0000527778,109270419157.894745,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct +2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,0.0000555556,140410184275.854004,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh index 73e14bf..24992b0 100755 --- a/gpu-bench/runpod-shell.sh +++ b/gpu-bench/runpod-shell.sh @@ -74,8 +74,9 @@ print_ssh_hint() { echo "On the pod:" cat <&2 + echo "tip: refresh gpu-bench/gpus.all.txt from the RunPod /pods enum" >&2 + rm -f "${body_file}" + return 2 + fi if [[ "${volume}" -gt 0 ]]; then volume=0 elif [[ "${disk}" -gt 15 ]]; then @@ -534,13 +555,21 @@ run_one_attempt() { set +e ssh_cmd "${mode}" "${host}" "${port}" \ "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' && \ - WORK_DIR='${REMOTE_DIR}' '${REMOTE_DIR}/remote-run.sh' \ + WORK_DIR='${REMOTE_DIR}' \ + MINER_SOURCE='${MINER_SOURCE}' \ + MINER_REPO='${MINER_REPO}' \ + MINER_BRANCH='${MINER_BRANCH}' \ + FORCE_MINER_BUILD='${FORCE_MINER_BUILD:-0}' \ + '${REMOTE_DIR}/remote-run.sh' \ --provider runpod \ --cost-per-hour '${cost}' \ --duration '${DURATION}' \ --warmup '${WARMUP_SECONDS}' \ --gpu-devices '${GPU_DEVICES}' \ - --notes 'gpuTypeId=${gpu_type};pod=${pod_id};ssh=${mode}'" + --miner-source '${MINER_SOURCE}' \ + --miner-repo '${MINER_REPO}' \ + --miner-branch '${MINER_BRANCH}' \ + --notes 'gpuTypeId=${gpu_type};pod=${pod_id};ssh=${mode};miner=${MINER_SOURCE}@${MINER_BRANCH}'" remote_rc=$? set -e @@ -636,7 +665,7 @@ runpod_sweep_main() { mkdir -p "${OUT_DIR}" if [[ ! -f "${RESULTS_CSV}" ]]; then printf '%s\n' \ - "timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,efficiency,sample_seconds,notes" \ + "timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes" \ >"${RESULTS_CSV}" fi From a1434d8edd45be794347db9fa05f1dc86b17fd94 Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 18:27:59 +0800 Subject: [PATCH 08/13] Fix Vulkan setup when apt GL packages mismatch the host driver. Verify with vulkaninfo and fall back to extracting the exact NVIDIA .run userspace so compute-only Community hosts can initialize WGPU. Co-authored-by: Cursor --- gpu-bench/remote-run.sh | 177 ++++++++++++++++++++++------------------ 1 file changed, 97 insertions(+), 80 deletions(-) diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh index bb76917..4d2a2e8 100755 --- a/gpu-bench/remote-run.sh +++ b/gpu-bench/remote-run.sh @@ -113,13 +113,15 @@ fi EXIT_COMPUTE_ONLY=42 find_libglx_nvidia() { + # Prefer exact .run extract over apt (apt major-version packages often + # fail vkCreateInstance against the host kernel module). local candidate for candidate in \ + "${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0" \ /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0 \ /usr/lib64/libGLX_nvidia.so.0 \ /usr/lib/libGLX_nvidia.so.0 \ - /usr/local/nvidia/lib64/libGLX_nvidia.so.0 \ - "${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0"; do + /usr/local/nvidia/lib64/libGLX_nvidia.so.0; do if [[ -e "${candidate}" ]]; then echo "${candidate}" return 0 @@ -128,41 +130,62 @@ find_libglx_nvidia() { ldconfig -p 2>/dev/null | awk '/libGLX_nvidia\.so\.0/ { print $NF; exit }' || true } -# Many Community hosts only mount compute libs. Install matching graphics -# userspace via apt, then (if needed) extract from the NVIDIA .run installer. -install_nvidia_gl_userspace() { - local ver major dest - ver="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \ - | head -n 1 | tr -d '[:space:]')" - if [[ -z "${ver}" ]]; then - return 1 - fi - major="${ver%%.*}" - dest="${WORK_DIR}/nvidia-gl" - mkdir -p "${dest}" +host_driver_version() { + nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \ + | head -n 1 | tr -d '[:space:]' +} - if command -v apt-get >/dev/null 2>&1; then - echo "Trying apt libnvidia-gl-${major} (host driver ${ver}) ..." >&2 - apt-get install -y -qq "libnvidia-gl-${major}" >/dev/null 2>&1 || true - if [[ -n "$(find_libglx_nvidia)" ]]; then - return 0 - fi - fi +vulkan_has_nvidia() { + command -v vulkaninfo >/dev/null 2>&1 || return 1 + vulkaninfo --summary 2>/dev/null | grep -qiE 'NVIDIA|GeForce|Tesla|Quadro|RTX|A100|L4|L40' +} - if [[ -e "${dest}/libGLX_nvidia.so.0" ]]; then +write_nvidia_icd() { + local lib="$1" + local icd_dir="/usr/share/vulkan/icd.d" + local icd="${icd_dir}/nvidia_icd.json" + mkdir -p "${icd_dir}" /etc/vulkan/icd.d + cat >"${icd}" </dev/null || true + export VK_ICD_FILENAMES="${icd}" + export NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" + export __GLX_VENDOR_LIBRARY_NAME=nvidia + local libdir + libdir="$(dirname "${lib}")" + export LD_LIBRARY_PATH="${libdir}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + echo "NVIDIA Vulkan ICD: ${icd} -> ${lib}" >&2 +} + +# Download + extract the exact host driver userspace (matches nvidia-smi version). +extract_nvidia_run_userspace() { + local ver="$1" + local dest="${WORK_DIR}/nvidia-gl" + mkdir -p "${dest}" + + if [[ -e "${dest}/libGLX_nvidia.so.0" && -f "${dest}/.driver_version" ]] \ + && [[ "$(cat "${dest}/.driver_version")" == "${ver}" ]]; then + echo "Using cached NVIDIA ${ver} userspace in ${dest}" >&2 export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" return 0 fi - echo "Extracting NVIDIA ${ver} userspace libs (compute-only host) ..." >&2 + echo "Extracting NVIDIA ${ver} userspace libs (exact match for host driver) ..." >&2 local tmp runfile url tmp="$(mktemp -d)" runfile="${tmp}/NVIDIA.run" - # GeForce/pro first; Tesla URL as fallback for datacenter hosts. for url in \ "https://download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ "https://us.download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ "https://us.download.nvidia.com/tesla/${ver}/NVIDIA-Linux-x86_64-${ver}.run"; do + echo " trying ${url}" >&2 if curl -fL --connect-timeout 20 --max-time 600 "${url}" -o "${runfile}"; then break fi @@ -175,7 +198,6 @@ install_nvidia_gl_userspace() { fi chmod +x "${runfile}" - # Extract only — do not install kernel modules into the container. if ! sh "${runfile}" --extract-only --target "${tmp}/extract" >/tmp/nvidia-extract.log 2>&1; then echo "error: NVIDIA .run extract failed; log:" >&2 tail -n 40 /tmp/nvidia-extract.log >&2 || true @@ -183,6 +205,8 @@ install_nvidia_gl_userspace() { return 1 fi + rm -rf "${dest}" + mkdir -p "${dest}" local f for f in \ libGLX_nvidia.so.* \ @@ -195,9 +219,8 @@ install_nvidia_gl_userspace() { libnvidia-rtcore.so.* \ libnvoptix.so.*; do # shellcheck disable=SC2086 - cp -n ${tmp}/extract/${f} "${dest}/" 2>/dev/null || true + cp -f ${tmp}/extract/${f} "${dest}/" 2>/dev/null || true done - # Stable SONAME links expected by the ICD. local so for so in libGLX_nvidia.so libEGL_nvidia.so; do if [[ ! -e "${dest}/${so}.0" ]]; then @@ -209,19 +232,13 @@ install_nvidia_gl_userspace() { fi done - if [[ -f "${tmp}/extract/nvidia_icd.json" ]]; then - mkdir -p /usr/share/vulkan/icd.d - # Rewrite library_path to our extracted lib if needed later. - cp -f "${tmp}/extract/nvidia_icd.json" /usr/share/vulkan/icd.d/nvidia_icd.json || true - fi - - echo "${dest}" >"${dest}/.ld_path" + echo "${ver}" >"${dest}/.driver_version" export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" echo "${dest}" >>/etc/ld.so.conf.d/nvidia-gl-bench.conf 2>/dev/null || true ldconfig 2>/dev/null || true - rm -rf "${tmp}" - if [[ -e "${dest}/libGLX_nvidia.so.0" ]] || [[ -n "$(find_libglx_nvidia)" ]]; then + + if [[ -e "${dest}/libGLX_nvidia.so.0" ]]; then echo "NVIDIA GL userspace ready in ${dest}" >&2 return 0 fi @@ -248,59 +265,59 @@ ensure_nvidia_vulkan() { chmod 700 /tmp/runtime-root 2>/dev/null || true export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" - local lib + local ver major lib + ver="$(host_driver_version)" + major="${ver%%.*}" + + # 1) Host-mounted lib (best case). lib="$(find_libglx_nvidia)" - if [[ -z "${lib}" || ! -e "${lib}" ]]; then - echo "libGLX_nvidia.so.0 not mounted (compute-only host); installing userspace ..." >&2 - if ! install_nvidia_gl_userspace; then - echo "error: NVIDIA Vulkan/GL library missing (libGLX_nvidia.so.0)." >&2 - echo "exit ${EXIT_COMPUTE_ONLY}: sweep should retry on another host" >&2 - exit "${EXIT_COMPUTE_ONLY}" + if [[ -n "${lib}" && -e "${lib}" ]]; then + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (host mount)" >&2 + return 0 fi + echo "Host/apt libGLX present but vulkaninfo failed; trying exact .run extract ..." >&2 + vulkaninfo --summary 2>&1 | tail -n 20 >&2 || true + else + echo "libGLX_nvidia.so.0 not mounted (compute-only host)" >&2 + fi + + # 2) apt major package — often wrong patch level vs host (e.g. 580.95.05). + if [[ -n "${major}" ]] && command -v apt-get >/dev/null 2>&1; then + echo "Trying apt libnvidia-gl-${major} (host driver ${ver}) ..." >&2 + apt-get install -y -qq "libnvidia-gl-${major}" >/dev/null 2>&1 || true lib="$(find_libglx_nvidia)" + if [[ -n "${lib}" && -e "${lib}" ]]; then + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (apt)" >&2 + return 0 + fi + echo "apt libnvidia-gl-${major} installed but vulkaninfo still fails (version skew?)" >&2 + fi fi - if [[ -z "${lib}" || ! -e "${lib}" ]]; then - echo "error: still no libGLX_nvidia.so.0 after userspace install" >&2 + # 3) Exact driver .run extract — required on many Community compute-only hosts. + if [[ -z "${ver}" ]]; then + echo "error: cannot determine NVIDIA driver version" >&2 exit "${EXIT_COMPUTE_ONLY}" fi - - # Prefer extracted dir on LD_LIBRARY_PATH when we installed locally. - if [[ -d "${WORK_DIR}/nvidia-gl" ]]; then - export LD_LIBRARY_PATH="${WORK_DIR}/nvidia-gl${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + if ! extract_nvidia_run_userspace "${ver}"; then + echo "error: failed to install matching NVIDIA GL userspace for ${ver}" >&2 + exit "${EXIT_COMPUTE_ONLY}" fi - - local icd_dir="/usr/share/vulkan/icd.d" - local icd="${icd_dir}/nvidia_icd.json" - mkdir -p "${icd_dir}" /etc/vulkan/icd.d - cat >"${icd}" </dev/null || true - echo "NVIDIA Vulkan ICD: ${icd} -> ${lib}" >&2 - - # Prefer NVIDIA ICD only (ignore any Mesa CPU ICDs that may already exist). - export VK_ICD_FILENAMES="${icd}" - export NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" - export __GLX_VENDOR_LIBRARY_NAME=nvidia - - if command -v vulkaninfo >/dev/null 2>&1; then - if ! vulkaninfo --summary 2>/dev/null | grep -qiE 'NVIDIA|GeForce|Tesla|Quadro|RTX|A100|L4|L40'; then - echo "error: vulkaninfo sees no NVIDIA GPU (only software/CPU adapters?)." >&2 - vulkaninfo --summary 2>&1 | tail -n 40 >&2 || true - echo "exit ${EXIT_COMPUTE_ONLY}: bad Vulkan host — sweep should retry" >&2 - exit "${EXIT_COMPUTE_ONLY}" - fi - echo "Vulkan NVIDIA adapter OK" >&2 - else - echo "warning: vulkan-tools not installed; skipping vulkaninfo check" >&2 + lib="${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0" + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (.run extract ${ver})" >&2 + return 0 fi + + echo "error: vulkaninfo still sees no NVIDIA GPU after .run extract." >&2 + vulkaninfo --summary 2>&1 | tail -n 40 >&2 || true + echo "exit ${EXIT_COMPUTE_ONLY}: bad Vulkan host — sweep should retry" >&2 + exit "${EXIT_COMPUTE_ONLY}" } ensure_rust() { From 74ae65975eac0462570f1b8d95e2b0d981e87bb2 Mon Sep 17 00:00:00 2001 From: illuzen Date: Sat, 1 Aug 2026 18:30:11 +0800 Subject: [PATCH 09/13] banner brokens --- gpu-bench/remote-run.sh | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh index 4d2a2e8..77e71d5 100755 --- a/gpu-bench/remote-run.sh +++ b/gpu-bench/remote-run.sh @@ -325,7 +325,9 @@ ensure_rust() { return 0 fi echo "Installing Rust toolchain (rustup) ..." >&2 - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + # rustup writes the welcome banner to stdout — must not leak into $(download_miner). + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain stable >&2 # shellcheck disable=SC1091 source "${HOME}/.cargo/env" if ! command -v cargo >/dev/null 2>&1; then @@ -350,13 +352,13 @@ download_miner_release() { local dest="${BIN_DIR}/quantus-miner" if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" ]]; then echo "Using existing ${dest}" >&2 - echo "${dest}" + printf '%s\n' "${dest}" return fi echo "Downloading miner release: ${MINER_URL}" >&2 curl -fL "${MINER_URL}" -o "${dest}" chmod +x "${dest}" - echo "${dest}" + printf '%s\n' "${dest}" } # Clone MINER_BRANCH from MINER_REPO and cargo build -p miner-cli --release. @@ -375,11 +377,11 @@ build_miner_from_git() { echo "Miner source: ${MINER_REPO} @ ${MINER_BRANCH}" >&2 if [[ -d "${src}/.git" ]]; then git -C "${src}" remote set-url origin "${MINER_REPO}" - git -C "${src}" fetch --depth 1 origin "${MINER_BRANCH}" - git -C "${src}" checkout -f FETCH_HEAD + git -C "${src}" fetch --depth 1 origin "${MINER_BRANCH}" >&2 + git -C "${src}" checkout -f FETCH_HEAD >&2 else rm -rf "${src}" - if ! git clone --depth 1 --branch "${MINER_BRANCH}" "${MINER_REPO}" "${src}"; then + if ! git clone --depth 1 --branch "${MINER_BRANCH}" "${MINER_REPO}" "${src}" >&2; then echo "error: git clone failed for ${MINER_REPO} branch ${MINER_BRANCH}" >&2 echo "Push the branch to origin, or set MINER_BRANCH / MINER_REPO." >&2 exit 1 @@ -390,7 +392,7 @@ build_miner_from_git() { if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" && -f "${rev_file}" ]]; then if [[ "$(cat "${rev_file}")" == "${built_rev}" ]]; then echo "Using existing ${dest} (rev ${built_rev})" >&2 - echo "${dest}" + printf '%s\n' "${dest}" return fi fi @@ -398,7 +400,7 @@ build_miner_from_git() { echo "Building quantus-miner (release, rev ${built_rev}) ..." >&2 ( cd "${src}" - cargo build -p miner-cli --release + cargo build -p miner-cli --release >&2 ) if [[ ! -x "${src}/target/release/quantus-miner" ]]; then echo "error: cargo build did not produce target/release/quantus-miner" >&2 @@ -409,7 +411,8 @@ build_miner_from_git() { chmod +x "${dest}" echo "${built_rev}" >"${rev_file}" echo "Built ${dest}" >&2 - echo "${dest}" + # Sole stdout line — consumed by MINER_BIN="$(download_miner)" + printf '%s\n' "${dest}" } download_miner() { @@ -431,7 +434,7 @@ download_node() { local dest="${BIN_DIR}/quantus-node" if [[ -x "${dest}" ]]; then echo "Using existing ${dest}" >&2 - echo "${dest}" + printf '%s\n' "${dest}" return fi @@ -456,7 +459,7 @@ download_node() { mv "${tmp}/quantus-node" "${dest}" chmod +x "${dest}" rm -rf "${tmp}" - echo "${dest}" + printf '%s\n' "${dest}" } stop_pidfile() { @@ -482,7 +485,11 @@ trap cleanup EXIT ensure_nvidia_vulkan NODE_BIN="$(download_node)" -MINER_BIN="$(download_miner)" +MINER_BIN="$(download_miner | tail -n 1)" +if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: miner binary missing or not executable: '${MINER_BIN}'" >&2 + exit 1 +fi # Fail fast with a clear hint if the host glibc is too old for the release binary. if ! "${NODE_BIN}" --version >/tmp/qnode-ver.txt 2>/tmp/qnode-ver.err; then From 7bfd6a8ea14c0b84dacd193b49e0c251f60a8d7f Mon Sep 17 00:00:00 2001 From: illuzen Date: Mon, 3 Aug 2026 11:36:46 +0800 Subject: [PATCH 10/13] pass thru batch sizes --- crates/miner-cli/src/main.rs | 6 +- gpu-bench/batch-tune.sh | 169 +++++++++++++++++++++++++++++++++++ gpu-bench/results.csv | 7 ++ gpu-bench/runpod-shell.sh | 4 + 4 files changed, 184 insertions(+), 2 deletions(-) create mode 100755 gpu-bench/batch-tune.sh diff --git a/crates/miner-cli/src/main.rs b/crates/miner-cli/src/main.rs index 8ab8568..c2e3696 100644 --- a/crates/miner-cli/src/main.rs +++ b/crates/miner-cli/src/main.rs @@ -257,8 +257,10 @@ async fn run_benchmark( let mut handles = Vec::new(); let total_hashes = Arc::new(std::sync::Mutex::new(0u64)); - let cpu_chunk = 10_000u64; - let gpu_chunk = 1_000_000u64; + // Range size must be >= engine batch size; otherwise search_range clamps + // this_batch_size to the range and --gpu-batch-size A/B is inert. + let cpu_chunk = cpu_batch_size.max(10_000); + let gpu_chunk = gpu_batch_size as u64; for worker_id in 0..total_workers { let (engine, nonces_per_batch) = if worker_id < effective_cpu_workers { diff --git a/gpu-bench/batch-tune.sh b/gpu-bench/batch-tune.sh new file mode 100755 index 0000000..771c76c --- /dev/null +++ b/gpu-bench/batch-tune.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# A/B GPU batch sizes on a pod (or any host with a built miner + NVIDIA GPU). +# Samples nvidia-smi util during each run and prints a comparison table. +# +# Usage (on pod, after miner is built): +# cd /workspace/quantus-gpu-bench +# ./batch-tune.sh +# ./batch-tune.sh --sizes "1000000 4194304 16777216" --duration 30 +# +# Or from laptop via runpod-shell (see comments at bottom of this file). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${WORK_DIR:-/workspace/quantus-gpu-bench}" +BIN_DIR="${BIN_DIR:-${WORK_DIR}/bin}" +OUT_CSV="${OUT_CSV:-${WORK_DIR}/batch-tune.csv}" +SIZES="${SIZES:-1000000 4194304 8388608 16777216}" +DURATION="${DURATION:-30}" +GPU_DEVICES="${GPU_DEVICES:-1}" +CPU_WORKERS="${CPU_WORKERS:-0}" +MINER_BIN="${MINER_BIN:-}" + +usage() { + cat <<'EOF' +Usage: ./batch-tune.sh [options] + + --sizes "N N N" batch sizes to try (default: 1M 4M 8M 16M) + --duration SECONDS per-size benchmark window (default: 30) + --gpu-devices N default 1 + --cpu-workers N default 0 (GPU-only) + --miner-bin PATH override miner binary + --out PATH results CSV (default: $WORK_DIR/batch-tune.csv) + --help + +Requires a miner built with the gpu_chunk=batch_size CLI fix (illuzen/gpu-bench). +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --sizes) SIZES="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --gpu-devices) GPU_DEVICES="$2"; shift 2 ;; + --cpu-workers) CPU_WORKERS="$2"; shift 2 ;; + --miner-bin) MINER_BIN="$2"; shift 2 ;; + --out) OUT_CSV="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "unknown arg: $1" >&2; usage; exit 1 ;; + esac +done + +resolve_miner() { + if [[ -n "${MINER_BIN}" && -x "${MINER_BIN}" ]]; then + echo "${MINER_BIN}" + return + fi + for cand in \ + "${BIN_DIR}/quantus-miner" \ + "${WORK_DIR}/quantus-miner/target/release/quantus-miner" \ + "${SCRIPT_DIR}/../target/release/quantus-miner" \ + "$(command -v quantus-miner 2>/dev/null || true)"; do + if [[ -n "${cand}" && -x "${cand}" ]]; then + echo "${cand}" + return + fi + done + echo "error: quantus-miner binary not found. Run remote-run.sh once, or set --miner-bin" >&2 + exit 1 +} + +sample_util() { + # Average GPU util % over ~DURATION seconds (1 Hz). + local secs="$1" + local sum=0 count=0 u + for ((i = 0; i < secs; i++)); do + u="$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -n1 | tr -d ' ' || echo 0)" + [[ "${u}" =~ ^[0-9]+$ ]] || u=0 + sum=$((sum + u)) + count=$((count + 1)) + sleep 1 + done + if [[ "${count}" -eq 0 ]]; then + echo 0 + else + echo $((sum / count)) + fi +} + +parse_rate_hs() { + # Parse "Average rate: 1.23M H/s" / "45.67K H/s" / "1234 H/s" from benchmark stdout. + local line unit num + line="$(grep -E 'Average rate:' "$1" | tail -n1 || true)" + [[ -n "${line}" ]] || { echo 0; return; } + num="$(echo "${line}" | sed -E 's/.*Average rate:[[:space:]]*([0-9.]+)([KMkm]?).*H\/s.*/\1/')" + unit="$(echo "${line}" | sed -E 's/.*Average rate:[[:space:]]*[0-9.]+([KMkm]?).*H\/s.*/\1/')" + case "${unit}" in + M|m) awk -v n="${num}" 'BEGIN { printf "%.0f", n * 1000000 }' ;; + K|k) awk -v n="${num}" 'BEGIN { printf "%.0f", n * 1000 }' ;; + *) awk -v n="${num}" 'BEGIN { printf "%.0f", n }' ;; + esac +} + +MINER_BIN="$(resolve_miner)" +GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -n1 | sed 's/,/;/g' || echo unknown)" +echo "miner: ${MINER_BIN}" +echo "gpu: ${GPU_NAME}" +echo "sizes: ${SIZES}" +echo "window: ${DURATION}s gpu_devices=${GPU_DEVICES} cpu_workers=${CPU_WORKERS}" +echo + +mkdir -p "$(dirname "${OUT_CSV}")" +if [[ ! -f "${OUT_CSV}" ]]; then + echo "timestamp,gpu_name,batch_size,duration_s,hashrate_hs,avg_util_pct,notes" >"${OUT_CSV}" +fi + +printf "%-12s %-14s %-10s\n" "batch_size" "hashrate" "avg_util%" +printf "%-12s %-14s %-10s\n" "----------" "--------" "---------" + +for bs in ${SIZES}; do + log="$(mktemp)" + util_log="$(mktemp)" + # Sample util in background while benchmark runs. + ( + # skip first ~3s of warmup + sleep 3 + sample_util $((DURATION > 5 ? DURATION - 3 : DURATION)) + ) >"${util_log}" & + util_pid=$! + + set +e + "${MINER_BIN}" benchmark \ + --gpu-devices "${GPU_DEVICES}" \ + --cpu-workers "${CPU_WORKERS}" \ + --gpu-batch-size "${bs}" \ + --duration "${DURATION}" \ + >"${log}" 2>&1 + rc=$? + set -e + + wait "${util_pid}" 2>/dev/null || true + util="$(cat "${util_log}" 2>/dev/null || echo 0)" + rate="$(parse_rate_hs "${log}")" + + if [[ "${rc}" -ne 0 || "${rate}" == "0" ]]; then + echo "--- failed batch_size=${bs} (rc=${rc}) ---" >&2 + tail -n 40 "${log}" >&2 || true + notes="FAILED" + rate_disp="FAIL" + else + notes="ok" + if (( rate >= 1000000 )); then + rate_disp="$(awk -v n="${rate}" 'BEGIN { printf "%.2fM" , n/1000000 }')" + elif (( rate >= 1000 )); then + rate_disp="$(awk -v n="${rate}" 'BEGIN { printf "%.2fK" , n/1000 }')" + else + rate_disp="${rate}" + fi + fi + + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "${ts},${GPU_NAME},${bs},${DURATION},${rate},${util},${notes}" >>"${OUT_CSV}" + printf "%-12s %-14s %-10s\n" "${bs}" "${rate_disp}" "${util}%" + + rm -f "${log}" "${util_log}" +done + +echo +echo "wrote ${OUT_CSV}" +echo "Best util/rate usually wins — if 16M ≈ 1M, dispatch shape is not the bottleneck." diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv index 2ac4f97..36820dd 100644 --- a/gpu-bench/results.csv +++ b/gpu-bench/results.csv @@ -5,3 +5,10 @@ timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_ 2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,0.0000333333,94848678000.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct 2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,0.0000527778,109270419157.894745,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct 2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,0.0000555556,140410184275.854004,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct +2026-08-01T10:39:32Z,runpod,NVIDIA GeForce RTX 3080,10240,68,580.95.05,6604543.833333,72.87,0.17,0.0000472222,139860928235.287048,60,gpuTypeId=NVIDIA GeForce RTX 3080;pod=l6v6j20c2bf0jr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:43:30Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,6196787.448276,63.76,0.18,0.0000500000,123935748965.520020,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=b0dzcmo8tarsno;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:47:53Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.65.06,6834865.566667,53.37,0.22,0.0000611111,111843254727.278183,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=tdmmxojjx85zbn;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T03:55:55Z,runpod,NVIDIA RTX A4500,20470,56,580.126.18,5587581.033333,67.63,0.19,0.0000527778,105869956421.046310,60,gpuTypeId=NVIDIA RTX A4500;pod=nk7hljiu43i8gc;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:02:04Z,runpod,NVIDIA RTX A5000,24564,64,550.144.03,6510972.633333,65.13,0.16,0.0000444444,146496884249.992493,60,gpuTypeId=NVIDIA RTX A5000;pod=sgeizrvavai3lt;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:09:46Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7669824.448276,53.41,0.2,0.0000555556,138056840068.967987,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=2o26qc0dczed6h;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:15:33Z,runpod,NVIDIA RTX 4000 SFF Ada Generation,20475,48,560.35.05,6308176.620690,62.55,0.18,0.0000500000,126163532413.800018,60,gpuTypeId=NVIDIA RTX 4000 SFF Ada Generation;pod=cxdke7crdl03qr;ssh=direct;miner=git@illuzen/gpu-bench diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh index 24992b0..309d6b6 100755 --- a/gpu-bench/runpod-shell.sh +++ b/gpu-bench/runpod-shell.sh @@ -78,6 +78,10 @@ print_ssh_hint() { bash -x ./remote-run.sh --cost-per-hour ${cost} --duration 30 --warmup 20 # FORCE_MINER_BUILD=1 ./remote-run.sh ... # after pushing new commits + # A/B gpu-batch-size (needs pushed CLI fix: range >= batch size): + # FORCE_MINER_BUILD=1 ./remote-run.sh --cost-per-hour ${cost} --duration 20 --warmup 15 + # ./batch-tune.sh --duration 30 + # Vulkan sanity: nvidia-smi ls -l /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so* 2>/dev/null || true From 4db7e86df3db5ba84df2d4e12099db8817b2bda1 Mon Sep 17 00:00:00 2001 From: illuzen Date: Mon, 3 Aug 2026 20:14:09 +0800 Subject: [PATCH 11/13] better benchmarks --- crates/miner-cli/src/main.rs | 39 ++++--- gpu-bench/README.md | 163 +++++++++++------------------ gpu-bench/batch-tune.sh | 7 +- gpu-bench/record.sh | 108 ++++++++++++++++--- gpu-bench/remote-run.sh | 197 +++++++++-------------------------- gpu-bench/runpod-shell.sh | 21 ++-- gpu-bench/runpod-sweep.sh | 15 ++- 7 files changed, 246 insertions(+), 304 deletions(-) diff --git a/crates/miner-cli/src/main.rs b/crates/miner-cli/src/main.rs index c2e3696..d2ece53 100644 --- a/crates/miner-cli/src/main.rs +++ b/crates/miner-cli/src/main.rs @@ -199,7 +199,13 @@ async fn run_benchmark( duration: u64, allow_integrated: bool, ) { - let effective_cpu_workers = cpu_workers.unwrap_or_else(num_cpus::get); + // When --gpu-devices is set and --cpu-workers is omitted, default to GPU-only + // so hardware A/B numbers aren't polluted by host CPU hashrate. + let effective_cpu_workers = match (cpu_workers, gpu_devices) { + (Some(n), _) => n, + (None, Some(_)) => 0, + (None, None) => num_cpus::get(), + }; // Initialize GPU engine (no throttle for benchmark) let (gpu_engine, effective_gpu_devices) = match miner_service::resolve_gpu_configuration( @@ -225,6 +231,8 @@ async fn run_benchmark( num_cpus::get() ); println!("GPU Devices: {}", effective_gpu_devices); + println!("GPU batch size: {} nonces", gpu_batch_size); + println!("CPU batch size: {} hashes", cpu_batch_size); println!("Duration: {} seconds", duration); println!(); @@ -246,7 +254,8 @@ async fn run_benchmark( // Random header hash for benchmark let mut header = [0u8; 32]; rand::rng().fill_bytes(&mut header); - let difficulty = U512::MAX; // High difficulty - no solutions expected + // Bitcoin-style: target = MAX / difficulty. MAX → target 1 → essentially never finds. + let difficulty = U512::MAX; let ref_engine = cpu_engine.as_ref().or(gpu_engine.as_ref()).unwrap(); let ctx = ref_engine.prepare_context(header, difficulty); @@ -257,8 +266,10 @@ async fn run_benchmark( let mut handles = Vec::new(); let total_hashes = Arc::new(std::sync::Mutex::new(0u64)); - // Range size must be >= engine batch size; otherwise search_range clamps - // this_batch_size to the range and --gpu-batch-size A/B is inert. + // Range per search_range call must be >= engine batch size; otherwise the GPU + // clamps this_batch_size to the range and --gpu-batch-size A/B is inert. + // One full batch per call also lets the wall-clock cancel + progress ticker arm + // between batches (cancel is only checked between GPU dispatches). let cpu_chunk = cpu_batch_size.max(10_000); let gpu_chunk = gpu_batch_size as u64; @@ -276,21 +287,22 @@ async fn run_benchmark( let handle = thread::spawn(move || { let stride = U512::from(1_000_000_000_000u64); - let worker_start = U512::from(worker_id as u64).saturating_mul(stride); - let worker_range = EngineRange { - start: worker_start, - end: worker_start - .saturating_add(U512::from(nonces_per_batch)) - .saturating_sub(U512::from(1u64)), - }; + let mut nonce = U512::from(worker_id as u64).saturating_mul(stride); loop { if cancel.load(std::sync::atomic::Ordering::Relaxed) { break; } + let worker_range = EngineRange { + start: nonce, + end: nonce + .saturating_add(U512::from(nonces_per_batch)) + .saturating_sub(U512::from(1u64)), + }; + let cancel_check = AtomicBoolCancelCheck(&cancel); - let result = engine.search_range(&ctx, worker_range.clone(), &cancel_check); + let result = engine.search_range(&ctx, worker_range, &cancel_check); match result { engine_cpu::EngineStatus::Found { hash_count, .. } @@ -302,11 +314,12 @@ async fn run_benchmark( engine_cpu::EngineStatus::Running { .. } => {} } - // Exit if device is lost if matches!(result, engine_cpu::EngineStatus::DeviceLost { .. }) { break; } + nonce = nonce.saturating_add(U512::from(nonces_per_batch)); + if start.elapsed() >= Duration::from_secs(duration) { break; } diff --git a/gpu-bench/README.md b/gpu-bench/README.md index 0cfcf69..543b1a6 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -1,15 +1,14 @@ # GPU miner bench (provider-agnostic) -Spin up a Quantus `--dev` node + GPU miner on any rented NVIDIA host (Clore, -RunPod, Vast.ai, etc.), scrape Prometheus hashrate into [`results.csv`](results.csv), -and compare hardware. You supply `--provider` and `--cost-per-hour` — the scripts -have no provider-specific logic (except the optional RunPod API sweep). +Rent an NVIDIA GPU, build `quantus-miner`, run **`benchmark` across batch sizes** +(no node), and append rows to [`results.csv`](results.csv) for hardware +comparison. You supply `--provider` and `--cost-per-hour`. ## Miner binary (git build by default) `remote-run.sh` / the RunPod sweep **clone + `cargo build -p miner-cli --release`** from a git branch (default `illuzen/gpu-bench`) so you can iterate without -cutting a GitHub release. Push the branch before sweeping. +cutting a GitHub release. **Push the branch before sweeping.** ```bash export MINER_BRANCH=illuzen/gpu-bench # default @@ -17,29 +16,27 @@ export MINER_BRANCH=illuzen/gpu-bench # default # export MINER_SOURCE=release ``` -`quantus-node` still comes from the latest -[chain release](https://github.com/Quantus-Network/chain/releases/latest) -linux x86_64 tarball. Container disk defaults to **50GB** for cargo `target/`. +Container disk defaults to **50GB** for cargo `target/`. No `quantus-node` +download — hardware benches use `quantus-miner benchmark` only. -## Same-host (manual) +## On-pod one-shot ```bash -cd gpu-bench -./setup.sh --dev # native --dev node + GPU miner (no rewards hash) -./record.sh --provider clore --cost-per-hour 0.35 # or runpod, vast, ... -./setup.sh stop +./remote-run.sh --provider runpod --cost-per-hour 0.69 --duration 30 +# default batch sizes: 1M 4M 8M 16M (one CSV row each, notes include batch=N) +./remote-run.sh --cost-per-hour 0.39 --batch-sizes "1000000 16777216" ``` -For Planck (non-dev) mining, set `REWARDS_INNER_HASH` via `./setup.sh wormhole` -and run `./setup.sh` without `--dev`. - -On-pod one-shot (builds miner from git + downloads node): +Or call record directly if the miner is already built: ```bash -./remote-run.sh --provider runpod --cost-per-hour 0.69 --duration 60 +MINER_BIN=./bin/quantus-miner ./record.sh --benchmark \ + --provider runpod --cost-per-hour 0.42 \ + --batch-sizes "1000000 4194304 16777216" --duration 30 ``` -Hardware-only (no node): `./record.sh --benchmark --provider runpod --cost-per-hour 0.42` +Local full stack (node + serve) is still available via `./setup.sh --dev` + +`./record.sh --live` when you need end-to-end mining, not just hardware H/s. ## RunPod API sweep @@ -51,127 +48,89 @@ Uses the **REST API** (not MCP): create Pod → SSH → `remote-run.sh` → scp - SSH public key added in RunPod **Settings → SSH Public Keys** - `curl`, `ssh`, `scp`, `python3` -### 2. How to make a RunPod template (optional but nice) - -You don’t *need* a custom template — the sweep defaults to a slim RunPod CUDA -base image (`runpod/base:…-cuda…`) with port `22/tcp`. **Don’t use PyTorch** -images unless you need Torch; they’re much larger and unused here. NVIDIA -drivers are on the host; the image just needs CUDA userspace + RunPod’s -`/start.sh` (SSH). - -**Console** - -1. [Templates → New Template](https://www.console.runpod.io/user/templates) -2. Image: e.g. `runpod/base:1.1.0-cuda1281-ubuntu2404` (use **24.04** — node needs GLIBC ≥ 2.38) -3. Env: `NVIDIA_DRIVER_CAPABILITIES=all` (required for Vulkan/WGPU; compute-only → llvmpipe only) -4. Container disk ~50GB if building miner from git (cargo `target/`); volume 0 unless you need persistence -5. Expose **TCP 22** (SSH). `runpod/base` / `runpod/pytorch` already start `sshd`. -6. Save → copy the template id → `export TEMPLATE_ID=...` - -**API** +### 2. Image / template tips -```bash -curl -sS -X POST https://rest.runpod.io/v1/templates \ - -H "Authorization: Bearer $RUNPOD_API_KEY" \ - -H "Content-Type: application/json" \ - -d '{ - "name": "quantus-gpu-bench", - "imageName": "runpod/base:1.1.0-cuda1281-ubuntu2404", - "category": "NVIDIA", - "containerDiskInGb": 50, - "volumeInGb": 0, - "volumeMountPath": "/workspace", - "ports": ["22/tcp", "9900/http"], - "env": { "NVIDIA_DRIVER_CAPABILITIES": "all" }, - "isPublic": false - }' -``` - -### 3. Debug one Pod interactively (recommended first) +Default image: `runpod/base:1.1.0-cuda1281-ubuntu2404` with +`NVIDIA_DRIVER_CAPABILITIES=all` (Vulkan/WGPU). Container disk ~50GB for git +builds. Expose TCP 22 for SSH. -Creates a Pod, uploads scripts, prints SSH, and **leaves it running** so you -can fix Vulkan / miner commands without pay-per-recreate: +### 3. Debug one Pod interactively ```bash export RUNPOD_API_KEY=... -chmod +x runpod-shell.sh remote-run.sh record.sh runpod-sweep.sh +chmod +x runpod-shell.sh remote-run.sh record.sh runpod-sweep.sh batch-tune.sh ./runpod-shell.sh "NVIDIA L4" -# ssh … (printed), or: ./runpod-shell.sh --ssh +./runpod-shell.sh --ssh +# on pod: +# cd /workspace/quantus-gpu-bench +# ./remote-run.sh --cost-per-hour … --duration 30 -# edit remote-run.sh locally, then: -./runpod-shell.sh --upload -# on pod: bash -x ./remote-run.sh --cost-per-hour … --duration 30 - -./runpod-shell.sh --delete # stop billing when done +./runpod-shell.sh --delete ``` ### 4. Run the sweep ```bash export RUNPOD_API_KEY=... -# optional: export TEMPLATE_ID=... -# optional: export CLOUD_TYPE=SECURE # default COMMUNITY -# optional: export SSH_KEY=~/.ssh/id_ed25519 - -chmod +x remote-run.sh record.sh runpod-sweep.sh setup.sh +# optional: export BATCH_SIZES="1000000 16777216" +# optional: export DURATION=30 +# optional: export CLOUD_TYPE=SECURE ./runpod-sweep.sh --gpus-file gpus.example.txt -# full NVIDIA catalog: ./runpod-sweep.sh --gpus-file gpus.all.txt -# or: -./runpod-sweep.sh "NVIDIA GeForce RTX 4090" "NVIDIA GeForce RTX 3090" ``` -Successful rows append to the shared dataset [`results.csv`](results.csv) -(commit new rows so others can reuse them). Per-pod temps go under -`sweep-out/` (gitignored). - -On failure the Pod is deleted unless `KEEP_ON_FAILURE=1`. - -If Community has no capacity for a GPU, the sweep retries on **Secure** -(`FALLBACK_SECURE=1` by default). If a Pod is RUNNING but never gets a public -IP (common on Community), it falls back to proxy SSH -(`podId@ssh.runpod.io`) and pipes files instead of `scp`. +Successful rows append to [`results.csv`](results.csv) (one row per batch size +per Pod). Commit new rows so others can reuse them. ### What each Pod does -1. Build `quantus-miner` from git (`MINER_BRANCH`) + download `quantus-node` release -2. `quantus-node --dev --miner-listen-port 9833` (local chain, no Planck sync) -3. `quantus-miner serve --gpu-devices 1 --cpu-workers 0` → `:9900` -4. Sample Prometheus (`miner_gpu_hash_rate`) + `nvidia-smi` into CSV +1. Build `quantus-miner` from git (`MINER_BRANCH`) +2. Ensure NVIDIA Vulkan ICD (not Mesa llvmpipe) +3. Smoke-test `benchmark` (5s) +4. Sweep `--gpu-batch-size` values → append CSV rows (`notes` includes `batch=N`) 5. Tear down +## Knobs that matter for differently shaped GPUs + +| Knob | Where | Notes | +|------|--------|--------| +| `--gpu-batch-size` | CLI / sweep | **Main runtime knob.** Nonces per dispatch; interacts with tier workgroup hints (`nonces_per_thread` vs occupancy). | +| GPU tier table | `crates/engine-gpu/src/gpu_tiers.rs` | Per-name `workgroup_divisor` + `min_workgroups`. Not a CLI flag — edit + rebuild to retune a class of cards. | +| `--gpu-devices` | CLI | How many GPUs; bench defaults to GPU-only when this is set. | +| `--allow-integrated` | CLI | Include iGPUs when a discrete GPU is present. | +| `--gpu-throttle-ms` | `serve` only | Delay between batches; not used in `benchmark`. | +| threads/workgroup | hardcoded `256` | Not exposed. | + +If util stays low across 1M→16M, dispatch shape may not be the bottleneck (driver, kernel, or tier mis-detect). Check miner logs for `tier:` / `workgroups:`. + ## Collaborative dataset -[`results.csv`](results.csv) is the shared hardware comparison table. After a -sweep or `record.sh` run, commit any new rows you want others to see. +[`results.csv`](results.csv) is the shared hardware comparison table. Prefer the +row with the best hashrate (or hash_per_dollar) per GPU when ranking hardware. ## Spreadsheet columns | Column | Source | |--------|--------| | `cloud_provider` | `runpod` / flag | -| `gpu_model`, `vram_mb`, `sm_count` | `nvidia-smi` (not the RunPod catalog) | -| `hashrate` | avg `miner_gpu_hash_rate` from `:9900/metrics` | -| `gpu_utilization_pct` | avg `utilization.gpu` | -| `cost_per_hour` | Pod `costPerHr` from API (sweep) or your flag | +| `gpu_model`, `vram_mb`, `sm_count` | `nvidia-smi` | +| `hashrate` | `benchmark` Total hashes / Total time | +| `gpu_utilization_pct` | avg `utilization.gpu` during the run | +| `cost_per_hour` | Pod `costPerHr` (sweep) or your flag | | `cost_per_sec` | `cost_per_hour / 3600` | -| `hash_per_dollar` | `hashrate / cost_per_sec` (hashes per $) | +| `hash_per_dollar` | `hashrate / cost_per_sec` | +| `notes` | includes `batch=N` for sweep rows | ## Scripts | Script | Role | |--------|------| -| [`setup.sh`](setup.sh) | Local native node+miner (`--dev` or Planck) | -| [`remote-run.sh`](remote-run.sh) | On-box: fetch bins, `--dev`, mine, `record.sh` | -| [`record.sh`](record.sh) | Prometheus / nvidia-smi → CSV row | +| [`remote-run.sh`](remote-run.sh) | On-box: build miner, Vulkan, batch-size benchmark → CSV | +| [`record.sh`](record.sh) | `--benchmark` (or `--live`) → CSV row(s) | +| [`batch-tune.sh`](batch-tune.sh) | Quick util/hashrate table without cost columns | | [`runpod-sweep.sh`](runpod-sweep.sh) | RunPod REST API multi-GPU loop | | [`runpod-shell.sh`](runpod-shell.sh) | One Pod + SSH; keep alive for manual debug | +| [`setup.sh`](setup.sh) | Local native node+miner (`--dev` or Planck) | | [`gpus.all.txt`](gpus.all.txt) | Full RunPod NVIDIA `gpuTypeId` list | - -## Notes - -- `--dev` is for **hardware comparison**, not mainnet rewards. -- Prefer this over `chain/miner-stack` on cloud GPUs (Compose / nested Docker). -- Catalog “vCPU” is host CPU, not CUDA SMs — SM count comes from inside the Pod. diff --git a/gpu-bench/batch-tune.sh b/gpu-bench/batch-tune.sh index 771c76c..20a523f 100755 --- a/gpu-bench/batch-tune.sh +++ b/gpu-bench/batch-tune.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash -# A/B GPU batch sizes on a pod (or any host with a built miner + NVIDIA GPU). -# Samples nvidia-smi util during each run and prints a comparison table. +# Quick A/B of GPU batch sizes (util + hashrate table). No cost columns. +# For CSV rows with cost/hash_per_dollar, prefer: +# ./remote-run.sh --cost-per-hour … --batch-sizes "…" # # Usage (on pod, after miner is built): # cd /workspace/quantus-gpu-bench # ./batch-tune.sh # ./batch-tune.sh --sizes "1000000 4194304 16777216" --duration 30 -# -# Or from laptop via runpod-shell (see comments at bottom of this file). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh index 3e38ca9..7e3371a 100755 --- a/gpu-bench/record.sh +++ b/gpu-bench/record.sh @@ -16,6 +16,9 @@ DURATION=60 NOTES="" DRY_RUN=0 GPU_DEVICES_FLAG="" +GPU_BATCH_SIZE="${GPU_BATCH_SIZE:-}" +# Space-separated list; when set with --benchmark, runs one row per size. +BATCH_SIZES="${BATCH_SIZES:-}" usage() { cat <<'EOF' @@ -28,13 +31,16 @@ Options: --cost-per-hour USD Hourly cost in USD (e.g. 0.35) --duration SECONDS Sample / benchmark window (default: 60) --gpu-devices N GPUs for --benchmark (default: GPU_DEVICES or 1) + --gpu-batch-size N Single GPU batch size for --benchmark + --batch-sizes "N N" Sweep several batch sizes (one CSV row each) --notes TEXT Optional notes column --dry-run Print the CSV row but do not append -h, --help Show this help Examples: ./record.sh --provider vast.ai --cost-per-hour 0.35 - ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 60 + ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 30 + ./record.sh --benchmark --batch-sizes "1000000 4194304 16777216" --provider runpod --cost-per-hour 0.39 EOF } @@ -373,35 +379,50 @@ run_live() { emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" } -run_benchmark() { - local miner_bin - miner_bin="$(resolve_miner_bin)" - local gpu_devices="${GPU_DEVICES_FLAG:-${GPU_DEVICES:-1}}" +run_benchmark_once() { + local miner_bin="$1" + local gpu_devices="$2" + local batch_size="$3" + local notes_extra="$4" local util_file bench_log util_file="$(mktemp)" bench_log="$(mktemp)" - trap 'rm -f "${util_file}" "${bench_log}"' RETURN - echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}) ..." >&2 + local batch_label="default" + if [[ -n "${batch_size}" ]]; then + batch_label="${batch_size}" + fi + + echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}, batch=${batch_label}) ..." >&2 sample_util_during "${util_file}" "${DURATION}" & local sampler_pid=$! set +e - "${miner_bin}" benchmark \ - --cpu-workers 0 \ - --gpu-devices "${gpu_devices}" \ - --duration "${DURATION}" \ - >"${bench_log}" 2>&1 + if [[ -n "${batch_size}" ]]; then + "${miner_bin}" benchmark \ + --cpu-workers 0 \ + --gpu-devices "${gpu_devices}" \ + --gpu-batch-size "${batch_size}" \ + --duration "${DURATION}" \ + >"${bench_log}" 2>&1 + else + "${miner_bin}" benchmark \ + --cpu-workers 0 \ + --gpu-devices "${gpu_devices}" \ + --duration "${DURATION}" \ + >"${bench_log}" 2>&1 + fi local bench_rc=$? set -e wait "${sampler_pid}" 2>/dev/null || true if [[ "${bench_rc}" -ne 0 ]]; then - echo "error: benchmark failed (exit ${bench_rc}). Output:" >&2 + echo "error: benchmark failed (exit ${bench_rc}, batch=${batch_label}). Output:" >&2 cat "${bench_log}" >&2 - exit 1 + rm -f "${util_file}" "${bench_log}" + return 1 fi cat "${bench_log}" >&2 @@ -409,8 +430,9 @@ run_benchmark() { local hashrate util_avg hashrate="$(parse_benchmark_hashrate "$(cat "${bench_log}")")" if [[ -z "${hashrate}" ]]; then - echo "error: could not parse hashrate from benchmark output" >&2 - exit 1 + echo "error: could not parse hashrate from benchmark output (batch=${batch_label})" >&2 + rm -f "${util_file}" "${bench_log}" + return 1 fi util_avg="$(average_list <"${util_file}")" if [[ -n "${util_avg}" ]]; then @@ -423,9 +445,55 @@ run_benchmark() { read -r hash_per_dollar } < <(compute_cost_metrics "${hashrate}" "${COST_PER_HOUR}") + local saved_notes="${NOTES}" + if [[ -n "${notes_extra}" ]]; then + if [[ -n "${NOTES}" ]]; then + NOTES="${NOTES};${notes_extra}" + else + NOTES="${notes_extra}" + fi + fi + local ts ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" + NOTES="${saved_notes}" + + rm -f "${util_file}" "${bench_log}" + return 0 +} + +run_benchmark() { + local miner_bin + miner_bin="$(resolve_miner_bin)" + local gpu_devices="${GPU_DEVICES_FLAG:-${GPU_DEVICES:-1}}" + + local sizes=() + if [[ -n "${BATCH_SIZES}" ]]; then + # shellcheck disable=SC2206 + sizes=(${BATCH_SIZES}) + elif [[ -n "${GPU_BATCH_SIZE}" ]]; then + sizes=("${GPU_BATCH_SIZE}") + else + sizes=("") + fi + + local bs note_extra + local any_ok=0 + for bs in "${sizes[@]}"; do + note_extra="" + if [[ -n "${bs}" ]]; then + note_extra="batch=${bs}" + fi + if run_benchmark_once "${miner_bin}" "${gpu_devices}" "${bs}" "${note_extra}"; then + any_ok=1 + fi + done + + if [[ "${any_ok}" -ne 1 ]]; then + echo "error: all benchmark runs failed" >&2 + exit 1 + fi } # --- args --- @@ -455,6 +523,14 @@ while [[ $# -gt 0 ]]; do GPU_DEVICES_FLAG="${2:-}" shift 2 ;; + --gpu-batch-size) + GPU_BATCH_SIZE="${2:-}" + shift 2 + ;; + --batch-sizes) + BATCH_SIZES="${2:-}" + shift 2 + ;; --notes) NOTES="${2:-}" shift 2 diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh index 77e71d5..a07ee49 100755 --- a/gpu-bench/remote-run.sh +++ b/gpu-bench/remote-run.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Run on a GPU host (e.g. RunPod): fetch node release + build miner from git, -# start --dev node + GPU miner, sample Prometheus into results.csv. +# Run on a GPU host (e.g. RunPod): build miner from git, ensure NVIDIA Vulkan, +# run quantus-miner benchmark across batch sizes → results.csv (no node). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -17,15 +17,13 @@ MINER_BRANCH="${MINER_BRANCH:-illuzen/gpu-bench}" MINER_VERSION="${MINER_VERSION:-v3.3.1}" MINER_URL="${MINER_URL:-https://github.com/Quantus-Network/quantus-miner/releases/download/${MINER_VERSION}/quantus-miner-linux-x86_64}" FORCE_MINER_BUILD="${FORCE_MINER_BUILD:-0}" -NODE_URL="${NODE_URL:-}" # optional override; default = latest chain release PROVIDER="${PROVIDER:-runpod}" COST_PER_HOUR="${COST_PER_HOUR:-}" -DURATION="${DURATION:-60}" +DURATION="${DURATION:-30}" GPU_DEVICES="${GPU_DEVICES:-1}" -METRICS_PORT="${METRICS_PORT:-9900}" -MINER_LISTEN_PORT="${MINER_LISTEN_PORT:-9833}" -WARMUP_SECONDS="${WARMUP_SECONDS:-30}" +# Default sweep: 1M → 16M. Override with --batch-sizes or BATCH_SIZES. +BATCH_SIZES="${BATCH_SIZES:-1000000 4194304 8388608 16777216}" NOTES="${NOTES:-}" usage() { @@ -34,9 +32,9 @@ Usage: ./remote-run.sh [options] --provider NAME cloud_provider column (default: runpod) --cost-per-hour USD required for hash_per_dollar column - --duration SECONDS Prometheus sample window (default: 60) + --duration SECONDS per-batch-size benchmark window (default: 30) --gpu-devices N default 1 - --warmup SECONDS wait after miner start before sampling (default: 30) + --batch-sizes "N N N" GPU batch sizes to sweep (default: 1M 4M 8M 16M) --notes TEXT --miner-branch REF git branch/tag/commit to build (default: illuzen/gpu-bench) --miner-repo URL git remote (default: Quantus-Network/quantus-miner) @@ -46,10 +44,11 @@ Usage: ./remote-run.sh [options] --force-miner-build rebuild even if binary exists --help -Builds quantus-miner from git (or downloads a release), downloads quantus-node, -runs --dev + GPU miner, samples :9900/metrics into results.csv. +Builds quantus-miner from git (or downloads a release), sets up NVIDIA Vulkan, +runs `quantus-miner benchmark` for each batch size (no node), appends rows to +results.csv (notes include batch=N). -Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, NODE_URL +Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, BATCH_SIZES EOF } @@ -66,14 +65,15 @@ while [[ $# -gt 0 ]]; do --cost-per-hour) COST_PER_HOUR="${2:-}"; shift 2 ;; --duration) DURATION="${2:-}"; shift 2 ;; --gpu-devices) GPU_DEVICES="${2:-}"; shift 2 ;; - --warmup) WARMUP_SECONDS="${2:-}"; shift 2 ;; + --batch-sizes) BATCH_SIZES="${2:-}"; shift 2 ;; --notes) NOTES="${2:-}"; shift 2 ;; --miner-branch) MINER_BRANCH="${2:-}"; shift 2 ;; --miner-repo) MINER_REPO="${2:-}"; shift 2 ;; --miner-source) MINER_SOURCE="${2:-}"; shift 2 ;; --miner-url) MINER_URL="${2:-}"; MINER_SOURCE="release"; shift 2 ;; --force-miner-build) FORCE_MINER_BUILD=1; shift ;; - --node-url) NODE_URL="${2:-}"; shift 2 ;; + # Deprecated no-ops (node path removed) + --warmup | --node-url) shift 2 ;; -h | --help) usage; exit 0 ;; *) echo "error: unknown option: $1" >&2 @@ -430,159 +430,46 @@ download_miner() { esac } -download_node() { - local dest="${BIN_DIR}/quantus-node" - if [[ -x "${dest}" ]]; then - echo "Using existing ${dest}" >&2 - printf '%s\n' "${dest}" - return - fi - - local url="${NODE_URL}" - if [[ -z "${url}" ]]; then - echo "Resolving latest quantus-node release ..." >&2 - local release_json tag - release_json="$(curl -fsSL https://api.github.com/repos/Quantus-Network/chain/releases/latest)" - tag="$(echo "${release_json}" | grep -o '"tag_name": "[^"]*"' | head -n 1 | cut -d'"' -f4)" - url="https://github.com/Quantus-Network/chain/releases/download/${tag}/quantus-node-${tag}-x86_64-unknown-linux-gnu.tar.gz" - fi - - echo "Downloading node: ${url}" >&2 - local tmp - tmp="$(mktemp -d)" - curl -fL "${url}" -o "${tmp}/node.tar.gz" - tar -xzf "${tmp}/node.tar.gz" -C "${tmp}" - if [[ ! -f "${tmp}/quantus-node" ]]; then - echo "error: archive missing quantus-node" >&2 - exit 1 - fi - mv "${tmp}/quantus-node" "${dest}" - chmod +x "${dest}" - rm -rf "${tmp}" - printf '%s\n' "${dest}" -} - -stop_pidfile() { - local pidfile="$1" - if [[ ! -f "${pidfile}" ]]; then - return 0 - fi - local pid - pid="$(cat "${pidfile}")" - if kill -0 "${pid}" 2>/dev/null; then - kill "${pid}" 2>/dev/null || true - sleep 1 - kill -9 "${pid}" 2>/dev/null || true - fi - rm -f "${pidfile}" -} - -cleanup() { - stop_pidfile "${RUN_DIR}/miner.pid" - stop_pidfile "${RUN_DIR}/node.pid" -} -trap cleanup EXIT - ensure_nvidia_vulkan -NODE_BIN="$(download_node)" MINER_BIN="$(download_miner | tail -n 1)" if [[ ! -x "${MINER_BIN}" ]]; then echo "error: miner binary missing or not executable: '${MINER_BIN}'" >&2 exit 1 fi -# Fail fast with a clear hint if the host glibc is too old for the release binary. -if ! "${NODE_BIN}" --version >/tmp/qnode-ver.txt 2>/tmp/qnode-ver.err; then - if grep -q 'GLIBC_' /tmp/qnode-ver.err 2>/dev/null; then - echo "error: quantus-node needs a newer glibc than this image provides:" >&2 - cat /tmp/qnode-ver.err >&2 - echo "Use an Ubuntu 24.04+ image, e.g. IMAGE_NAME=runpod/base:1.1.0-cuda1281-ubuntu2404" >&2 - exit 1 - fi -fi - echo "GPU:" nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv || nvidia-smi || true -mkdir -p "${RUN_DIR}/node-data" -echo "Starting quantus-node --dev ..." -nohup "${NODE_BIN}" \ - --dev \ - --base-path "${RUN_DIR}/node-data" \ - --rpc-port 9944 \ - --prometheus-port 9615 \ - --prometheus-external \ - --miner-listen-port "${MINER_LISTEN_PORT}" \ - --rpc-cors all \ - >"${RUN_DIR}/node.log" 2>&1 & -echo $! >"${RUN_DIR}/node.pid" -sleep 3 - -if ! kill -0 "$(cat "${RUN_DIR}/node.pid")" 2>/dev/null; then - echo "error: node failed to start; log:" >&2 - tail -n 80 "${RUN_DIR}/node.log" >&2 || true - if grep -q 'GLIBC_' "${RUN_DIR}/node.log" 2>/dev/null; then - echo "hint: IMAGE_NAME=runpod/base:1.1.0-cuda1281-ubuntu2404 (GLIBC 2.38+)" >&2 - fi - exit 1 -fi - -echo "Starting quantus-miner (gpu-devices=${GPU_DEVICES}) ..." -# Keep VK_ICD_FILENAMES / NVIDIA caps in the miner environment. -nohup env \ +# Smoke-test GPU path before spending minutes on a batch sweep. +echo "Smoke-testing benchmark (5s) ..." +set +e +smoke_log="$(mktemp)" +env \ VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" \ __GLX_VENDOR_LIBRARY_NAME=nvidia \ LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" \ XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" \ - "${MINER_BIN}" serve \ - --node-addr "127.0.0.1:${MINER_LISTEN_PORT}" \ - --gpu-devices "${GPU_DEVICES}" \ + "${MINER_BIN}" benchmark \ --cpu-workers 0 \ - --metrics-port "${METRICS_PORT}" \ - >"${RUN_DIR}/miner.log" 2>&1 & -echo $! >"${RUN_DIR}/miner.pid" -sleep 3 - -if ! kill -0 "$(cat "${RUN_DIR}/miner.pid")" 2>/dev/null; then - echo "error: miner failed to start; log:" >&2 - tail -n 80 "${RUN_DIR}/miner.log" >&2 || true - if grep -qiE 'No usable GPU|llvmpipe|Vulkan' "${RUN_DIR}/miner.log" 2>/dev/null; then + --gpu-devices "${GPU_DEVICES}" \ + --gpu-batch-size 1000000 \ + --duration 5 \ + >"${smoke_log}" 2>&1 +smoke_rc=$? +set -e +if [[ "${smoke_rc}" -ne 0 ]]; then + echo "error: benchmark smoke test failed; log:" >&2 + cat "${smoke_log}" >&2 + if grep -qiE 'No usable GPU|llvmpipe|Vulkan|Failed to initialize GPU' "${smoke_log}" 2>/dev/null; then echo "hint: need NVIDIA Vulkan ICD + NVIDIA_DRIVER_CAPABILITIES=all (not Mesa llvmpipe)" >&2 + rm -f "${smoke_log}" + exit "${EXIT_COMPUTE_ONLY}" fi + rm -f "${smoke_log}" exit 1 fi - -# Wait for metrics + non-zero hashrate (jobs from --dev) -echo "Warming up ${WARMUP_SECONDS}s / waiting for hashrate ..." -ready=0 -for ((i = 0; i < WARMUP_SECONDS + 60; i++)); do - if ! kill -0 "$(cat "${RUN_DIR}/miner.pid")" 2>/dev/null; then - echo "error: miner exited during warmup; log:" >&2 - tail -n 80 "${RUN_DIR}/miner.log" >&2 || true - exit 1 - fi - if curl -sf "http://127.0.0.1:${METRICS_PORT}/metrics" \ - | awk '/^miner_gpu_hash_rate[[:space:]]/ { if ($2+0 > 0) found=1 } END { exit !found }'; then - ready=1 - break - fi - sleep 1 -done - -if [[ "${ready}" -ne 1 ]]; then - echo "warning: no positive miner_gpu_hash_rate yet; sampling anyway" >&2 - echo "--- miner log ---" >&2 - tail -n 40 "${RUN_DIR}/miner.log" >&2 || true - echo "--- node log ---" >&2 - tail -n 40 "${RUN_DIR}/node.log" >&2 || true -fi - -# Point record.sh at our run dir / metrics -export METRICS_PORT -mkdir -p "${SCRIPT_DIR}/.run" 2>/dev/null || true -# record.sh reads .run/metrics.port relative to its own dir — use WORK_DIR copy -echo "${METRICS_PORT}" >"${WORK_DIR}/.run/metrics.port" +rm -f "${smoke_log}" NOTE_ARGS=() if [[ -n "${NOTES}" ]]; then @@ -590,12 +477,22 @@ if [[ -n "${NOTES}" ]]; then fi cd "${WORK_DIR}" -./record.sh --live \ +echo "Batch-size sweep: ${BATCH_SIZES} (${DURATION}s each)" +export MINER_BIN +env \ + VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ + NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" \ + __GLX_VENDOR_LIBRARY_NAME=nvidia \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" \ + XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" \ + MINER_BIN="${MINER_BIN}" \ + ./record.sh --benchmark \ --provider "${PROVIDER}" \ --cost-per-hour "${COST_PER_HOUR}" \ --duration "${DURATION}" \ + --gpu-devices "${GPU_DEVICES}" \ + --batch-sizes "${BATCH_SIZES}" \ "${NOTE_ARGS[@]}" echo "Done. Results: ${RESULTS_CSV}" -cat "${RESULTS_CSV}" -# Leave processes up until EXIT trap on script end — stop after record so CSV is final +tail -n 8 "${RESULTS_CSV}" || true diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh index 309d6b6..0ba1057 100755 --- a/gpu-bench/runpod-shell.sh +++ b/gpu-bench/runpod-shell.sh @@ -21,7 +21,7 @@ Same env as runpod-sweep.sh (RUNPOD_API_KEY, IMAGE_NAME, CLOUD_TYPE, SSH_KEY, On the pod, typical debug loop: cd /workspace/quantus-gpu-bench - bash -x ./remote-run.sh --cost-per-hour 0.39 --duration 30 --warmup 20 + bash -x ./remote-run.sh --cost-per-hour 0.39 --duration 30 # or step through (see printed hints) EOF @@ -74,13 +74,10 @@ print_ssh_hint() { echo "On the pod:" cat <= batch size): - # FORCE_MINER_BUILD=1 ./remote-run.sh --cost-per-hour ${cost} --duration 20 --warmup 15 - # ./batch-tune.sh --duration 30 + # ./remote-run.sh --cost-per-hour ${cost} --batch-sizes "1000000 16777216" # Vulkan sanity: nvidia-smi @@ -150,9 +147,10 @@ cmd_create() { ssh_upload_files "${mode}" "${host}" "${port}" \ "${SCRIPT_DIR}/remote-run.sh" \ - "${SCRIPT_DIR}/record.sh" + "${SCRIPT_DIR}/record.sh" \ + "${SCRIPT_DIR}/batch-tune.sh" ssh_cmd "${mode}" "${host}" "${port}" \ - "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh'" + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' '${REMOTE_DIR}/batch-tune.sh'" write_state "${pod_id}" "${mode}" "${host}" "${port}" "${cost}" "${gpu_type}" print_ssh_hint "${mode}" "${host}" "${port}" "${cost}" "${pod_id}" @@ -163,9 +161,10 @@ cmd_upload() { echo "Uploading scripts to ${POD_ID} (${REMOTE_DIR}) ..." >&2 ssh_upload_files "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ "${SCRIPT_DIR}/remote-run.sh" \ - "${SCRIPT_DIR}/record.sh" + "${SCRIPT_DIR}/record.sh" \ + "${SCRIPT_DIR}/batch-tune.sh" ssh_cmd "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ - "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh'" + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' '${REMOTE_DIR}/batch-tune.sh'" echo "Done." >&2 } diff --git a/gpu-bench/runpod-sweep.sh b/gpu-bench/runpod-sweep.sh index b3064e7..77d76bf 100755 --- a/gpu-bench/runpod-sweep.sh +++ b/gpu-bench/runpod-sweep.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Orchestrate RunPod Pods via REST API: for each GPU type, create → SSH → -# remote-run.sh (--dev node + miner + Prometheus CSV) → scp results → delete. +# remote-run.sh (miner build + benchmark batch-size sweep → CSV) → scp → delete. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -13,18 +13,16 @@ SSH_USER="${SSH_USER:-root}" CLOUD_TYPE="${CLOUD_TYPE:-COMMUNITY}" # Prefer runpod/base (CUDA + SSH, no PyTorch). Drivers come from the host. # Use a current Hub tag — stale tags leave Pods RUNNING with runtime=null forever. -# Ubuntu 24.04 required: recent quantus-node releases need GLIBC_2.38+ -# (22.04 ships 2.35 and will fail with "GLIBC_2.38 not found"). +# Ubuntu 24.04 base is fine for miner builds; keep NVIDIA_DRIVER_CAPABILITIES=all. IMAGE_NAME="${IMAGE_NAME:-runpod/base:1.1.0-cuda1281-ubuntu2404}" TEMPLATE_ID="${TEMPLATE_ID:-}" # Community hosts often reject large disks ("machine does not have the resources"). -# Binaries + --dev chain fit in a small container disk; volume defaults to 0. # Cargo build of miner needs more disk than a release binary download. CONTAINER_DISK_GB="${CONTAINER_DISK_GB:-50}" VOLUME_GB="${VOLUME_GB:-0}" -DURATION="${DURATION:-60}" -WARMUP_SECONDS="${WARMUP_SECONDS:-45}" +DURATION="${DURATION:-30}" GPU_DEVICES="${GPU_DEVICES:-1}" +BATCH_SIZES="${BATCH_SIZES:-1000000 4194304 8388608 16777216}" REMOTE_DIR="${REMOTE_DIR:-/workspace/quantus-gpu-bench}" MINER_SOURCE="${MINER_SOURCE:-git}" MINER_REPO="${MINER_REPO:-https://github.com/Quantus-Network/quantus-miner.git}" @@ -54,7 +52,8 @@ Environment: CLOUD_TYPE COMMUNITY (default) or SECURE IMAGE_NAME Docker image (default: runpod/base … ubuntu2404 for GLIBC) TEMPLATE_ID Optional RunPod template id (skips IMAGE_NAME) - DURATION Sample seconds per GPU (default 60) + DURATION Benchmark seconds per batch size (default 30) + BATCH_SIZES Space-separated gpu-batch-size list (default 1M 4M 8M 16M) RESULTS_CSV Collaborative dataset to append (default ./results.csv) OUT_DIR Per-pod temp rows (default ./sweep-out, gitignored) KEEP_ON_FAILURE=1 Do not delete Pod if remote-run fails @@ -564,8 +563,8 @@ run_one_attempt() { --provider runpod \ --cost-per-hour '${cost}' \ --duration '${DURATION}' \ - --warmup '${WARMUP_SECONDS}' \ --gpu-devices '${GPU_DEVICES}' \ + --batch-sizes '${BATCH_SIZES}' \ --miner-source '${MINER_SOURCE}' \ --miner-repo '${MINER_REPO}' \ --miner-branch '${MINER_BRANCH}' \ From 0ed276c5ff717fc600ce9e4923ba8b550ef541d3 Mon Sep 17 00:00:00 2001 From: illuzen Date: Mon, 3 Aug 2026 22:51:32 +0800 Subject: [PATCH 12/13] collect results --- gpu-bench/results.csv | 67 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv index 36820dd..976001e 100644 --- a/gpu-bench/results.csv +++ b/gpu-bench/results.csv @@ -5,10 +5,63 @@ timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_ 2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,0.0000333333,94848678000.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct 2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,0.0000527778,109270419157.894745,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct 2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,0.0000555556,140410184275.854004,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct -2026-08-01T10:39:32Z,runpod,NVIDIA GeForce RTX 3080,10240,68,580.95.05,6604543.833333,72.87,0.17,0.0000472222,139860928235.287048,60,gpuTypeId=NVIDIA GeForce RTX 3080;pod=l6v6j20c2bf0jr;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-01T10:43:30Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,6196787.448276,63.76,0.18,0.0000500000,123935748965.520020,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=b0dzcmo8tarsno;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-01T10:47:53Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.65.06,6834865.566667,53.37,0.22,0.0000611111,111843254727.278183,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=tdmmxojjx85zbn;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-02T03:55:55Z,runpod,NVIDIA RTX A4500,20470,56,580.126.18,5587581.033333,67.63,0.19,0.0000527778,105869956421.046310,60,gpuTypeId=NVIDIA RTX A4500;pod=nk7hljiu43i8gc;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-02T04:02:04Z,runpod,NVIDIA RTX A5000,24564,64,550.144.03,6510972.633333,65.13,0.16,0.0000444444,146496884249.992493,60,gpuTypeId=NVIDIA RTX A5000;pod=sgeizrvavai3lt;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-02T04:09:46Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7669824.448276,53.41,0.2,0.0000555556,138056840068.967987,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=2o26qc0dczed6h;ssh=direct;miner=git@illuzen/gpu-bench -2026-08-02T04:15:33Z,runpod,NVIDIA RTX 4000 SFF Ada Generation,20475,48,560.35.05,6308176.620690,62.55,0.18,0.0000500000,126163532413.800018,60,gpuTypeId=NVIDIA RTX 4000 SFF Ada Generation;pod=cxdke7crdl03qr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:39:32Z,runpod,NVIDIA GeForce RTX 3080,10240,68,580.95.05,6604543.833333,72.87,0.17,0.0000472222,139860928235.287048,60,gpuTypeId=NVIDIA GeForce RTX 3080;pod=l6v6j20c2bf0jr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:43:30Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,6196787.448276,63.76,0.18,0.0000500000,123935748965.520020,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=b0dzcmo8tarsno;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:47:53Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.65.06,6834865.566667,53.37,0.22,0.0000611111,111843254727.278183,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=tdmmxojjx85zbn;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T03:55:55Z,runpod,NVIDIA RTX A4500,20470,56,580.126.18,5587581.033333,67.63,0.19,0.0000527778,105869956421.046310,60,gpuTypeId=NVIDIA RTX A4500;pod=nk7hljiu43i8gc;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:02:04Z,runpod,NVIDIA RTX A5000,24564,64,550.144.03,6510972.633333,65.13,0.16,0.0000444444,146496884249.992493,60,gpuTypeId=NVIDIA RTX A5000;pod=sgeizrvavai3lt;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:09:46Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7669824.448276,53.41,0.2,0.0000555556,138056840068.967987,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=2o26qc0dczed6h;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:15:33Z,runpod,NVIDIA RTX 4000 SFF Ada Generation,20475,48,560.35.05,6308176.620690,62.55,0.18,0.0000500000,126163532413.800018,60,gpuTypeId=NVIDIA RTX 4000 SFF Ada Generation;pod=cxdke7crdl03qr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-03T11:17:05Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.195.03,13750571.900000,36.00,0.39,0.0001083333,126928356000.000000,20, +2026-08-03T12:39:22Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,42248835.662009,98.93,0.99,0.0002750000,153632129680.032715,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:39:53Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,42082385.647841,99.87,0.99,0.0002750000,153026856901.239990,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:40:24Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,39295472.691030,95.40,0.99,0.0002750000,142892627967.381836,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:40:55Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,37085894.820191,99.93,0.99,0.0002750000,134857799346.149094,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T12:49:27Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,6038487.060385,93.33,0.24,0.0000666667,90577305905.774994,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:49:58Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,5448962.558294,99.40,0.24,0.0000666667,81734438374.410004,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:50:29Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,5134779.381443,98.93,0.24,0.0000666667,77021690721.645004,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:51:00Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,4896074.708171,98.87,0.24,0.0000666667,73441120622.564987,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T12:55:05Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,24866844.207723,92.40,0.84,0.0002333333,106572189461.669998,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:55:35Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,22867880.851064,93.60,0.84,0.0002333333,98005203647.417145,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:56:06Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,22001993.094290,94.00,0.84,0.0002333333,94294256118.385712,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:56:37Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,21368759.764860,94.20,0.84,0.0002333333,91580398992.257141,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:01:04Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,16039933.444260,99.53,0.57,0.0001583333,101304842805.852631,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:01:35Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,14665398.601399,99.93,0.57,0.0001583333,92623570114.098953,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:02:05Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,13948466.910542,93.33,0.57,0.0001583333,88095580487.633698,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:02:36Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,13271364.007910,100.00,0.57,0.0001583333,83819141102.589478,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:06:42Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,21930116.472546,99.00,0.74,0.0002055556,106687053109.683243,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:07:13Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,19058694.792703,99.93,0.74,0.0002055556,92717974667.203781,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:07:43Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,18066915.838304,97.67,0.74,0.0002055556,87893104078.235687,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:08:14Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,17284602.725158,100.00,0.74,0.0002055556,84087256500.768646,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:13:21Z,runpod,NVIDIA L4,23034,60,550.127.05,9290709.290709,93.33,0.39,0.0001083333,85760393452.698456,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:13:51Z,runpod,NVIDIA L4,23034,60,550.127.05,8610822.781457,99.07,0.39,0.0001083333,79484517982.679993,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:14:22Z,runpod,NVIDIA L4,23034,60,550.127.05,7973439.265814,100.00,0.39,0.0001083333,73600977838.283066,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:14:54Z,runpod,NVIDIA L4,23034,60,550.127.05,7591500.452489,98.73,0.39,0.0001083333,70075388792.206146,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:31:40Z,runpod,NVIDIA L40S,46068,142,570.124.06,31713810.316140,92.27,0.99,0.0002750000,115322946604.145447,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:32:10Z,runpod,NVIDIA L40S,46068,142,570.124.06,29362911.214333,92.80,0.99,0.0002750000,106774222597.574539,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:32:41Z,runpod,NVIDIA L40S,46068,142,570.124.06,27740105.820106,96.67,0.99,0.0002750000,100873112073.112717,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:33:13Z,runpod,NVIDIA L40S,46068,142,570.124.06,26507780.381830,93.87,0.99,0.0002750000,96391928661.199997,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:37:33Z,runpod,NVIDIA A40,46068,84,570.195.03,8873379.860419,93.20,0.44,0.0001222222,72600380676.155457,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:38:04Z,runpod,NVIDIA A40,46068,84,570.195.03,8407904.830759,93.33,0.44,0.0001222222,68791948615.300919,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:38:35Z,runpod,NVIDIA A40,46068,84,570.195.03,8261925.147735,93.80,0.44,0.0001222222,67597569390.559090,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:39:06Z,runpod,NVIDIA A40,46068,84,570.195.03,8186670.136630,98.73,0.44,0.0001222222,66981846572.427277,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:42:54Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,8009305.417082,93.33,1.49,0.0004138889,19351341947.312214,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:43:26Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6875908.196721,93.33,1.49,0.0004138889,16612932555.835972,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:43:58Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6713034.571063,100.00,1.49,0.0004138889,16219412386.460939,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:44:29Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6613882.785808,98.13,1.49,0.0004138889,15979851026.113289,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:09:24Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,17382617.382617,93.27,2.99,0.0008305556,20928903872.047222,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:09:55Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,12549446.808511,93.33,2.99,0.0008305556,15109701843.023277,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:10:26Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,10544727.224611,93.47,2.99,0.0008305556,12695992645.016586,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:10:57Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,9875405.101373,95.13,2.99,0.0008305556,11890119854.495918,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:21:12Z,runpod,NVIDIA H200,143771,132,550.144.03,16516816.516817,93.33,4.39,0.0012194444,13544542018.346514,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:21:43Z,runpod,NVIDIA H200,143771,132,550.144.03,11705034.418605,96.07,4.39,0.0012194444,9598661482.227335,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:22:14Z,runpod,NVIDIA H200,143771,132,550.144.03,9956804.747774,99.33,4.39,0.0012194444,8165033506.147243,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:22:45Z,runpod,NVIDIA H200,143771,132,550.144.03,9227197.411841,96.67,4.39,0.0012194444,7566722251.168017,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:27:14Z,runpod,NVIDIA B200,183359,160,580.105.08,19182180.851064,92.40,5.89,0.0016361111,11724253151.753889,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:27:45Z,runpod,NVIDIA B200,183359,160,580.105.08,13615163.696588,92.93,5.89,0.0016361111,8321662021.683668,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:28:16Z,runpod,NVIDIA B200,183359,160,580.105.08,11096042.328042,93.27,5.89,0.0016361111,6781961355.000205,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:28:47Z,runpod,NVIDIA B200,183359,160,580.105.08,10390062.059974,93.27,5.89,0.0016361111,6350462379.610595,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:33:46Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,19733777.038270,93.13,7.39,0.0020527778,9613206676.288500,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:34:18Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,13939195.746095,92.86,7.39,0.0020527778,6790406588.084168,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:34:49Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,11441547.837658,93.27,7.39,0.0020527778,5573690421.592531,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:35:21Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,10758073.741584,93.07,7.39,0.0020527778,5240739576.414398,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 From edb8aa1eb0b730c281ea992431efd85488fcf783 Mon Sep 17 00:00:00 2001 From: illuzen Date: Mon, 3 Aug 2026 23:22:16 +0800 Subject: [PATCH 13/13] more realistic benchmark --- Cargo.lock | 1 + crates/miner-cli/Cargo.toml | 1 + crates/miner-cli/src/main.rs | 307 ++++++++++++++++++++++++++++++++--- gpu-bench/README.md | 11 +- gpu-bench/batch-tune.sh | 39 +++-- gpu-bench/gpus.example.txt | 9 +- gpu-bench/record.sh | 58 +++++-- gpu-bench/remote-run.sh | 29 +++- gpu-bench/results.csv | 4 + gpu-bench/runpod-shell.sh | 5 +- gpu-bench/runpod-sweep.sh | 10 +- 11 files changed, 405 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd40224..09a4473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,6 +1514,7 @@ dependencies = [ "metrics", "miner-service", "num_cpus", + "pow-core", "primitive-types 0.13.1", "rand 0.9.2", "tokio", diff --git a/crates/miner-cli/Cargo.toml b/crates/miner-cli/Cargo.toml index f57af9c..f3ee78d 100644 --- a/crates/miner-cli/Cargo.toml +++ b/crates/miner-cli/Cargo.toml @@ -14,6 +14,7 @@ env_logger = { workspace = true } log = { workspace = true } engine-cpu = { path = "../engine-cpu" } engine-gpu = { path = "../engine-gpu" } +pow-core = { path = "../pow-core" } primitive-types = { workspace = true } rand = { workspace = true } num_cpus = { workspace = true } diff --git a/crates/miner-cli/src/main.rs b/crates/miner-cli/src/main.rs index d2ece53..ba89dfc 100644 --- a/crates/miner-cli/src/main.rs +++ b/crates/miner-cli/src/main.rs @@ -1,16 +1,19 @@ use clap::{Parser, Subcommand}; -use engine_cpu::{AtomicBoolCancelCheck, EngineRange, MinerEngine}; +use engine_cpu::{AtomicBoolCancelCheck, EngineRange, JobIdCancelCheck, MinerEngine}; use miner_service::{run, ServiceConfig}; +use pow_core::JobContext; use primitive_types::U512; use rand::RngCore; -use std::sync::atomic::AtomicBool; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; use std::thread; use std::time::{Duration, Instant}; // CLI defaults const DEFAULT_GPU_BATCH_SIZE: u32 = 1_000_000; const DEFAULT_CPU_BATCH_SIZE: u64 = 10_000; +/// Default difficulty when `--job-interval` is set: ~1s to find at 10 MH/s. +const DEFAULT_JOB_DIFFICULTY: u64 = 10_000_000; #[derive(Subcommand, Debug)] enum Command { @@ -85,6 +88,16 @@ enum Command { #[arg(short, long, default_value_t = 10)] duration: u64, + /// Simulate node job churn: every N seconds push a new random header + /// (like NewJob). 0 = sustained single job (default). + #[arg(long = "job-interval", default_value_t = 0.0)] + job_interval: f64, + + /// PoW difficulty for job simulation (decimal). Use "max" for unreachable + /// (cancel-only churn). Default when --job-interval > 0: 10000000. + #[arg(long = "difficulty")] + difficulty: Option, + /// Allow integrated GPUs (APUs) even when discrete GPUs are available #[arg(long = "allow-integrated", env = "MINER_ALLOW_INTEGRATED")] allow_integrated: bool, @@ -161,6 +174,8 @@ async fn main() { gpu_batch_size, cpu_batch_size, duration, + job_interval, + difficulty, allow_integrated, verbose, } => { @@ -171,6 +186,8 @@ async fn main() { gpu_batch_size, cpu_batch_size, duration, + job_interval, + difficulty, allow_integrated, ) .await; @@ -191,12 +208,39 @@ fn init_logger(verbose: bool) { env_logger::init(); } +fn parse_difficulty(raw: Option<&str>, job_interval: f64) -> U512 { + match raw { + None if job_interval > 0.0 => U512::from(DEFAULT_JOB_DIFFICULTY), + None => U512::MAX, + Some(s) if s.eq_ignore_ascii_case("max") => U512::MAX, + Some(s) => match U512::from_dec_str(s) { + Ok(d) if !d.is_zero() => d, + Ok(_) => { + eprintln!("❌ ERROR: --difficulty must be non-zero (or \"max\")"); + std::process::exit(1); + } + Err(_) => { + eprintln!("❌ ERROR: invalid --difficulty '{s}' (decimal or \"max\")"); + std::process::exit(1); + } + }, + } +} + +fn random_header() -> [u8; 32] { + let mut header = [0u8; 32]; + rand::rng().fill_bytes(&mut header); + header +} + async fn run_benchmark( cpu_workers: Option, gpu_devices: Option, gpu_batch_size: u32, cpu_batch_size: u64, duration: u64, + job_interval: f64, + difficulty_arg: Option, allow_integrated: bool, ) { // When --gpu-devices is set and --cpu-workers is omitted, default to GPU-only @@ -222,6 +266,7 @@ async fn run_benchmark( }; let total_workers = effective_cpu_workers + effective_gpu_devices; + let difficulty = parse_difficulty(difficulty_arg.as_deref(), job_interval); println!("🚀 Quantus Miner Benchmark"); println!("=========================="); @@ -234,6 +279,16 @@ async fn run_benchmark( println!("GPU batch size: {} nonces", gpu_batch_size); println!("CPU batch size: {} hashes", cpu_batch_size); println!("Duration: {} seconds", duration); + if job_interval > 0.0 { + println!("Job interval: {:.2}s (simulated NewJob)", job_interval); + if difficulty == U512::MAX { + println!("Difficulty: max (cancel-only churn, no finds)"); + } else { + println!("Difficulty: {difficulty}"); + } + } else { + println!("Job interval: off (sustained single job)"); + } println!(); if total_workers == 0 { @@ -241,6 +296,11 @@ async fn run_benchmark( std::process::exit(1); } + if job_interval < 0.0 { + eprintln!("❌ ERROR: --job-interval must be >= 0"); + std::process::exit(1); + } + // Create CPU engine let cpu_engine: Option> = if effective_cpu_workers > 0 { Some(Arc::new(engine_cpu::FastCpuEngine::new(cpu_batch_size))) @@ -248,28 +308,55 @@ async fn run_benchmark( None }; + if job_interval > 0.0 { + run_benchmark_with_jobs( + cpu_engine, + gpu_engine, + effective_cpu_workers, + effective_gpu_devices, + duration, + job_interval, + difficulty, + ) + .await; + } else { + run_benchmark_sustained( + cpu_engine, + gpu_engine, + effective_cpu_workers, + effective_gpu_devices, + gpu_batch_size, + cpu_batch_size, + duration, + ) + .await; + } +} + +/// Continuous hashing on one header (difficulty MAX). Measures peak sustained H/s. +async fn run_benchmark_sustained( + cpu_engine: Option>, + gpu_engine: Option>, + effective_cpu_workers: usize, + effective_gpu_devices: usize, + gpu_batch_size: u32, + cpu_batch_size: u64, + duration: u64, +) { + let total_workers = effective_cpu_workers + effective_gpu_devices; let cancel_flag = Arc::new(AtomicBool::new(false)); let benchmark_start = Instant::now(); - // Random header hash for benchmark - let mut header = [0u8; 32]; - rand::rng().fill_bytes(&mut header); - // Bitcoin-style: target = MAX / difficulty. MAX → target 1 → essentially never finds. + let header = random_header(); let difficulty = U512::MAX; - let ref_engine = cpu_engine.as_ref().or(gpu_engine.as_ref()).unwrap(); let ctx = ref_engine.prepare_context(header, difficulty); - println!("⛏️ Starting benchmark..."); + println!("⛏️ Starting sustained benchmark..."); - // Spawn worker threads let mut handles = Vec::new(); - let total_hashes = Arc::new(std::sync::Mutex::new(0u64)); + let total_hashes = Arc::new(Mutex::new(0u64)); - // Range per search_range call must be >= engine batch size; otherwise the GPU - // clamps this_batch_size to the range and --gpu-batch-size A/B is inert. - // One full batch per call also lets the wall-clock cancel + progress ticker arm - // between batches (cancel is only checked between GPU dispatches). let cpu_chunk = cpu_batch_size.max(10_000); let gpu_chunk = gpu_batch_size as u64; @@ -290,7 +377,7 @@ async fn run_benchmark( let mut nonce = U512::from(worker_id as u64).saturating_mul(stride); loop { - if cancel.load(std::sync::atomic::Ordering::Relaxed) { + if cancel.load(Ordering::Relaxed) { break; } @@ -331,14 +418,178 @@ async fn run_benchmark( handles.push(handle); } - // Progress updates + progress_and_join( + handles, + cancel_flag, + total_hashes, + benchmark_start, + duration, + None, + None, + ) + .await; +} + +/// Serve-like path: open-ended search, JobId cancel, periodic NewJob, idle after Found. +async fn run_benchmark_with_jobs( + cpu_engine: Option>, + gpu_engine: Option>, + effective_cpu_workers: usize, + effective_gpu_devices: usize, + duration: u64, + job_interval: f64, + difficulty: U512, +) { + let total_workers = effective_cpu_workers + effective_gpu_devices; + let stop_flag = Arc::new(AtomicBool::new(false)); + let current_job_id = Arc::new(AtomicU64::new(0)); + let job_ctx: Arc> = Arc::new(RwLock::new(JobContext::new( + random_header(), + difficulty, + ))); + let total_hashes = Arc::new(Mutex::new(0u64)); + let finds = Arc::new(AtomicU64::new(0)); + let jobs_started = Arc::new(AtomicU64::new(0)); + + // Publish job 1 (ctx before id bump so workers never see a stale header). + { + *job_ctx.write().unwrap() = JobContext::new(random_header(), difficulty); + let id = current_job_id.fetch_add(1, Ordering::SeqCst) + 1; + jobs_started.store(id, Ordering::Relaxed); + } + + println!("⛏️ Starting job-simulation benchmark..."); + + let mut handles = Vec::new(); + let benchmark_start = Instant::now(); + + for worker_id in 0..total_workers { + let engine = if worker_id < effective_cpu_workers { + cpu_engine.as_ref().unwrap().clone() + } else { + gpu_engine.as_ref().unwrap().clone() + }; + + let stop = stop_flag.clone(); + let job_id_counter = current_job_id.clone(); + let job_ctx = job_ctx.clone(); + let hashes = total_hashes.clone(); + let finds = finds.clone(); + + let handle = thread::spawn(move || { + let stride = U512::from(1_000_000_000_000u64); + let worker_start = U512::from(worker_id as u64).saturating_mul(stride); + + loop { + if stop.load(Ordering::Relaxed) { + break; + } + + let my_job_id = job_id_counter.load(Ordering::SeqCst); + if my_job_id == 0 { + thread::sleep(Duration::from_millis(1)); + continue; + } + + let ctx = job_ctx.read().unwrap().clone(); + let cancel_check = JobIdCancelCheck { + current_job_id: &job_id_counter, + my_job_id, + }; + + // Match serve: open-ended range; engine batches internally. + let range = EngineRange { + start: worker_start, + end: U512::MAX, + }; + + let result = engine.search_range(&ctx, range, &cancel_check); + + match result { + engine_cpu::EngineStatus::Found { hash_count, .. } => { + *hashes.lock().unwrap() += hash_count; + finds.fetch_add(1, Ordering::Relaxed); + // Serve parks the GPU until the next NewJob after a find. + while !stop.load(Ordering::Relaxed) + && job_id_counter.load(Ordering::SeqCst) == my_job_id + { + thread::sleep(Duration::from_millis(1)); + } + } + engine_cpu::EngineStatus::Cancelled { hash_count } + | engine_cpu::EngineStatus::Exhausted { hash_count } => { + *hashes.lock().unwrap() += hash_count; + } + engine_cpu::EngineStatus::DeviceLost { hash_count } => { + *hashes.lock().unwrap() += hash_count; + break; + } + engine_cpu::EngineStatus::Running { .. } => {} + } + } + + engine_gpu::GpuEngine::clear_worker_resources(); + }); + + handles.push(handle); + } + + // Job feeder (simulated node) + let feeder_stop = stop_flag.clone(); + let feeder_job_id = current_job_id.clone(); + let feeder_ctx = job_ctx.clone(); + let feeder_jobs = jobs_started.clone(); + let feeder = thread::spawn(move || { + let interval = Duration::from_secs_f64(job_interval); + while !feeder_stop.load(Ordering::Relaxed) { + thread::sleep(interval); + if feeder_stop.load(Ordering::Relaxed) { + break; + } + *feeder_ctx.write().unwrap() = JobContext::new(random_header(), difficulty); + let id = feeder_job_id.fetch_add(1, Ordering::SeqCst) + 1; + feeder_jobs.store(id, Ordering::Relaxed); + log::info!("simulated NewJob id={id}"); + } + }); + + let stats = Some((finds.clone(), jobs_started.clone())); + progress_and_join( + handles, + stop_flag.clone(), + total_hashes, + benchmark_start, + duration, + stats, + Some(current_job_id.clone()), + ) + .await; + + stop_flag.store(true, Ordering::Relaxed); + current_job_id.fetch_add(1, Ordering::SeqCst); + let _ = feeder.join(); +} + +async fn progress_and_join( + handles: Vec>, + stop_flag: Arc, + total_hashes: Arc>, + benchmark_start: Instant, + duration: u64, + job_stats: Option<(Arc, Arc)>, + job_id_to_bump: Option>, +) { let mut last_update = Instant::now(); loop { tokio::time::sleep(Duration::from_millis(100)).await; if benchmark_start.elapsed() >= Duration::from_secs(duration) { - cancel_flag.store(true, std::sync::atomic::Ordering::Relaxed); + stop_flag.store(true, Ordering::Relaxed); + // Cancel in-flight GPU batches (JobIdCancelCheck) and wake Found-waiters. + if let Some(ref job_id) = job_id_to_bump { + job_id.fetch_add(1, Ordering::SeqCst); + } break; } @@ -347,13 +598,22 @@ async fn run_benchmark( let elapsed = benchmark_start.elapsed().as_secs_f64(); if current > 0 { let rate = current as f64 / elapsed; - println!("⏱️ {:.1}s - {} H/s", elapsed, format_hash_rate(rate)); + if let Some((finds, jobs)) = &job_stats { + println!( + "⏱️ {:.1}s - {} H/s (jobs={}, finds={})", + elapsed, + format_hash_rate(rate), + jobs.load(Ordering::Relaxed), + finds.load(Ordering::Relaxed) + ); + } else { + println!("⏱️ {:.1}s - {} H/s", elapsed, format_hash_rate(rate)); + } } last_update = Instant::now(); } } - // Wait for threads for handle in handles { let _ = handle.join(); } @@ -368,10 +628,9 @@ async fn run_benchmark( println!("Total time: {:.2}s", total_elapsed.as_secs_f64()); println!("Total hashes: {}", final_hashes); println!("Average rate: {} H/s", format_hash_rate(avg_rate)); - - if total_workers > 1 { - let per_worker = avg_rate / total_workers as f64; - println!("Per-worker: {} H/s", format_hash_rate(per_worker)); + if let Some((finds, jobs)) = job_stats { + println!("Jobs started: {}", jobs.load(Ordering::Relaxed)); + println!("Solutions found: {}", finds.load(Ordering::Relaxed)); } println!("✅ Benchmark completed!"); diff --git a/gpu-bench/README.md b/gpu-bench/README.md index 543b1a6..2a9f2af 100644 --- a/gpu-bench/README.md +++ b/gpu-bench/README.md @@ -23,8 +23,9 @@ download — hardware benches use `quantus-miner benchmark` only. ```bash ./remote-run.sh --provider runpod --cost-per-hour 0.69 --duration 30 -# default batch sizes: 1M 4M 8M 16M (one CSV row each, notes include batch=N) -./remote-run.sh --cost-per-hour 0.39 --batch-sizes "1000000 16777216" +# default: batch sizes 256K 512K 1M 4M, --job-interval 2 (simulated NewJob) +./remote-run.sh --cost-per-hour 0.39 --job-interval 0 # sustained peak H/s +./remote-run.sh --cost-per-hour 0.39 --difficulty max # cancel-only churn ``` Or call record directly if the miner is already built: @@ -32,9 +33,13 @@ Or call record directly if the miner is already built: ```bash MINER_BIN=./bin/quantus-miner ./record.sh --benchmark \ --provider runpod --cost-per-hour 0.42 \ - --batch-sizes "1000000 4194304 16777216" --duration 30 + --job-interval 2 --batch-sizes "262144 524288 1000000" --duration 30 ``` +`benchmark --job-interval N` rotates a random header every N seconds (like node +`NewJob`), with default difficulty `10000000`. After a find, workers idle until +the next job — same cliff as `serve`. + Local full stack (node + serve) is still available via `./setup.sh --dev` + `./record.sh --live` when you need end-to-end mining, not just hardware H/s. diff --git a/gpu-bench/batch-tune.sh b/gpu-bench/batch-tune.sh index 20a523f..809c305 100755 --- a/gpu-bench/batch-tune.sh +++ b/gpu-bench/batch-tune.sh @@ -13,25 +13,27 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORK_DIR="${WORK_DIR:-/workspace/quantus-gpu-bench}" BIN_DIR="${BIN_DIR:-${WORK_DIR}/bin}" OUT_CSV="${OUT_CSV:-${WORK_DIR}/batch-tune.csv}" -SIZES="${SIZES:-1000000 4194304 8388608 16777216}" +SIZES="${SIZES:-262144 524288 1000000 4194304}" DURATION="${DURATION:-30}" GPU_DEVICES="${GPU_DEVICES:-1}" CPU_WORKERS="${CPU_WORKERS:-0}" +JOB_INTERVAL="${JOB_INTERVAL:-2}" +DIFFICULTY="${DIFFICULTY:-}" MINER_BIN="${MINER_BIN:-}" usage() { cat <<'EOF' Usage: ./batch-tune.sh [options] - --sizes "N N N" batch sizes to try (default: 1M 4M 8M 16M) + --sizes "N N N" batch sizes to try (default: 256K 512K 1M 4M) --duration SECONDS per-size benchmark window (default: 30) + --job-interval SEC simulated NewJob period (default: 2; 0 = sustained) + --difficulty DEC difficulty for job sim (or max) --gpu-devices N default 1 --cpu-workers N default 0 (GPU-only) --miner-bin PATH override miner binary --out PATH results CSV (default: $WORK_DIR/batch-tune.csv) --help - -Requires a miner built with the gpu_chunk=batch_size CLI fix (illuzen/gpu-bench). EOF } @@ -39,6 +41,8 @@ while [[ $# -gt 0 ]]; do case "$1" in --sizes) SIZES="$2"; shift 2 ;; --duration) DURATION="$2"; shift 2 ;; + --job-interval) JOB_INTERVAL="$2"; shift 2 ;; + --difficulty) DIFFICULTY="$2"; shift 2 ;; --gpu-devices) GPU_DEVICES="$2"; shift 2 ;; --cpu-workers) CPU_WORKERS="$2"; shift 2 ;; --miner-bin) MINER_BIN="$2"; shift 2 ;; @@ -104,12 +108,12 @@ GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head echo "miner: ${MINER_BIN}" echo "gpu: ${GPU_NAME}" echo "sizes: ${SIZES}" -echo "window: ${DURATION}s gpu_devices=${GPU_DEVICES} cpu_workers=${CPU_WORKERS}" +echo "window: ${DURATION}s gpu_devices=${GPU_DEVICES} cpu_workers=${CPU_WORKERS} job_interval=${JOB_INTERVAL}" echo mkdir -p "$(dirname "${OUT_CSV}")" if [[ ! -f "${OUT_CSV}" ]]; then - echo "timestamp,gpu_name,batch_size,duration_s,hashrate_hs,avg_util_pct,notes" >"${OUT_CSV}" + echo "timestamp,gpu_name,batch_size,duration_s,hashrate_hs,avg_util_pct,job_interval,notes" >"${OUT_CSV}" fi printf "%-12s %-14s %-10s\n" "batch_size" "hashrate" "avg_util%" @@ -126,13 +130,22 @@ for bs in ${SIZES}; do ) >"${util_log}" & util_pid=$! + bench_cmd=( + "${MINER_BIN}" benchmark + --gpu-devices "${GPU_DEVICES}" + --cpu-workers "${CPU_WORKERS}" + --gpu-batch-size "${bs}" + --duration "${DURATION}" + ) + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + bench_cmd+=(--job-interval "${JOB_INTERVAL}") + fi + if [[ -n "${DIFFICULTY}" ]]; then + bench_cmd+=(--difficulty "${DIFFICULTY}") + fi + set +e - "${MINER_BIN}" benchmark \ - --gpu-devices "${GPU_DEVICES}" \ - --cpu-workers "${CPU_WORKERS}" \ - --gpu-batch-size "${bs}" \ - --duration "${DURATION}" \ - >"${log}" 2>&1 + "${bench_cmd[@]}" >"${log}" 2>&1 rc=$? set -e @@ -157,7 +170,7 @@ for bs in ${SIZES}; do fi ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - echo "${ts},${GPU_NAME},${bs},${DURATION},${rate},${util},${notes}" >>"${OUT_CSV}" + echo "${ts},${GPU_NAME},${bs},${DURATION},${rate},${util},${JOB_INTERVAL},${notes}" >>"${OUT_CSV}" printf "%-12s %-14s %-10s\n" "${bs}" "${rate_disp}" "${util}%" rm -f "${log}" "${util_log}" diff --git a/gpu-bench/gpus.example.txt b/gpu-bench/gpus.example.txt index 21b8b94..a6a0c13 100644 --- a/gpu-bench/gpus.example.txt +++ b/gpu-bench/gpus.example.txt @@ -1,7 +1,10 @@ # One RunPod gpuTypeId per line (from console / API). # Lines starting with # are ignored. #NVIDIA GeForce RTX 4090 -NVIDIA GeForce RTX 3090 +#NVIDIA GeForce RTX 3080 +#NVIDIA GeForce RTX 3090 #NVIDIA RTX A5000 -NVIDIA L4 -NVIDIA RTX A4000 +#NVIDIA L4 +#NVIDIA RTX A4000 +NVIDIA GeForce RTX 5090 + diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh index 7e3371a..1fcc6f3 100755 --- a/gpu-bench/record.sh +++ b/gpu-bench/record.sh @@ -19,6 +19,8 @@ GPU_DEVICES_FLAG="" GPU_BATCH_SIZE="${GPU_BATCH_SIZE:-}" # Space-separated list; when set with --benchmark, runs one row per size. BATCH_SIZES="${BATCH_SIZES:-}" +JOB_INTERVAL="${JOB_INTERVAL:-0}" +DIFFICULTY="${DIFFICULTY:-}" usage() { cat <<'EOF' @@ -33,6 +35,8 @@ Options: --gpu-devices N GPUs for --benchmark (default: GPU_DEVICES or 1) --gpu-batch-size N Single GPU batch size for --benchmark --batch-sizes "N N" Sweep several batch sizes (one CSV row each) + --job-interval SECONDS Simulated NewJob period (0 = sustained; default 0) + --difficulty DEC|max Difficulty for job simulation --notes TEXT Optional notes column --dry-run Print the CSV row but do not append -h, --help Show this help @@ -40,7 +44,8 @@ Options: Examples: ./record.sh --provider vast.ai --cost-per-hour 0.35 ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 30 - ./record.sh --benchmark --batch-sizes "1000000 4194304 16777216" --provider runpod --cost-per-hour 0.39 + ./record.sh --benchmark --job-interval 2 --batch-sizes "262144 524288 1000000" \ + --provider runpod --cost-per-hour 0.39 EOF } @@ -394,25 +399,28 @@ run_benchmark_once() { batch_label="${batch_size}" fi - echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}, batch=${batch_label}) ..." >&2 + local bench_cmd=( + "${miner_bin}" benchmark + --cpu-workers 0 + --gpu-devices "${gpu_devices}" + --duration "${DURATION}" + ) + if [[ -n "${batch_size}" ]]; then + bench_cmd+=(--gpu-batch-size "${batch_size}") + fi + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + bench_cmd+=(--job-interval "${JOB_INTERVAL}") + fi + if [[ -n "${DIFFICULTY}" ]]; then + bench_cmd+=(--difficulty "${DIFFICULTY}") + fi + + echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}, batch=${batch_label}, job_interval=${JOB_INTERVAL}) ..." >&2 sample_util_during "${util_file}" "${DURATION}" & local sampler_pid=$! set +e - if [[ -n "${batch_size}" ]]; then - "${miner_bin}" benchmark \ - --cpu-workers 0 \ - --gpu-devices "${gpu_devices}" \ - --gpu-batch-size "${batch_size}" \ - --duration "${DURATION}" \ - >"${bench_log}" 2>&1 - else - "${miner_bin}" benchmark \ - --cpu-workers 0 \ - --gpu-devices "${gpu_devices}" \ - --duration "${DURATION}" \ - >"${bench_log}" 2>&1 - fi + "${bench_cmd[@]}" >"${bench_log}" 2>&1 local bench_rc=$? set -e @@ -485,6 +493,16 @@ run_benchmark() { if [[ -n "${bs}" ]]; then note_extra="batch=${bs}" fi + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + if [[ -n "${note_extra}" ]]; then + note_extra="${note_extra};job_interval=${JOB_INTERVAL}" + else + note_extra="job_interval=${JOB_INTERVAL}" + fi + if [[ -n "${DIFFICULTY}" ]]; then + note_extra="${note_extra};difficulty=${DIFFICULTY}" + fi + fi if run_benchmark_once "${miner_bin}" "${gpu_devices}" "${bs}" "${note_extra}"; then any_ok=1 fi @@ -531,6 +549,14 @@ while [[ $# -gt 0 ]]; do BATCH_SIZES="${2:-}" shift 2 ;; + --job-interval) + JOB_INTERVAL="${2:-}" + shift 2 + ;; + --difficulty) + DIFFICULTY="${2:-}" + shift 2 + ;; --notes) NOTES="${2:-}" shift 2 diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh index a07ee49..2e65a67 100755 --- a/gpu-bench/remote-run.sh +++ b/gpu-bench/remote-run.sh @@ -22,8 +22,12 @@ PROVIDER="${PROVIDER:-runpod}" COST_PER_HOUR="${COST_PER_HOUR:-}" DURATION="${DURATION:-30}" GPU_DEVICES="${GPU_DEVICES:-1}" -# Default sweep: 1M → 16M. Override with --batch-sizes or BATCH_SIZES. -BATCH_SIZES="${BATCH_SIZES:-1000000 4194304 8388608 16777216}" +# Downward + 1M + one larger step. Override with --batch-sizes or BATCH_SIZES. +BATCH_SIZES="${BATCH_SIZES:-262144 524288 1000000 4194304}" +# Simulate NewJob churn (seconds). 0 = sustained peak H/s (no job switches). +JOB_INTERVAL="${JOB_INTERVAL:-2}" +# Optional decimal difficulty for job sim (default in miner: 10000000). Use "max" for cancel-only. +DIFFICULTY="${DIFFICULTY:-}" NOTES="${NOTES:-}" usage() { @@ -34,7 +38,9 @@ Usage: ./remote-run.sh [options] --cost-per-hour USD required for hash_per_dollar column --duration SECONDS per-batch-size benchmark window (default: 30) --gpu-devices N default 1 - --batch-sizes "N N N" GPU batch sizes to sweep (default: 1M 4M 8M 16M) + --batch-sizes "N N N" GPU batch sizes (default: 256K 512K 1M 4M) + --job-interval SECONDS simulated NewJob period (default: 2; 0 = sustained) + --difficulty DEC|max PoW difficulty for job sim (miner default 1e7) --notes TEXT --miner-branch REF git branch/tag/commit to build (default: illuzen/gpu-bench) --miner-repo URL git remote (default: Quantus-Network/quantus-miner) @@ -46,9 +52,10 @@ Usage: ./remote-run.sh [options] Builds quantus-miner from git (or downloads a release), sets up NVIDIA Vulkan, runs `quantus-miner benchmark` for each batch size (no node), appends rows to -results.csv (notes include batch=N). +results.csv (notes include batch=N; job_interval=… when set). -Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, BATCH_SIZES +Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, + BATCH_SIZES, JOB_INTERVAL, DIFFICULTY EOF } @@ -66,6 +73,8 @@ while [[ $# -gt 0 ]]; do --duration) DURATION="${2:-}"; shift 2 ;; --gpu-devices) GPU_DEVICES="${2:-}"; shift 2 ;; --batch-sizes) BATCH_SIZES="${2:-}"; shift 2 ;; + --job-interval) JOB_INTERVAL="${2:-}"; shift 2 ;; + --difficulty) DIFFICULTY="${2:-}"; shift 2 ;; --notes) NOTES="${2:-}"; shift 2 ;; --miner-branch) MINER_BRANCH="${2:-}"; shift 2 ;; --miner-repo) MINER_REPO="${2:-}"; shift 2 ;; @@ -477,7 +486,14 @@ if [[ -n "${NOTES}" ]]; then fi cd "${WORK_DIR}" -echo "Batch-size sweep: ${BATCH_SIZES} (${DURATION}s each)" +echo "Batch-size sweep: ${BATCH_SIZES} (${DURATION}s each, job_interval=${JOB_INTERVAL})" +EXTRA_ARGS=() +if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + EXTRA_ARGS+=(--job-interval "${JOB_INTERVAL}") +fi +if [[ -n "${DIFFICULTY}" ]]; then + EXTRA_ARGS+=(--difficulty "${DIFFICULTY}") +fi export MINER_BIN env \ VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ @@ -492,6 +508,7 @@ env \ --duration "${DURATION}" \ --gpu-devices "${GPU_DEVICES}" \ --batch-sizes "${BATCH_SIZES}" \ + "${EXTRA_ARGS[@]}" \ "${NOTE_ARGS[@]}" echo "Done. Results: ${RESULTS_CSV}" diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv index 976001e..0eaf64e 100644 --- a/gpu-bench/results.csv +++ b/gpu-bench/results.csv @@ -65,3 +65,7 @@ timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_ 2026-08-03T14:34:18Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,13939195.746095,92.86,7.39,0.0020527778,6790406588.084168,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 2026-08-03T14:34:49Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,11441547.837658,93.27,7.39,0.0020527778,5573690421.592531,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 2026-08-03T14:35:21Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,10758073.741584,93.07,7.39,0.0020527778,5240739576.414398,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T15:00:16Z,runpod,NVIDIA L4,23034,60,580.159.04,9066755.230820,93.33,0.39,0.0001083333,83693125207.569229,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T15:00:46Z,runpod,NVIDIA L4,23034,60,580.159.04,8044630.687831,100.00,0.39,0.0001083333,74258129426.132309,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T15:01:17Z,runpod,NVIDIA L4,23034,60,580.159.04,7447958.434725,100.00,0.39,0.0001083333,68750385551.307678,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T15:01:48Z,runpod,NVIDIA L4,23034,60,580.159.04,7134570.101407,100.00,0.39,0.0001083333,65857570166.833839,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh index 0ba1057..1b9db7c 100755 --- a/gpu-bench/runpod-shell.sh +++ b/gpu-bench/runpod-shell.sh @@ -74,10 +74,11 @@ print_ssh_hint() { echo "On the pod:" cat <