diff --git a/Cargo.lock b/Cargo.lock index cd40224..09a4473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,6 +1514,7 @@ dependencies = [ "metrics", "miner-service", "num_cpus", + "pow-core", "primitive-types 0.13.1", "rand 0.9.2", "tokio", diff --git a/README.md b/README.md index ff54f48..7fba24e 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,26 @@ cargo build -p miner-cli --release - **Linux**: `nvidia-smi` (NVIDIA) or `radeontop` (AMD) - **Windows**: Task Manager GPU tab +### Cloud GPU bench + hardware spreadsheet + +Provider-agnostic tooling to run the miner on a rented NVIDIA GPU and append +hashrate / utilization / VRAM / hash_per_dollar rows to a CSV: + +```bash +cd gpu-bench +./setup.sh --dev # local --dev node + GPU miner (no sync / no rewards hash) +./record.sh --provider runpod --cost-per-hour 0.69 +``` + +RunPod multi-GPU sweep (REST API + SSH, release binaries, Prometheus → CSV): + +```bash +export RUNPOD_API_KEY=... +./runpod-sweep.sh --gpus-file gpus.example.txt +``` + +See [`gpu-bench/README.md`](gpu-bench/README.md). + ## Examples ```bash diff --git a/crates/engine-gpu/src/gpu_tiers.rs b/crates/engine-gpu/src/gpu_tiers.rs index 3089d73..c5e897a 100644 --- a/crates/engine-gpu/src/gpu_tiers.rs +++ b/crates/engine-gpu/src/gpu_tiers.rs @@ -41,35 +41,185 @@ impl CompiledGpuTier { // Use word boundaries (\b) to avoid substring issues like "550" matching "5500" const NVIDIA_TIERS: &[GpuTier] = &[ - // Blackwell (RTX 50 series) + // --- Workstation / datacenter first (avoid "rtx 40" / "rtx 50" substring traps) --- + // Blackwell PRO (e.g. "NVIDIA RTX PRO 6000 Blackwell Server Edition") GpuTier { - pattern: r"\b50[89]0\b", + pattern: r"rtx pro 6000|pro 6000 blackwell", + name: "NVIDIA RTX PRO 6000 (Blackwell)", + workgroup_divisor: 6, + min_workgroups: 5120, + }, + GpuTier { + pattern: r"rtx pro 5000|pro 5000 blackwell", + name: "NVIDIA RTX PRO 5000 (Blackwell)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"rtx pro 4500|pro 4500 blackwell", + name: "NVIDIA RTX PRO 4500 (Blackwell)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"rtx pro 4000|pro 4000 blackwell|rtx pro", + name: "NVIDIA RTX PRO (Blackwell)", + workgroup_divisor: 9, + min_workgroups: 3584, + }, + // Ada Lovelace workstation (must precede consumer `\brtx 50xx` / `\brtx 40xx`) + GpuTier { + pattern: r"rtx 6000 ada|6000 ada generation", + name: "NVIDIA RTX 6000 Ada (Workstation)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"rtx 5000 ada|5000 ada generation", + name: "NVIDIA RTX 5000 Ada (Workstation)", + workgroup_divisor: 9, + min_workgroups: 3584, + }, + GpuTier { + pattern: r"rtx 4000 ada|4000 ada generation|4000 sff ada", + name: "NVIDIA RTX 4000 Ada (Workstation)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx 2000 ada|2000 ada generation", + name: "NVIDIA RTX 2000 Ada (Workstation)", + workgroup_divisor: 14, + min_workgroups: 1536, + }, + // Datacenter — largest / newest first. L40S before L40 before L4. + GpuTier { + pattern: r"\bb300\b|\bb200\b", + name: "NVIDIA B200/B300 (Blackwell DC)", + workgroup_divisor: 6, + min_workgroups: 5120, + }, + GpuTier { + pattern: r"\bh200\b", + name: "NVIDIA H200 (Hopper)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"\bh100\b", + name: "NVIDIA H100 (Hopper)", + workgroup_divisor: 7, + min_workgroups: 4608, + }, + GpuTier { + pattern: r"\ba100\b", + name: "NVIDIA A100 (Ampere DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\bl40s\b", + name: "NVIDIA L40S (Ada DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\bl40\b", + name: "NVIDIA L40 (Ada DC)", + workgroup_divisor: 8, + min_workgroups: 4096, + }, + GpuTier { + pattern: r"\ba40\b", + name: "NVIDIA A40 (Ampere DC)", + workgroup_divisor: 10, + min_workgroups: 3072, + }, + GpuTier { + pattern: r"\bl4\b", + name: "NVIDIA L4 (Ada DC)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"\ba30\b", + name: "NVIDIA A30 (Ampere DC)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + // Ampere workstation (RTX Axxxx) — split by class; was one shared pro bucket + GpuTier { + pattern: r"rtx a6000|\ba6000\b", + name: "NVIDIA RTX A6000 (Ampere Pro)", + workgroup_divisor: 10, + min_workgroups: 3072, + }, + GpuTier { + pattern: r"rtx a5000|\ba5000\b", + name: "NVIDIA RTX A5000 (Ampere Pro)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx a4500|\ba4500\b", + name: "NVIDIA RTX A4500 (Ampere Pro)", + workgroup_divisor: 11, + min_workgroups: 2560, + }, + GpuTier { + pattern: r"rtx a4000|\ba4000\b", + name: "NVIDIA RTX A4000 (Ampere Pro)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + GpuTier { + pattern: r"rtx a2000|\ba2000\b", + name: "NVIDIA RTX A2000 (Ampere Pro)", + workgroup_divisor: 16, + min_workgroups: 1024, + }, + GpuTier { + pattern: r"rtx a\d{4}", + name: "NVIDIA RTX A-series (Ampere Pro)", + workgroup_divisor: 12, + min_workgroups: 2048, + }, + GpuTier { + pattern: r"tesla|\bv100\b|quadro", + name: "NVIDIA Tesla/Quadro (Legacy Pro)", + workgroup_divisor: 14, + min_workgroups: 1536, + }, + // --- GeForce consumer --- + // Blackwell (RTX 50 series). Do NOT use bare "rtx 50" (matches "RTX 5000 Ada"). + GpuTier { + pattern: r"\b50[89]0\b|\brtx 50[89]0\b", name: "NVIDIA RTX 50 Flagship (Blackwell)", workgroup_divisor: 6, min_workgroups: 5120, }, GpuTier { - pattern: r"\b50[67]0\b|rtx 50", + pattern: r"\b50[67]0\b|\brtx 50[67]0\b", name: "NVIDIA RTX 50 (Blackwell)", workgroup_divisor: 7, min_workgroups: 4608, }, - // Ada Lovelace (RTX 40 series) + // Ada Lovelace (RTX 40 series). Do NOT use bare "rtx 40" (matches "RTX 4000 Ada"). GpuTier { - pattern: r"\b40[89]0\b", + pattern: r"\b40[89]0\b|\brtx 40[89]0\b", name: "NVIDIA RTX 40 Flagship (Ada)", workgroup_divisor: 8, min_workgroups: 4096, }, GpuTier { - pattern: r"\b40[67]0\b|rtx 40", + pattern: r"\b40[67]0\b|\brtx 40[67]0\b", name: "NVIDIA RTX 40 (Ada)", workgroup_divisor: 10, min_workgroups: 3072, }, // Ampere/Turing (RTX 30/20 series) GpuTier { - pattern: r"\b30[5-9]0\b|\b20[6-8]0\b|rtx 30|rtx 20", + pattern: r"\b30[5-9]0\b|\b20[6-8]0\b|\brtx 30[5-9]0\b|\brtx 20[6-8]0\b", name: "NVIDIA RTX 30/20 (Ampere/Turing)", workgroup_divisor: 12, min_workgroups: 2048, @@ -121,13 +271,6 @@ const NVIDIA_TIERS: &[GpuTier] = &[ workgroup_divisor: 28, min_workgroups: 256, }, - // Professional - GpuTier { - pattern: r"quadro|rtx a\d|tesla|\ba100\b|\bh100\b|\bl4\b", - name: "NVIDIA Quadro/Professional", - workgroup_divisor: 10, - min_workgroups: 2560, - }, ]; const AMD_TIERS: &[GpuTier] = &[ @@ -648,6 +791,63 @@ mod tests { assert_eq!(tier.name, "NVIDIA GTX 16/10 (Turing/Pascal)"); } + #[test] + fn test_nvidia_ada_workstation_not_confused_with_geforce() { + // Must not match bare "rtx 40" / "rtx 50" GeForce tiers. + let tier = detect_gpu_tier("NVIDIA RTX 4000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 4000 Ada (Workstation)"); + assert!(!tier.is_fallback); + + let tier = detect_gpu_tier("NVIDIA RTX 5000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 5000 Ada (Workstation)"); + assert!(!tier.is_fallback); + + let tier = detect_gpu_tier("NVIDIA RTX 6000 Ada Generation", 0x10DE, false); + assert_eq!(tier.name, "NVIDIA RTX 6000 Ada (Workstation)"); + assert!(!tier.is_fallback); + } + + #[test] + fn test_nvidia_ampere_pro_split_by_class() { + let a2000 = detect_gpu_tier("NVIDIA RTX A2000", 0x10DE, false); + assert_eq!(a2000.name, "NVIDIA RTX A2000 (Ampere Pro)"); + assert_eq!(a2000.workgroup_divisor, 16); + + let a4500 = detect_gpu_tier("NVIDIA RTX A4500", 0x10DE, false); + assert_eq!(a4500.name, "NVIDIA RTX A4500 (Ampere Pro)"); + assert_eq!(a4500.workgroup_divisor, 11); + + let a6000 = detect_gpu_tier("NVIDIA RTX A6000", 0x10DE, false); + assert_eq!(a6000.name, "NVIDIA RTX A6000 (Ampere Pro)"); + assert_eq!(a6000.workgroup_divisor, 10); + + // Smaller cards should use a more conservative (higher) divisor. + assert!(a2000.workgroup_divisor > a4500.workgroup_divisor); + assert!(a4500.workgroup_divisor >= a6000.workgroup_divisor); + } + + #[test] + fn test_nvidia_datacenter_l40_before_l4() { + let l4 = detect_gpu_tier("NVIDIA L4", 0x10DE, false); + assert_eq!(l4.name, "NVIDIA L4 (Ada DC)"); + + let l40 = detect_gpu_tier("NVIDIA L40", 0x10DE, false); + assert_eq!(l40.name, "NVIDIA L40 (Ada DC)"); + + let l40s = detect_gpu_tier("NVIDIA L40S", 0x10DE, false); + assert_eq!(l40s.name, "NVIDIA L40S (Ada DC)"); + + let h100 = detect_gpu_tier("NVIDIA H100 80GB HBM3", 0x10DE, false); + assert_eq!(h100.name, "NVIDIA H100 (Hopper)"); + + let pro = detect_gpu_tier( + "NVIDIA RTX PRO 6000 Blackwell Server Edition", + 0x10DE, + false, + ); + assert_eq!(pro.name, "NVIDIA RTX PRO 6000 (Blackwell)"); + } + #[test] fn test_amd_rdna_vs_polaris() { // RDNA 1 - should NOT match Polaris diff --git a/crates/miner-cli/Cargo.toml b/crates/miner-cli/Cargo.toml index f57af9c..f3ee78d 100644 --- a/crates/miner-cli/Cargo.toml +++ b/crates/miner-cli/Cargo.toml @@ -14,6 +14,7 @@ env_logger = { workspace = true } log = { workspace = true } engine-cpu = { path = "../engine-cpu" } engine-gpu = { path = "../engine-gpu" } +pow-core = { path = "../pow-core" } primitive-types = { workspace = true } rand = { workspace = true } num_cpus = { workspace = true } diff --git a/crates/miner-cli/src/main.rs b/crates/miner-cli/src/main.rs index 8ab8568..ba89dfc 100644 --- a/crates/miner-cli/src/main.rs +++ b/crates/miner-cli/src/main.rs @@ -1,16 +1,19 @@ use clap::{Parser, Subcommand}; -use engine_cpu::{AtomicBoolCancelCheck, EngineRange, MinerEngine}; +use engine_cpu::{AtomicBoolCancelCheck, EngineRange, JobIdCancelCheck, MinerEngine}; use miner_service::{run, ServiceConfig}; +use pow_core::JobContext; use primitive_types::U512; use rand::RngCore; -use std::sync::atomic::AtomicBool; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; use std::thread; use std::time::{Duration, Instant}; // CLI defaults const DEFAULT_GPU_BATCH_SIZE: u32 = 1_000_000; const DEFAULT_CPU_BATCH_SIZE: u64 = 10_000; +/// Default difficulty when `--job-interval` is set: ~1s to find at 10 MH/s. +const DEFAULT_JOB_DIFFICULTY: u64 = 10_000_000; #[derive(Subcommand, Debug)] enum Command { @@ -85,6 +88,16 @@ enum Command { #[arg(short, long, default_value_t = 10)] duration: u64, + /// Simulate node job churn: every N seconds push a new random header + /// (like NewJob). 0 = sustained single job (default). + #[arg(long = "job-interval", default_value_t = 0.0)] + job_interval: f64, + + /// PoW difficulty for job simulation (decimal). Use "max" for unreachable + /// (cancel-only churn). Default when --job-interval > 0: 10000000. + #[arg(long = "difficulty")] + difficulty: Option, + /// Allow integrated GPUs (APUs) even when discrete GPUs are available #[arg(long = "allow-integrated", env = "MINER_ALLOW_INTEGRATED")] allow_integrated: bool, @@ -161,6 +174,8 @@ async fn main() { gpu_batch_size, cpu_batch_size, duration, + job_interval, + difficulty, allow_integrated, verbose, } => { @@ -171,6 +186,8 @@ async fn main() { gpu_batch_size, cpu_batch_size, duration, + job_interval, + difficulty, allow_integrated, ) .await; @@ -191,15 +208,48 @@ fn init_logger(verbose: bool) { env_logger::init(); } +fn parse_difficulty(raw: Option<&str>, job_interval: f64) -> U512 { + match raw { + None if job_interval > 0.0 => U512::from(DEFAULT_JOB_DIFFICULTY), + None => U512::MAX, + Some(s) if s.eq_ignore_ascii_case("max") => U512::MAX, + Some(s) => match U512::from_dec_str(s) { + Ok(d) if !d.is_zero() => d, + Ok(_) => { + eprintln!("❌ ERROR: --difficulty must be non-zero (or \"max\")"); + std::process::exit(1); + } + Err(_) => { + eprintln!("❌ ERROR: invalid --difficulty '{s}' (decimal or \"max\")"); + std::process::exit(1); + } + }, + } +} + +fn random_header() -> [u8; 32] { + let mut header = [0u8; 32]; + rand::rng().fill_bytes(&mut header); + header +} + async fn run_benchmark( cpu_workers: Option, gpu_devices: Option, gpu_batch_size: u32, cpu_batch_size: u64, duration: u64, + job_interval: f64, + difficulty_arg: Option, allow_integrated: bool, ) { - let effective_cpu_workers = cpu_workers.unwrap_or_else(num_cpus::get); + // When --gpu-devices is set and --cpu-workers is omitted, default to GPU-only + // so hardware A/B numbers aren't polluted by host CPU hashrate. + let effective_cpu_workers = match (cpu_workers, gpu_devices) { + (Some(n), _) => n, + (None, Some(_)) => 0, + (None, None) => num_cpus::get(), + }; // Initialize GPU engine (no throttle for benchmark) let (gpu_engine, effective_gpu_devices) = match miner_service::resolve_gpu_configuration( @@ -216,6 +266,7 @@ async fn run_benchmark( }; let total_workers = effective_cpu_workers + effective_gpu_devices; + let difficulty = parse_difficulty(difficulty_arg.as_deref(), job_interval); println!("🚀 Quantus Miner Benchmark"); println!("=========================="); @@ -225,7 +276,19 @@ async fn run_benchmark( num_cpus::get() ); println!("GPU Devices: {}", effective_gpu_devices); + println!("GPU batch size: {} nonces", gpu_batch_size); + println!("CPU batch size: {} hashes", cpu_batch_size); println!("Duration: {} seconds", duration); + if job_interval > 0.0 { + println!("Job interval: {:.2}s (simulated NewJob)", job_interval); + if difficulty == U512::MAX { + println!("Difficulty: max (cancel-only churn, no finds)"); + } else { + println!("Difficulty: {difficulty}"); + } + } else { + println!("Job interval: off (sustained single job)"); + } println!(); if total_workers == 0 { @@ -233,6 +296,11 @@ async fn run_benchmark( std::process::exit(1); } + if job_interval < 0.0 { + eprintln!("❌ ERROR: --job-interval must be >= 0"); + std::process::exit(1); + } + // Create CPU engine let cpu_engine: Option> = if effective_cpu_workers > 0 { Some(Arc::new(engine_cpu::FastCpuEngine::new(cpu_batch_size))) @@ -240,25 +308,57 @@ async fn run_benchmark( None }; + if job_interval > 0.0 { + run_benchmark_with_jobs( + cpu_engine, + gpu_engine, + effective_cpu_workers, + effective_gpu_devices, + duration, + job_interval, + difficulty, + ) + .await; + } else { + run_benchmark_sustained( + cpu_engine, + gpu_engine, + effective_cpu_workers, + effective_gpu_devices, + gpu_batch_size, + cpu_batch_size, + duration, + ) + .await; + } +} + +/// Continuous hashing on one header (difficulty MAX). Measures peak sustained H/s. +async fn run_benchmark_sustained( + cpu_engine: Option>, + gpu_engine: Option>, + effective_cpu_workers: usize, + effective_gpu_devices: usize, + gpu_batch_size: u32, + cpu_batch_size: u64, + duration: u64, +) { + let total_workers = effective_cpu_workers + effective_gpu_devices; let cancel_flag = Arc::new(AtomicBool::new(false)); let benchmark_start = Instant::now(); - // Random header hash for benchmark - let mut header = [0u8; 32]; - rand::rng().fill_bytes(&mut header); - let difficulty = U512::MAX; // High difficulty - no solutions expected - + let header = random_header(); + let difficulty = U512::MAX; let ref_engine = cpu_engine.as_ref().or(gpu_engine.as_ref()).unwrap(); let ctx = ref_engine.prepare_context(header, difficulty); - println!("⛏️ Starting benchmark..."); + println!("⛏️ Starting sustained benchmark..."); - // Spawn worker threads let mut handles = Vec::new(); - let total_hashes = Arc::new(std::sync::Mutex::new(0u64)); + let total_hashes = Arc::new(Mutex::new(0u64)); - let cpu_chunk = 10_000u64; - let gpu_chunk = 1_000_000u64; + let cpu_chunk = cpu_batch_size.max(10_000); + let gpu_chunk = gpu_batch_size as u64; for worker_id in 0..total_workers { let (engine, nonces_per_batch) = if worker_id < effective_cpu_workers { @@ -274,21 +374,22 @@ async fn run_benchmark( let handle = thread::spawn(move || { let stride = U512::from(1_000_000_000_000u64); - let worker_start = U512::from(worker_id as u64).saturating_mul(stride); - let worker_range = EngineRange { - start: worker_start, - end: worker_start - .saturating_add(U512::from(nonces_per_batch)) - .saturating_sub(U512::from(1u64)), - }; + let mut nonce = U512::from(worker_id as u64).saturating_mul(stride); loop { - if cancel.load(std::sync::atomic::Ordering::Relaxed) { + if cancel.load(Ordering::Relaxed) { break; } + let worker_range = EngineRange { + start: nonce, + end: nonce + .saturating_add(U512::from(nonces_per_batch)) + .saturating_sub(U512::from(1u64)), + }; + let cancel_check = AtomicBoolCancelCheck(&cancel); - let result = engine.search_range(&ctx, worker_range.clone(), &cancel_check); + let result = engine.search_range(&ctx, worker_range, &cancel_check); match result { engine_cpu::EngineStatus::Found { hash_count, .. } @@ -300,11 +401,12 @@ async fn run_benchmark( engine_cpu::EngineStatus::Running { .. } => {} } - // Exit if device is lost if matches!(result, engine_cpu::EngineStatus::DeviceLost { .. }) { break; } + nonce = nonce.saturating_add(U512::from(nonces_per_batch)); + if start.elapsed() >= Duration::from_secs(duration) { break; } @@ -316,14 +418,178 @@ async fn run_benchmark( handles.push(handle); } - // Progress updates + progress_and_join( + handles, + cancel_flag, + total_hashes, + benchmark_start, + duration, + None, + None, + ) + .await; +} + +/// Serve-like path: open-ended search, JobId cancel, periodic NewJob, idle after Found. +async fn run_benchmark_with_jobs( + cpu_engine: Option>, + gpu_engine: Option>, + effective_cpu_workers: usize, + effective_gpu_devices: usize, + duration: u64, + job_interval: f64, + difficulty: U512, +) { + let total_workers = effective_cpu_workers + effective_gpu_devices; + let stop_flag = Arc::new(AtomicBool::new(false)); + let current_job_id = Arc::new(AtomicU64::new(0)); + let job_ctx: Arc> = Arc::new(RwLock::new(JobContext::new( + random_header(), + difficulty, + ))); + let total_hashes = Arc::new(Mutex::new(0u64)); + let finds = Arc::new(AtomicU64::new(0)); + let jobs_started = Arc::new(AtomicU64::new(0)); + + // Publish job 1 (ctx before id bump so workers never see a stale header). + { + *job_ctx.write().unwrap() = JobContext::new(random_header(), difficulty); + let id = current_job_id.fetch_add(1, Ordering::SeqCst) + 1; + jobs_started.store(id, Ordering::Relaxed); + } + + println!("⛏️ Starting job-simulation benchmark..."); + + let mut handles = Vec::new(); + let benchmark_start = Instant::now(); + + for worker_id in 0..total_workers { + let engine = if worker_id < effective_cpu_workers { + cpu_engine.as_ref().unwrap().clone() + } else { + gpu_engine.as_ref().unwrap().clone() + }; + + let stop = stop_flag.clone(); + let job_id_counter = current_job_id.clone(); + let job_ctx = job_ctx.clone(); + let hashes = total_hashes.clone(); + let finds = finds.clone(); + + let handle = thread::spawn(move || { + let stride = U512::from(1_000_000_000_000u64); + let worker_start = U512::from(worker_id as u64).saturating_mul(stride); + + loop { + if stop.load(Ordering::Relaxed) { + break; + } + + let my_job_id = job_id_counter.load(Ordering::SeqCst); + if my_job_id == 0 { + thread::sleep(Duration::from_millis(1)); + continue; + } + + let ctx = job_ctx.read().unwrap().clone(); + let cancel_check = JobIdCancelCheck { + current_job_id: &job_id_counter, + my_job_id, + }; + + // Match serve: open-ended range; engine batches internally. + let range = EngineRange { + start: worker_start, + end: U512::MAX, + }; + + let result = engine.search_range(&ctx, range, &cancel_check); + + match result { + engine_cpu::EngineStatus::Found { hash_count, .. } => { + *hashes.lock().unwrap() += hash_count; + finds.fetch_add(1, Ordering::Relaxed); + // Serve parks the GPU until the next NewJob after a find. + while !stop.load(Ordering::Relaxed) + && job_id_counter.load(Ordering::SeqCst) == my_job_id + { + thread::sleep(Duration::from_millis(1)); + } + } + engine_cpu::EngineStatus::Cancelled { hash_count } + | engine_cpu::EngineStatus::Exhausted { hash_count } => { + *hashes.lock().unwrap() += hash_count; + } + engine_cpu::EngineStatus::DeviceLost { hash_count } => { + *hashes.lock().unwrap() += hash_count; + break; + } + engine_cpu::EngineStatus::Running { .. } => {} + } + } + + engine_gpu::GpuEngine::clear_worker_resources(); + }); + + handles.push(handle); + } + + // Job feeder (simulated node) + let feeder_stop = stop_flag.clone(); + let feeder_job_id = current_job_id.clone(); + let feeder_ctx = job_ctx.clone(); + let feeder_jobs = jobs_started.clone(); + let feeder = thread::spawn(move || { + let interval = Duration::from_secs_f64(job_interval); + while !feeder_stop.load(Ordering::Relaxed) { + thread::sleep(interval); + if feeder_stop.load(Ordering::Relaxed) { + break; + } + *feeder_ctx.write().unwrap() = JobContext::new(random_header(), difficulty); + let id = feeder_job_id.fetch_add(1, Ordering::SeqCst) + 1; + feeder_jobs.store(id, Ordering::Relaxed); + log::info!("simulated NewJob id={id}"); + } + }); + + let stats = Some((finds.clone(), jobs_started.clone())); + progress_and_join( + handles, + stop_flag.clone(), + total_hashes, + benchmark_start, + duration, + stats, + Some(current_job_id.clone()), + ) + .await; + + stop_flag.store(true, Ordering::Relaxed); + current_job_id.fetch_add(1, Ordering::SeqCst); + let _ = feeder.join(); +} + +async fn progress_and_join( + handles: Vec>, + stop_flag: Arc, + total_hashes: Arc>, + benchmark_start: Instant, + duration: u64, + job_stats: Option<(Arc, Arc)>, + job_id_to_bump: Option>, +) { let mut last_update = Instant::now(); loop { tokio::time::sleep(Duration::from_millis(100)).await; if benchmark_start.elapsed() >= Duration::from_secs(duration) { - cancel_flag.store(true, std::sync::atomic::Ordering::Relaxed); + stop_flag.store(true, Ordering::Relaxed); + // Cancel in-flight GPU batches (JobIdCancelCheck) and wake Found-waiters. + if let Some(ref job_id) = job_id_to_bump { + job_id.fetch_add(1, Ordering::SeqCst); + } break; } @@ -332,13 +598,22 @@ async fn run_benchmark( let elapsed = benchmark_start.elapsed().as_secs_f64(); if current > 0 { let rate = current as f64 / elapsed; - println!("⏱️ {:.1}s - {} H/s", elapsed, format_hash_rate(rate)); + if let Some((finds, jobs)) = &job_stats { + println!( + "⏱️ {:.1}s - {} H/s (jobs={}, finds={})", + elapsed, + format_hash_rate(rate), + jobs.load(Ordering::Relaxed), + finds.load(Ordering::Relaxed) + ); + } else { + println!("⏱️ {:.1}s - {} H/s", elapsed, format_hash_rate(rate)); + } } last_update = Instant::now(); } } - // Wait for threads for handle in handles { let _ = handle.join(); } @@ -353,10 +628,9 @@ async fn run_benchmark( println!("Total time: {:.2}s", total_elapsed.as_secs_f64()); println!("Total hashes: {}", final_hashes); println!("Average rate: {} H/s", format_hash_rate(avg_rate)); - - if total_workers > 1 { - let per_worker = avg_rate / total_workers as f64; - println!("Per-worker: {} H/s", format_hash_rate(per_worker)); + if let Some((finds, jobs)) = job_stats { + println!("Jobs started: {}", jobs.load(Ordering::Relaxed)); + println!("Solutions found: {}", finds.load(Ordering::Relaxed)); } println!("✅ Benchmark completed!"); diff --git a/gpu-bench/.env.example b/gpu-bench/.env.example new file mode 100644 index 0000000..35f43c6 --- /dev/null +++ b/gpu-bench/.env.example @@ -0,0 +1,21 @@ +# Not required for ./setup.sh --dev or remote-run.sh / runpod-sweep.sh. +# Required only for Planck (non-dev) mining. +# Generate with: ./setup.sh wormhole +REWARDS_INNER_HASH=0xyour_inner_hash_here + +CHAIN=planck +NODE_NAME=gpu-bench-node + +# Optional overrides +# GPU_DEVICES=1 +# METRICS_PORT=9900 +# HOST_MINER_LISTEN_PORT=9833 +# P2P_PORT=30333 +# RPC_PORT=9944 +# PROMETHEUS_PORT=9615 +# QUANTUS_NODE_BIN=/path/to/quantus-node # skip GitHub download if set +# QUANTUS_CHAIN_DIR=/path/to/chain # use local release build if present +# QUANTUS_MINER_DIR=/path/to/quantus-miner # default: parent of this directory +# MINER_BIN=/path/to/quantus-miner # skip cargo build if set +# MINER_LOG=info +# NODE_VERSION=latest # only used with: ./setup.sh start --docker diff --git a/gpu-bench/.gitignore b/gpu-bench/.gitignore new file mode 100644 index 0000000..c268006 --- /dev/null +++ b/gpu-bench/.gitignore @@ -0,0 +1,5 @@ +.run/ +*.log +.env +results.local.csv +sweep-out/ diff --git a/gpu-bench/README.md b/gpu-bench/README.md new file mode 100644 index 0000000..2a9f2af --- /dev/null +++ b/gpu-bench/README.md @@ -0,0 +1,141 @@ +# GPU miner bench (provider-agnostic) + +Rent an NVIDIA GPU, build `quantus-miner`, run **`benchmark` across batch sizes** +(no node), and append rows to [`results.csv`](results.csv) for hardware +comparison. You supply `--provider` and `--cost-per-hour`. + +## Miner binary (git build by default) + +`remote-run.sh` / the RunPod sweep **clone + `cargo build -p miner-cli --release`** +from a git branch (default `illuzen/gpu-bench`) so you can iterate without +cutting a GitHub release. **Push the branch before sweeping.** + +```bash +export MINER_BRANCH=illuzen/gpu-bench # default +# escape hatch: +# export MINER_SOURCE=release +``` + +Container disk defaults to **50GB** for cargo `target/`. No `quantus-node` +download — hardware benches use `quantus-miner benchmark` only. + +## On-pod one-shot + +```bash +./remote-run.sh --provider runpod --cost-per-hour 0.69 --duration 30 +# default: batch sizes 256K 512K 1M 4M, --job-interval 2 (simulated NewJob) +./remote-run.sh --cost-per-hour 0.39 --job-interval 0 # sustained peak H/s +./remote-run.sh --cost-per-hour 0.39 --difficulty max # cancel-only churn +``` + +Or call record directly if the miner is already built: + +```bash +MINER_BIN=./bin/quantus-miner ./record.sh --benchmark \ + --provider runpod --cost-per-hour 0.42 \ + --job-interval 2 --batch-sizes "262144 524288 1000000" --duration 30 +``` + +`benchmark --job-interval N` rotates a random header every N seconds (like node +`NewJob`), with default difficulty `10000000`. After a find, workers idle until +the next job — same cliff as `serve`. + +Local full stack (node + serve) is still available via `./setup.sh --dev` + +`./record.sh --live` when you need end-to-end mining, not just hardware H/s. + +## RunPod API sweep + +Uses the **REST API** (not MCP): create Pod → SSH → `remote-run.sh` → scp CSV → delete. + +### 1. Prerequisites (laptop) + +- `RUNPOD_API_KEY` from [RunPod settings](https://www.runpod.io/console/user/settings) +- SSH public key added in RunPod **Settings → SSH Public Keys** +- `curl`, `ssh`, `scp`, `python3` + +### 2. Image / template tips + +Default image: `runpod/base:1.1.0-cuda1281-ubuntu2404` with +`NVIDIA_DRIVER_CAPABILITIES=all` (Vulkan/WGPU). Container disk ~50GB for git +builds. Expose TCP 22 for SSH. + +### 3. Debug one Pod interactively + +```bash +export RUNPOD_API_KEY=... +chmod +x runpod-shell.sh remote-run.sh record.sh runpod-sweep.sh batch-tune.sh + +./runpod-shell.sh "NVIDIA L4" +./runpod-shell.sh --ssh +# on pod: +# cd /workspace/quantus-gpu-bench +# ./remote-run.sh --cost-per-hour … --duration 30 + +./runpod-shell.sh --delete +``` + +### 4. Run the sweep + +```bash +export RUNPOD_API_KEY=... +# optional: export BATCH_SIZES="1000000 16777216" +# optional: export DURATION=30 +# optional: export CLOUD_TYPE=SECURE + +./runpod-sweep.sh --gpus-file gpus.example.txt +./runpod-sweep.sh --gpus-file gpus.all.txt +``` + +Successful rows append to [`results.csv`](results.csv) (one row per batch size +per Pod). Commit new rows so others can reuse them. + +### What each Pod does + +1. Build `quantus-miner` from git (`MINER_BRANCH`) +2. Ensure NVIDIA Vulkan ICD (not Mesa llvmpipe) +3. Smoke-test `benchmark` (5s) +4. Sweep `--gpu-batch-size` values → append CSV rows (`notes` includes `batch=N`) +5. Tear down + +## Knobs that matter for differently shaped GPUs + +| Knob | Where | Notes | +|------|--------|--------| +| `--gpu-batch-size` | CLI / sweep | **Main runtime knob.** Nonces per dispatch; interacts with tier workgroup hints (`nonces_per_thread` vs occupancy). | +| GPU tier table | `crates/engine-gpu/src/gpu_tiers.rs` | Per-name `workgroup_divisor` + `min_workgroups`. Not a CLI flag — edit + rebuild to retune a class of cards. | +| `--gpu-devices` | CLI | How many GPUs; bench defaults to GPU-only when this is set. | +| `--allow-integrated` | CLI | Include iGPUs when a discrete GPU is present. | +| `--gpu-throttle-ms` | `serve` only | Delay between batches; not used in `benchmark`. | +| threads/workgroup | hardcoded `256` | Not exposed. | + +If util stays low across 1M→16M, dispatch shape may not be the bottleneck (driver, kernel, or tier mis-detect). Check miner logs for `tier:` / `workgroups:`. + +## Collaborative dataset + +[`results.csv`](results.csv) is the shared hardware comparison table. Prefer the +row with the best hashrate (or hash_per_dollar) per GPU when ranking hardware. + +## Spreadsheet columns + +| Column | Source | +|--------|--------| +| `cloud_provider` | `runpod` / flag | +| `gpu_model`, `vram_mb`, `sm_count` | `nvidia-smi` | +| `hashrate` | `benchmark` Total hashes / Total time | +| `gpu_utilization_pct` | avg `utilization.gpu` during the run | +| `cost_per_hour` | Pod `costPerHr` (sweep) or your flag | +| `cost_per_sec` | `cost_per_hour / 3600` | +| `hash_per_dollar` | `hashrate / cost_per_sec` | +| `notes` | includes `batch=N` for sweep rows | + +## Scripts + +| Script | Role | +|--------|------| +| [`remote-run.sh`](remote-run.sh) | On-box: build miner, Vulkan, batch-size benchmark → CSV | +| [`record.sh`](record.sh) | `--benchmark` (or `--live`) → CSV row(s) | +| [`batch-tune.sh`](batch-tune.sh) | Quick util/hashrate table without cost columns | +| [`runpod-sweep.sh`](runpod-sweep.sh) | RunPod REST API multi-GPU loop | +| [`runpod-shell.sh`](runpod-shell.sh) | One Pod + SSH; keep alive for manual debug | +| [`setup.sh`](setup.sh) | Local native node+miner (`--dev` or Planck) | +| [`gpus.all.txt`](gpus.all.txt) | Full RunPod NVIDIA `gpuTypeId` list | diff --git a/gpu-bench/batch-tune.sh b/gpu-bench/batch-tune.sh new file mode 100755 index 0000000..809c305 --- /dev/null +++ b/gpu-bench/batch-tune.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Quick A/B of GPU batch sizes (util + hashrate table). No cost columns. +# For CSV rows with cost/hash_per_dollar, prefer: +# ./remote-run.sh --cost-per-hour … --batch-sizes "…" +# +# Usage (on pod, after miner is built): +# cd /workspace/quantus-gpu-bench +# ./batch-tune.sh +# ./batch-tune.sh --sizes "1000000 4194304 16777216" --duration 30 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${WORK_DIR:-/workspace/quantus-gpu-bench}" +BIN_DIR="${BIN_DIR:-${WORK_DIR}/bin}" +OUT_CSV="${OUT_CSV:-${WORK_DIR}/batch-tune.csv}" +SIZES="${SIZES:-262144 524288 1000000 4194304}" +DURATION="${DURATION:-30}" +GPU_DEVICES="${GPU_DEVICES:-1}" +CPU_WORKERS="${CPU_WORKERS:-0}" +JOB_INTERVAL="${JOB_INTERVAL:-2}" +DIFFICULTY="${DIFFICULTY:-}" +MINER_BIN="${MINER_BIN:-}" + +usage() { + cat <<'EOF' +Usage: ./batch-tune.sh [options] + + --sizes "N N N" batch sizes to try (default: 256K 512K 1M 4M) + --duration SECONDS per-size benchmark window (default: 30) + --job-interval SEC simulated NewJob period (default: 2; 0 = sustained) + --difficulty DEC difficulty for job sim (or max) + --gpu-devices N default 1 + --cpu-workers N default 0 (GPU-only) + --miner-bin PATH override miner binary + --out PATH results CSV (default: $WORK_DIR/batch-tune.csv) + --help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --sizes) SIZES="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --job-interval) JOB_INTERVAL="$2"; shift 2 ;; + --difficulty) DIFFICULTY="$2"; shift 2 ;; + --gpu-devices) GPU_DEVICES="$2"; shift 2 ;; + --cpu-workers) CPU_WORKERS="$2"; shift 2 ;; + --miner-bin) MINER_BIN="$2"; shift 2 ;; + --out) OUT_CSV="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) echo "unknown arg: $1" >&2; usage; exit 1 ;; + esac +done + +resolve_miner() { + if [[ -n "${MINER_BIN}" && -x "${MINER_BIN}" ]]; then + echo "${MINER_BIN}" + return + fi + for cand in \ + "${BIN_DIR}/quantus-miner" \ + "${WORK_DIR}/quantus-miner/target/release/quantus-miner" \ + "${SCRIPT_DIR}/../target/release/quantus-miner" \ + "$(command -v quantus-miner 2>/dev/null || true)"; do + if [[ -n "${cand}" && -x "${cand}" ]]; then + echo "${cand}" + return + fi + done + echo "error: quantus-miner binary not found. Run remote-run.sh once, or set --miner-bin" >&2 + exit 1 +} + +sample_util() { + # Average GPU util % over ~DURATION seconds (1 Hz). + local secs="$1" + local sum=0 count=0 u + for ((i = 0; i < secs; i++)); do + u="$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null | head -n1 | tr -d ' ' || echo 0)" + [[ "${u}" =~ ^[0-9]+$ ]] || u=0 + sum=$((sum + u)) + count=$((count + 1)) + sleep 1 + done + if [[ "${count}" -eq 0 ]]; then + echo 0 + else + echo $((sum / count)) + fi +} + +parse_rate_hs() { + # Parse "Average rate: 1.23M H/s" / "45.67K H/s" / "1234 H/s" from benchmark stdout. + local line unit num + line="$(grep -E 'Average rate:' "$1" | tail -n1 || true)" + [[ -n "${line}" ]] || { echo 0; return; } + num="$(echo "${line}" | sed -E 's/.*Average rate:[[:space:]]*([0-9.]+)([KMkm]?).*H\/s.*/\1/')" + unit="$(echo "${line}" | sed -E 's/.*Average rate:[[:space:]]*[0-9.]+([KMkm]?).*H\/s.*/\1/')" + case "${unit}" in + M|m) awk -v n="${num}" 'BEGIN { printf "%.0f", n * 1000000 }' ;; + K|k) awk -v n="${num}" 'BEGIN { printf "%.0f", n * 1000 }' ;; + *) awk -v n="${num}" 'BEGIN { printf "%.0f", n }' ;; + esac +} + +MINER_BIN="$(resolve_miner)" +GPU_NAME="$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -n1 | sed 's/,/;/g' || echo unknown)" +echo "miner: ${MINER_BIN}" +echo "gpu: ${GPU_NAME}" +echo "sizes: ${SIZES}" +echo "window: ${DURATION}s gpu_devices=${GPU_DEVICES} cpu_workers=${CPU_WORKERS} job_interval=${JOB_INTERVAL}" +echo + +mkdir -p "$(dirname "${OUT_CSV}")" +if [[ ! -f "${OUT_CSV}" ]]; then + echo "timestamp,gpu_name,batch_size,duration_s,hashrate_hs,avg_util_pct,job_interval,notes" >"${OUT_CSV}" +fi + +printf "%-12s %-14s %-10s\n" "batch_size" "hashrate" "avg_util%" +printf "%-12s %-14s %-10s\n" "----------" "--------" "---------" + +for bs in ${SIZES}; do + log="$(mktemp)" + util_log="$(mktemp)" + # Sample util in background while benchmark runs. + ( + # skip first ~3s of warmup + sleep 3 + sample_util $((DURATION > 5 ? DURATION - 3 : DURATION)) + ) >"${util_log}" & + util_pid=$! + + bench_cmd=( + "${MINER_BIN}" benchmark + --gpu-devices "${GPU_DEVICES}" + --cpu-workers "${CPU_WORKERS}" + --gpu-batch-size "${bs}" + --duration "${DURATION}" + ) + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + bench_cmd+=(--job-interval "${JOB_INTERVAL}") + fi + if [[ -n "${DIFFICULTY}" ]]; then + bench_cmd+=(--difficulty "${DIFFICULTY}") + fi + + set +e + "${bench_cmd[@]}" >"${log}" 2>&1 + rc=$? + set -e + + wait "${util_pid}" 2>/dev/null || true + util="$(cat "${util_log}" 2>/dev/null || echo 0)" + rate="$(parse_rate_hs "${log}")" + + if [[ "${rc}" -ne 0 || "${rate}" == "0" ]]; then + echo "--- failed batch_size=${bs} (rc=${rc}) ---" >&2 + tail -n 40 "${log}" >&2 || true + notes="FAILED" + rate_disp="FAIL" + else + notes="ok" + if (( rate >= 1000000 )); then + rate_disp="$(awk -v n="${rate}" 'BEGIN { printf "%.2fM" , n/1000000 }')" + elif (( rate >= 1000 )); then + rate_disp="$(awk -v n="${rate}" 'BEGIN { printf "%.2fK" , n/1000 }')" + else + rate_disp="${rate}" + fi + fi + + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "${ts},${GPU_NAME},${bs},${DURATION},${rate},${util},${JOB_INTERVAL},${notes}" >>"${OUT_CSV}" + printf "%-12s %-14s %-10s\n" "${bs}" "${rate_disp}" "${util}%" + + rm -f "${log}" "${util_log}" +done + +echo +echo "wrote ${OUT_CSV}" +echo "Best util/rate usually wins — if 16M ≈ 1M, dispatch shape is not the bottleneck." diff --git a/gpu-bench/gpus.all.txt b/gpu-bench/gpus.all.txt new file mode 100644 index 0000000..eebb01a --- /dev/null +++ b/gpu-bench/gpus.all.txt @@ -0,0 +1,54 @@ +# RunPod NVIDIA gpuTypeId list (must match REST POST /pods enum). +# AMD omitted — WGPU Vulkan path is NVIDIA. +# Refresh from API errors or https://docs.runpod.io/references/gpu-types +# Usage: ./runpod-sweep.sh --gpus-file gpus.all.txt + +# Consumer / GeForce +NVIDIA GeForce RTX 3070 +NVIDIA GeForce RTX 3080 +NVIDIA GeForce RTX 3080 Ti +NVIDIA GeForce RTX 3090 +NVIDIA GeForce RTX 3090 Ti +NVIDIA GeForce RTX 4070 Ti +NVIDIA GeForce RTX 4080 +NVIDIA GeForce RTX 4080 SUPER +NVIDIA GeForce RTX 4090 +NVIDIA GeForce RTX 5080 +NVIDIA GeForce RTX 5090 + +# Pro / Ada / Ampere workstation +NVIDIA RTX A2000 +NVIDIA RTX A4000 +NVIDIA RTX A4500 +NVIDIA RTX A5000 +NVIDIA RTX A6000 +NVIDIA RTX 2000 Ada Generation +NVIDIA RTX 4000 Ada Generation +NVIDIA RTX 4000 SFF Ada Generation +NVIDIA RTX 5000 Ada Generation +NVIDIA RTX 6000 Ada Generation +NVIDIA RTX PRO 4000 Blackwell +NVIDIA RTX PRO 4500 Blackwell +NVIDIA RTX PRO 5000 Blackwell +NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition +NVIDIA RTX PRO 6000 Blackwell Server Edition +NVIDIA RTX PRO 6000 Blackwell Workstation Edition + +# Data center +NVIDIA L4 +NVIDIA L40 +NVIDIA L40S +NVIDIA A40 +NVIDIA A100 80GB PCIe +NVIDIA A100-SXM4-40GB +NVIDIA A100-SXM4-80GB +NVIDIA H100 PCIe +NVIDIA H100 NVL +NVIDIA H100 80GB HBM3 +NVIDIA H200 +NVIDIA H200 NVL +NVIDIA B200 +NVIDIA B300 SXM6 AC +NVIDIA B300 SXM6 AC MIG 1g.34gb +Tesla V100-PCIE-16GB +Tesla V100-SXM2-16GB diff --git a/gpu-bench/gpus.example.txt b/gpu-bench/gpus.example.txt new file mode 100644 index 0000000..a6a0c13 --- /dev/null +++ b/gpu-bench/gpus.example.txt @@ -0,0 +1,10 @@ +# One RunPod gpuTypeId per line (from console / API). +# Lines starting with # are ignored. +#NVIDIA GeForce RTX 4090 +#NVIDIA GeForce RTX 3080 +#NVIDIA GeForce RTX 3090 +#NVIDIA RTX A5000 +#NVIDIA L4 +#NVIDIA RTX A4000 +NVIDIA GeForce RTX 5090 + diff --git a/gpu-bench/init-node.sh b/gpu-bench/init-node.sh new file mode 100755 index 0000000..429a774 --- /dev/null +++ b/gpu-bench/init-node.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +NODE_KEY_PATH="/node-keys" +NODE_KEY_FILE="$NODE_KEY_PATH/key_node" + +# Generate node key if it doesn't exist +if [ ! -f "$NODE_KEY_FILE" ]; then + echo "Generating node key..." + mkdir -p "$NODE_KEY_PATH" + /usr/local/bin/quantus-node key generate-node-key --file "$NODE_KEY_FILE" + echo "Node key generated at: $NODE_KEY_FILE" +fi + +# Start node with original arguments +exec /usr/local/bin/quantus-node "$@" diff --git a/gpu-bench/record.sh b/gpu-bench/record.sh new file mode 100755 index 0000000..1fcc6f3 --- /dev/null +++ b/gpu-bench/record.sh @@ -0,0 +1,610 @@ +#!/usr/bin/env bash +# Sample GPU miner performance and append one row to results.csv. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUN_DIR="${SCRIPT_DIR}/.run" +ENV_FILE="${SCRIPT_DIR}/.env" +RESULTS_CSV="${SCRIPT_DIR}/results.csv" +CSV_HEADER="timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes" +DEFAULT_MINER_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +MODE="live" +PROVIDER="" +COST_PER_HOUR="" +DURATION=60 +NOTES="" +DRY_RUN=0 +GPU_DEVICES_FLAG="" +GPU_BATCH_SIZE="${GPU_BATCH_SIZE:-}" +# Space-separated list; when set with --benchmark, runs one row per size. +BATCH_SIZES="${BATCH_SIZES:-}" +JOB_INTERVAL="${JOB_INTERVAL:-0}" +DIFFICULTY="${DIFFICULTY:-}" + +usage() { + cat <<'EOF' +Usage: ./record.sh [options] + +Options: + --live Sample a running miner (default; requires setup.sh) + --benchmark Run quantus-miner benchmark (no node required) + --provider NAME Cloud provider label (e.g. vast.ai, runpod) + --cost-per-hour USD Hourly cost in USD (e.g. 0.35) + --duration SECONDS Sample / benchmark window (default: 60) + --gpu-devices N GPUs for --benchmark (default: GPU_DEVICES or 1) + --gpu-batch-size N Single GPU batch size for --benchmark + --batch-sizes "N N" Sweep several batch sizes (one CSV row each) + --job-interval SECONDS Simulated NewJob period (0 = sustained; default 0) + --difficulty DEC|max Difficulty for job simulation + --notes TEXT Optional notes column + --dry-run Print the CSV row but do not append + -h, --help Show this help + +Examples: + ./record.sh --provider vast.ai --cost-per-hour 0.35 + ./record.sh --benchmark --provider runpod --cost-per-hour 0.42 --duration 30 + ./record.sh --benchmark --job-interval 2 --batch-sizes "262144 524288 1000000" \ + --provider runpod --cost-per-hour 0.39 +EOF +} + +load_env() { + if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + set -a + source "${ENV_FILE}" + set +a + fi +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +csv_escape() { + local s="${1:-}" + if [[ "${s}" == *","* || "${s}" == *"\""* || "${s}" == *$'\n'* ]]; then + s="${s//\"/\"\"}" + printf '"%s"' "${s}" + else + printf '%s' "${s}" + fi +} + +ensure_results_header() { + if [[ ! -f "${RESULTS_CSV}" ]]; then + printf '%s\n' "${CSV_HEADER}" >"${RESULTS_CSV}" + return + fi + if [[ ! -s "${RESULTS_CSV}" ]]; then + printf '%s\n' "${CSV_HEADER}" >"${RESULTS_CSV}" + fi +} + +# Best-effort SM / multiprocessor count. Many cloud drivers omit +# --query-gpu=multiprocessor_count (fails the whole CSV row), so we try +# several sources after name/VRAM are already known. +query_sm_count() { + local name="${1:-}" + local sm="" + + sm="$(nvidia-smi --query-gpu=multiprocessor_count --format=csv,noheader,nounits 2>/dev/null \ + | head -n 1 | tr -d '[:space:]')" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + sm="$(nvidia-smi -q 2>/dev/null \ + | awk -F: 'tolower($0) ~ /multiprocessor count/ { + gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit + }')" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + sm="$(nvidia-smi -q -x 2>/dev/null \ + | sed -n 's/.*\([0-9][0-9]*\)<\/multiprocessor_count>.*/\1/p' \ + | head -n 1)" + if [[ "${sm}" =~ ^[0-9]+$ ]]; then + echo "${sm}" + return + fi + + # Static fallback for common RunPod SKUs (architecture SM counts). + case "${name}" in + *"RTX 3070"*) echo 46 ;; + *"RTX 3080 Ti"*) echo 80 ;; + *"RTX 3080"*) echo 68 ;; + *"RTX 3090 Ti"*) echo 84 ;; + *"RTX 3090"*) echo 82 ;; + *"RTX 4070 Ti"*) echo 60 ;; + *"RTX 4080 SUPER"*) echo 80 ;; + *"RTX 4080"*) echo 76 ;; + *"RTX 4090"*) echo 128 ;; + *"RTX 5080"*) echo 84 ;; + *"RTX 5090"*) echo 170 ;; + *"RTX A2000"*) echo 26 ;; + *"RTX A4000"*) echo 48 ;; + *"RTX A4500"*) echo 56 ;; + *"RTX A5000"*) echo 64 ;; + *"RTX A6000"*) echo 84 ;; + *"RTX 2000 Ada"*) echo 22 ;; + *"RTX 4000 Ada"*|*"RTX 4000 SFF Ada"*) echo 48 ;; + *"RTX 5000 Ada"*) echo 100 ;; + *"RTX 6000 Ada"*) echo 142 ;; + *"RTX PRO 4000"*) echo 48 ;; + *"RTX PRO 4500"*) echo 80 ;; + *"RTX PRO 5000"*) echo 140 ;; + *"RTX PRO 6000"*) echo 188 ;; + *"NVIDIA L4"|*" L4") echo 60 ;; + *"L40S"*) echo 142 ;; + *"L40"*) echo 142 ;; + *"NVIDIA A40"|*" A40") echo 84 ;; + *"A100"*) echo 108 ;; + *"H100 NVL"*) echo 132 ;; + *"H100"*) echo 132 ;; + *"H200"*) echo 132 ;; + *"B200"*) echo 160 ;; + *"B300"*) echo 160 ;; + *"V100"*) echo 80 ;; + *) echo "" ;; + esac +} + +query_gpu_static() { + # Query name/VRAM/driver without multiprocessor_count — some drivers (e.g. 580.x) + # reject that field and, under pipefail, abort before any fallback. SM count is + # filled separately via query_sm_count (nvidia-smi / -q / static table). + local line + SM_COUNT="" + if ! line="$(nvidia-smi --query-gpu=name,memory.total,driver_version \ + --format=csv,noheader,nounits 2>/dev/null | head -n 1)" || [[ -z "${line}" ]]; then + echo "error: failed to query GPU via nvidia-smi" >&2 + exit 1 + fi + GPU_MODEL="$(echo "${line}" | awk -F', ' '{print $1}')" + VRAM_MB="$(echo "${line}" | awk -F', ' '{print $2}' | tr -d ' ')" + DRIVER_VERSION="$(echo "${line}" | awk -F', ' '{print $3}' | tr -d ' ')" + SM_COUNT="$(query_sm_count "${GPU_MODEL}")" +} + +read_gpu_util() { + nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null \ + | head -n 1 | tr -d ' ' +} + +read_prometheus_hashrate() { + local port="$1" + local metrics + if ! metrics="$(curl -sf "http://127.0.0.1:${port}/metrics")"; then + echo "" + return + fi + local gpu_hr total_hr + gpu_hr="$(echo "${metrics}" | awk '/^miner_gpu_hash_rate[[:space:]]/{print $2; exit}')" + if [[ -n "${gpu_hr}" ]]; then + echo "${gpu_hr}" + return + fi + total_hr="$(echo "${metrics}" | awk '/^miner_hash_rate[[:space:]]/{print $2; exit}')" + echo "${total_hr}" +} + +average_list() { + # stdin: one number per line; prints average or empty + awk ' + NF && $1+0 == $1 { + sum += $1 + n++ + } + END { + if (n > 0) printf "%.6f", sum / n + } + ' +} + +resolve_miner_bin() { + if [[ -n "${MINER_BIN:-}" ]]; then + if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: MINER_BIN is not executable: ${MINER_BIN}" >&2 + exit 1 + fi + echo "${MINER_BIN}" + return + fi + + local miner_dir="${QUANTUS_MINER_DIR:-${DEFAULT_MINER_DIR}}" + local bin="${miner_dir}/target/release/quantus-miner" + if [[ -x "${bin}" ]]; then + echo "${bin}" + return + fi + + if [[ ! -d "${miner_dir}" ]]; then + echo "error: quantus-miner not found at ${miner_dir}" >&2 + exit 1 + fi + require_cmd cargo + echo "Building quantus-miner (release) in ${miner_dir} ..." >&2 + ( + cd "${miner_dir}" + cargo build -p miner-cli --release + ) >&2 + if [[ ! -x "${bin}" ]]; then + echo "error: expected binary missing: ${bin}" >&2 + exit 1 + fi + echo "${bin}" +} + +parse_benchmark_hashrate() { + # Prefer Total hashes / Total time for a numeric H/s. + local out="$1" + local total_hashes total_time + total_hashes="$(echo "${out}" | awk -F': ' '/^Total hashes:/{gsub(/[^0-9]/,"",$2); print $2; exit}')" + total_time="$(echo "${out}" | awk -F': ' '/^Total time:/{gsub(/s$/,"",$2); print $2; exit}')" + if [[ -n "${total_hashes}" && -n "${total_time}" ]]; then + awk -v h="${total_hashes}" -v t="${total_time}" 'BEGIN { + if (t+0 > 0) printf "%.6f", h / t + }' + return + fi + echo "" +} + +sample_util_during() { + # Background util sampler; writes samples to $1 for $2 seconds every ~2s + local out_file="$1" + local seconds="$2" + local end=$((SECONDS + seconds)) + : >"${out_file}" + while (( SECONDS < end )); do + local u + u="$(read_gpu_util || true)" + if [[ -n "${u}" ]]; then + echo "${u}" >>"${out_file}" + fi + sleep 2 + done +} + +prompt_if_empty() { + local var_name="$1" + local prompt="$2" + local current="${!var_name:-}" + if [[ -n "${current}" ]]; then + return + fi + if [[ ! -t 0 ]]; then + echo "error: ${var_name} required (pass flag; stdin is not a TTY)" >&2 + exit 1 + fi + local value + read -r -p "${prompt}: " value + printf -v "${var_name}" '%s' "${value}" +} + +# From cost_per_hour ($/hr) and hashrate (H/s): +# cost_per_sec = cost_per_hour / 3600 +# hash_per_dollar = hashrate / cost_per_sec (hashes per $) +compute_cost_metrics() { + local hashrate="$1" + local cost_per_hour="$2" + awk -v h="${hashrate}" -v c="${cost_per_hour}" 'BEGIN { + if (c+0 <= 0) { print ""; print ""; exit } + cps = c / 3600 + hpd = h / cps + printf "%.10f\n%.6f\n", cps, hpd + }' +} + +emit_row() { + local timestamp="$1" + local hashrate="$2" + local util_avg="$3" + local cost_per_sec="$4" + local hash_per_dollar="$5" + + local row + row="$(csv_escape "${timestamp}"),$(csv_escape "${PROVIDER}"),$(csv_escape "${GPU_MODEL}"),$(csv_escape "${VRAM_MB}"),$(csv_escape "${SM_COUNT}"),$(csv_escape "${DRIVER_VERSION}"),$(csv_escape "${hashrate}"),$(csv_escape "${util_avg}"),$(csv_escape "${COST_PER_HOUR}"),$(csv_escape "${cost_per_sec}"),$(csv_escape "${hash_per_dollar}"),$(csv_escape "${DURATION}"),$(csv_escape "${NOTES}")" + + echo "${CSV_HEADER}" + echo "${row}" + + if [[ "${DRY_RUN}" -eq 1 ]]; then + echo "(dry-run: not written to ${RESULTS_CSV})" >&2 + return + fi + + ensure_results_header + echo "${row}" >>"${RESULTS_CSV}" + echo "Appended row to ${RESULTS_CSV}" >&2 +} + +run_live() { + require_cmd curl + local metrics_port="${METRICS_PORT:-9900}" + if [[ -f "${RUN_DIR}/metrics.port" ]]; then + metrics_port="$(cat "${RUN_DIR}/metrics.port")" + fi + + if ! curl -sf "http://127.0.0.1:${metrics_port}/metrics" >/dev/null; then + echo "error: miner metrics not reachable at http://127.0.0.1:${metrics_port}/metrics" >&2 + echo "Start the stack with ./setup.sh, or use --benchmark." >&2 + exit 1 + fi + + echo "Sampling live miner for ${DURATION}s (metrics :${metrics_port}) ..." >&2 + local hr_file util_file + hr_file="$(mktemp)" + util_file="$(mktemp)" + trap 'rm -f "${hr_file}" "${util_file}"' RETURN + + local end=$((SECONDS + DURATION)) + while (( SECONDS < end )); do + local hr util + hr="$(read_prometheus_hashrate "${metrics_port}")" + util="$(read_gpu_util || true)" + if [[ -n "${hr}" ]]; then + echo "${hr}" >>"${hr_file}" + fi + if [[ -n "${util}" ]]; then + echo "${util}" >>"${util_file}" + fi + sleep 2 + done + + local hashrate util_avg + hashrate="$(average_list <"${hr_file}")" + util_avg="$(average_list <"${util_file}")" + if [[ -z "${hashrate}" ]]; then + echo "error: no hashrate samples collected from :${metrics_port}" >&2 + exit 1 + fi + # Round util for spreadsheet readability + if [[ -n "${util_avg}" ]]; then + util_avg="$(awk -v u="${util_avg}" 'BEGIN { printf "%.2f", u }')" + fi + hashrate="$(awk -v h="${hashrate}" 'BEGIN { printf "%.6f", h }')" + + local cost_per_sec hash_per_dollar + { + read -r cost_per_sec + read -r hash_per_dollar + } < <(compute_cost_metrics "${hashrate}" "${COST_PER_HOUR}") + + local ts + ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" +} + +run_benchmark_once() { + local miner_bin="$1" + local gpu_devices="$2" + local batch_size="$3" + local notes_extra="$4" + + local util_file bench_log + util_file="$(mktemp)" + bench_log="$(mktemp)" + + local batch_label="default" + if [[ -n "${batch_size}" ]]; then + batch_label="${batch_size}" + fi + + local bench_cmd=( + "${miner_bin}" benchmark + --cpu-workers 0 + --gpu-devices "${gpu_devices}" + --duration "${DURATION}" + ) + if [[ -n "${batch_size}" ]]; then + bench_cmd+=(--gpu-batch-size "${batch_size}") + fi + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + bench_cmd+=(--job-interval "${JOB_INTERVAL}") + fi + if [[ -n "${DIFFICULTY}" ]]; then + bench_cmd+=(--difficulty "${DIFFICULTY}") + fi + + echo "Running GPU benchmark for ${DURATION}s (gpu-devices=${gpu_devices}, batch=${batch_label}, job_interval=${JOB_INTERVAL}) ..." >&2 + sample_util_during "${util_file}" "${DURATION}" & + local sampler_pid=$! + + set +e + "${bench_cmd[@]}" >"${bench_log}" 2>&1 + local bench_rc=$? + set -e + + wait "${sampler_pid}" 2>/dev/null || true + + if [[ "${bench_rc}" -ne 0 ]]; then + echo "error: benchmark failed (exit ${bench_rc}, batch=${batch_label}). Output:" >&2 + cat "${bench_log}" >&2 + rm -f "${util_file}" "${bench_log}" + return 1 + fi + + cat "${bench_log}" >&2 + + local hashrate util_avg + hashrate="$(parse_benchmark_hashrate "$(cat "${bench_log}")")" + if [[ -z "${hashrate}" ]]; then + echo "error: could not parse hashrate from benchmark output (batch=${batch_label})" >&2 + rm -f "${util_file}" "${bench_log}" + return 1 + fi + util_avg="$(average_list <"${util_file}")" + if [[ -n "${util_avg}" ]]; then + util_avg="$(awk -v u="${util_avg}" 'BEGIN { printf "%.2f", u }')" + fi + + local cost_per_sec hash_per_dollar + { + read -r cost_per_sec + read -r hash_per_dollar + } < <(compute_cost_metrics "${hashrate}" "${COST_PER_HOUR}") + + local saved_notes="${NOTES}" + if [[ -n "${notes_extra}" ]]; then + if [[ -n "${NOTES}" ]]; then + NOTES="${NOTES};${notes_extra}" + else + NOTES="${notes_extra}" + fi + fi + + local ts + ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + emit_row "${ts}" "${hashrate}" "${util_avg}" "${cost_per_sec}" "${hash_per_dollar}" + NOTES="${saved_notes}" + + rm -f "${util_file}" "${bench_log}" + return 0 +} + +run_benchmark() { + local miner_bin + miner_bin="$(resolve_miner_bin)" + local gpu_devices="${GPU_DEVICES_FLAG:-${GPU_DEVICES:-1}}" + + local sizes=() + if [[ -n "${BATCH_SIZES}" ]]; then + # shellcheck disable=SC2206 + sizes=(${BATCH_SIZES}) + elif [[ -n "${GPU_BATCH_SIZE}" ]]; then + sizes=("${GPU_BATCH_SIZE}") + else + sizes=("") + fi + + local bs note_extra + local any_ok=0 + for bs in "${sizes[@]}"; do + note_extra="" + if [[ -n "${bs}" ]]; then + note_extra="batch=${bs}" + fi + if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + if [[ -n "${note_extra}" ]]; then + note_extra="${note_extra};job_interval=${JOB_INTERVAL}" + else + note_extra="job_interval=${JOB_INTERVAL}" + fi + if [[ -n "${DIFFICULTY}" ]]; then + note_extra="${note_extra};difficulty=${DIFFICULTY}" + fi + fi + if run_benchmark_once "${miner_bin}" "${gpu_devices}" "${bs}" "${note_extra}"; then + any_ok=1 + fi + done + + if [[ "${any_ok}" -ne 1 ]]; then + echo "error: all benchmark runs failed" >&2 + exit 1 + fi +} + +# --- args --- +while [[ $# -gt 0 ]]; do + case "$1" in + --live) + MODE="live" + shift + ;; + --benchmark) + MODE="benchmark" + shift + ;; + --provider) + PROVIDER="${2:-}" + shift 2 + ;; + --cost-per-hour) + COST_PER_HOUR="${2:-}" + shift 2 + ;; + --duration) + DURATION="${2:-}" + shift 2 + ;; + --gpu-devices) + GPU_DEVICES_FLAG="${2:-}" + shift 2 + ;; + --gpu-batch-size) + GPU_BATCH_SIZE="${2:-}" + shift 2 + ;; + --batch-sizes) + BATCH_SIZES="${2:-}" + shift 2 + ;; + --job-interval) + JOB_INTERVAL="${2:-}" + shift 2 + ;; + --difficulty) + DIFFICULTY="${2:-}" + shift 2 + ;; + --notes) + NOTES="${2:-}" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +load_env +require_cmd nvidia-smi +if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed — NVIDIA driver required" >&2 + exit 1 +fi + +if ! [[ "${DURATION}" =~ ^[0-9]+$ ]] || [[ "${DURATION}" -lt 1 ]]; then + echo "error: --duration must be a positive integer" >&2 + exit 1 +fi + +prompt_if_empty PROVIDER "Cloud provider (e.g. vast.ai)" +prompt_if_empty COST_PER_HOUR "Cost per hour USD (e.g. 0.35)" + +if ! awk -v c="${COST_PER_HOUR}" 'BEGIN { exit !(c+0 > 0) }'; then + echo "error: --cost-per-hour must be a positive number" >&2 + exit 1 +fi + +query_gpu_static +echo "GPU: ${GPU_MODEL} | VRAM: ${VRAM_MB} MiB | SMs: ${SM_COUNT:-n/a} | driver: ${DRIVER_VERSION}" >&2 + +case "${MODE}" in + live) run_live ;; + benchmark) run_benchmark ;; + *) + echo "error: unknown mode ${MODE}" >&2 + exit 1 + ;; +esac diff --git a/gpu-bench/remote-run.sh b/gpu-bench/remote-run.sh new file mode 100755 index 0000000..2e65a67 --- /dev/null +++ b/gpu-bench/remote-run.sh @@ -0,0 +1,515 @@ +#!/usr/bin/env bash +# Run on a GPU host (e.g. RunPod): build miner from git, ensure NVIDIA Vulkan, +# run quantus-miner benchmark across batch sizes → results.csv (no node). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${WORK_DIR:-/workspace/quantus-gpu-bench}" +BIN_DIR="${WORK_DIR}/bin" +RUN_DIR="${WORK_DIR}/.run" +RESULTS_CSV="${WORK_DIR}/results.csv" + +# Miner: default builds from git (iterate without re-releasing). +# MINER_SOURCE=release still downloads a GitHub release binary. +MINER_SOURCE="${MINER_SOURCE:-git}" +MINER_REPO="${MINER_REPO:-https://github.com/Quantus-Network/quantus-miner.git}" +MINER_BRANCH="${MINER_BRANCH:-illuzen/gpu-bench}" +MINER_VERSION="${MINER_VERSION:-v3.3.1}" +MINER_URL="${MINER_URL:-https://github.com/Quantus-Network/quantus-miner/releases/download/${MINER_VERSION}/quantus-miner-linux-x86_64}" +FORCE_MINER_BUILD="${FORCE_MINER_BUILD:-0}" + +PROVIDER="${PROVIDER:-runpod}" +COST_PER_HOUR="${COST_PER_HOUR:-}" +DURATION="${DURATION:-30}" +GPU_DEVICES="${GPU_DEVICES:-1}" +# Downward + 1M + one larger step. Override with --batch-sizes or BATCH_SIZES. +BATCH_SIZES="${BATCH_SIZES:-262144 524288 1000000 4194304}" +# Simulate NewJob churn (seconds). 0 = sustained peak H/s (no job switches). +JOB_INTERVAL="${JOB_INTERVAL:-2}" +# Optional decimal difficulty for job sim (default in miner: 10000000). Use "max" for cancel-only. +DIFFICULTY="${DIFFICULTY:-}" +NOTES="${NOTES:-}" + +usage() { + cat <<'EOF' +Usage: ./remote-run.sh [options] + + --provider NAME cloud_provider column (default: runpod) + --cost-per-hour USD required for hash_per_dollar column + --duration SECONDS per-batch-size benchmark window (default: 30) + --gpu-devices N default 1 + --batch-sizes "N N N" GPU batch sizes (default: 256K 512K 1M 4M) + --job-interval SECONDS simulated NewJob period (default: 2; 0 = sustained) + --difficulty DEC|max PoW difficulty for job sim (miner default 1e7) + --notes TEXT + --miner-branch REF git branch/tag/commit to build (default: illuzen/gpu-bench) + --miner-repo URL git remote (default: Quantus-Network/quantus-miner) + --miner-source git|release + git = clone+cargo build (default); release = binary URL + --miner-url URL release binary URL (with --miner-source release) + --force-miner-build rebuild even if binary exists + --help + +Builds quantus-miner from git (or downloads a release), sets up NVIDIA Vulkan, +runs `quantus-miner benchmark` for each batch size (no node), appends rows to +results.csv (notes include batch=N; job_interval=… when set). + +Env: MINER_SOURCE, MINER_REPO, MINER_BRANCH, FORCE_MINER_BUILD, MINER_URL, + BATCH_SIZES, JOB_INTERVAL, DIFFICULTY +EOF +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --provider) PROVIDER="${2:-}"; shift 2 ;; + --cost-per-hour) COST_PER_HOUR="${2:-}"; shift 2 ;; + --duration) DURATION="${2:-}"; shift 2 ;; + --gpu-devices) GPU_DEVICES="${2:-}"; shift 2 ;; + --batch-sizes) BATCH_SIZES="${2:-}"; shift 2 ;; + --job-interval) JOB_INTERVAL="${2:-}"; shift 2 ;; + --difficulty) DIFFICULTY="${2:-}"; shift 2 ;; + --notes) NOTES="${2:-}"; shift 2 ;; + --miner-branch) MINER_BRANCH="${2:-}"; shift 2 ;; + --miner-repo) MINER_REPO="${2:-}"; shift 2 ;; + --miner-source) MINER_SOURCE="${2:-}"; shift 2 ;; + --miner-url) MINER_URL="${2:-}"; MINER_SOURCE="release"; shift 2 ;; + --force-miner-build) FORCE_MINER_BUILD=1; shift ;; + # Deprecated no-ops (node path removed) + --warmup | --node-url) shift 2 ;; + -h | --help) usage; exit 0 ;; + *) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${COST_PER_HOUR}" ]]; then + echo "error: --cost-per-hour is required" >&2 + exit 1 +fi + +require_cmd curl +require_cmd tar +require_cmd nvidia-smi +if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed" >&2 + exit 1 +fi + +mkdir -p "${BIN_DIR}" "${RUN_DIR}" +cd "${WORK_DIR}" + +# Copy record.sh next to us if we're invoked from gpu-bench/ +if [[ -f "${SCRIPT_DIR}/record.sh" && ! -f "${WORK_DIR}/record.sh" ]]; then + cp "${SCRIPT_DIR}/record.sh" "${WORK_DIR}/record.sh" + chmod +x "${WORK_DIR}/record.sh" +fi +if [[ ! -x "${WORK_DIR}/record.sh" ]]; then + echo "error: record.sh not found next to remote-run.sh" >&2 + exit 1 +fi + +# Exit code for sweep: host has CUDA but no usable NVIDIA Vulkan (retry new pod). +EXIT_COMPUTE_ONLY=42 + +find_libglx_nvidia() { + # Prefer exact .run extract over apt (apt major-version packages often + # fail vkCreateInstance against the host kernel module). + local candidate + for candidate in \ + "${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0" \ + /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0 \ + /usr/lib64/libGLX_nvidia.so.0 \ + /usr/lib/libGLX_nvidia.so.0 \ + /usr/local/nvidia/lib64/libGLX_nvidia.so.0; do + if [[ -e "${candidate}" ]]; then + echo "${candidate}" + return 0 + fi + done + ldconfig -p 2>/dev/null | awk '/libGLX_nvidia\.so\.0/ { print $NF; exit }' || true +} + +host_driver_version() { + nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null \ + | head -n 1 | tr -d '[:space:]' +} + +vulkan_has_nvidia() { + command -v vulkaninfo >/dev/null 2>&1 || return 1 + vulkaninfo --summary 2>/dev/null | grep -qiE 'NVIDIA|GeForce|Tesla|Quadro|RTX|A100|L4|L40' +} + +write_nvidia_icd() { + local lib="$1" + local icd_dir="/usr/share/vulkan/icd.d" + local icd="${icd_dir}/nvidia_icd.json" + mkdir -p "${icd_dir}" /etc/vulkan/icd.d + cat >"${icd}" </dev/null || true + export VK_ICD_FILENAMES="${icd}" + export NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" + export __GLX_VENDOR_LIBRARY_NAME=nvidia + local libdir + libdir="$(dirname "${lib}")" + export LD_LIBRARY_PATH="${libdir}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + echo "NVIDIA Vulkan ICD: ${icd} -> ${lib}" >&2 +} + +# Download + extract the exact host driver userspace (matches nvidia-smi version). +extract_nvidia_run_userspace() { + local ver="$1" + local dest="${WORK_DIR}/nvidia-gl" + mkdir -p "${dest}" + + if [[ -e "${dest}/libGLX_nvidia.so.0" && -f "${dest}/.driver_version" ]] \ + && [[ "$(cat "${dest}/.driver_version")" == "${ver}" ]]; then + echo "Using cached NVIDIA ${ver} userspace in ${dest}" >&2 + export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + return 0 + fi + + echo "Extracting NVIDIA ${ver} userspace libs (exact match for host driver) ..." >&2 + local tmp runfile url + tmp="$(mktemp -d)" + runfile="${tmp}/NVIDIA.run" + for url in \ + "https://download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ + "https://us.download.nvidia.com/XFree86/Linux-x86_64/${ver}/NVIDIA-Linux-x86_64-${ver}.run" \ + "https://us.download.nvidia.com/tesla/${ver}/NVIDIA-Linux-x86_64-${ver}.run"; do + echo " trying ${url}" >&2 + if curl -fL --connect-timeout 20 --max-time 600 "${url}" -o "${runfile}"; then + break + fi + rm -f "${runfile}" + done + if [[ ! -f "${runfile}" ]]; then + echo "error: could not download NVIDIA ${ver} .run installer" >&2 + rm -rf "${tmp}" + return 1 + fi + + chmod +x "${runfile}" + if ! sh "${runfile}" --extract-only --target "${tmp}/extract" >/tmp/nvidia-extract.log 2>&1; then + echo "error: NVIDIA .run extract failed; log:" >&2 + tail -n 40 /tmp/nvidia-extract.log >&2 || true + rm -rf "${tmp}" + return 1 + fi + + rm -rf "${dest}" + mkdir -p "${dest}" + local f + for f in \ + libGLX_nvidia.so.* \ + libEGL_nvidia.so.* \ + libnvidia-glcore.so.* \ + libnvidia-glsi.so.* \ + libnvidia-tls.so.* \ + libnvidia-glvkspirv.so.* \ + libnvidia-gpucomp.so.* \ + libnvidia-rtcore.so.* \ + libnvoptix.so.*; do + # shellcheck disable=SC2086 + cp -f ${tmp}/extract/${f} "${dest}/" 2>/dev/null || true + done + local so + for so in libGLX_nvidia.so libEGL_nvidia.so; do + if [[ ! -e "${dest}/${so}.0" ]]; then + local real + real="$(ls -1 "${dest}/${so}".* 2>/dev/null | grep -v '\.so$' | sort -V | tail -n 1 || true)" + if [[ -n "${real}" ]]; then + ln -sfn "$(basename "${real}")" "${dest}/${so}.0" + fi + fi + done + + echo "${ver}" >"${dest}/.driver_version" + export LD_LIBRARY_PATH="${dest}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + echo "${dest}" >>/etc/ld.so.conf.d/nvidia-gl-bench.conf 2>/dev/null || true + ldconfig 2>/dev/null || true + rm -rf "${tmp}" + + if [[ -e "${dest}/libGLX_nvidia.so.0" ]]; then + echo "NVIDIA GL userspace ready in ${dest}" >&2 + return 0 + fi + return 1 +} + +# WGPU uses Vulkan. RunPod/CUDA images often only expose compute; Mesa's +# llvmpipe then becomes the only adapter and the miner refuses it. +ensure_nvidia_vulkan() { + if command -v apt-get >/dev/null 2>&1; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/dev/null 2>&1 || true + # libegl1/libxext6 are required for libGLX_nvidia to export vk_icd* in + # headless containers (without them: ERROR_INCOMPATIBLE_DRIVER). + # Do NOT install mesa-vulkan-drivers (llvmpipe distracts / can win). + apt-get install -y -qq \ + curl ca-certificates \ + libvulkan1 vulkan-tools \ + libegl1 libxext6 libgl1 \ + >/dev/null 2>&1 || \ + apt-get install -y -qq curl ca-certificates libvulkan1 libegl1 libxext6 >/dev/null 2>&1 || true + fi + mkdir -p /tmp/runtime-root + chmod 700 /tmp/runtime-root 2>/dev/null || true + export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" + + local ver major lib + ver="$(host_driver_version)" + major="${ver%%.*}" + + # 1) Host-mounted lib (best case). + lib="$(find_libglx_nvidia)" + if [[ -n "${lib}" && -e "${lib}" ]]; then + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (host mount)" >&2 + return 0 + fi + echo "Host/apt libGLX present but vulkaninfo failed; trying exact .run extract ..." >&2 + vulkaninfo --summary 2>&1 | tail -n 20 >&2 || true + else + echo "libGLX_nvidia.so.0 not mounted (compute-only host)" >&2 + fi + + # 2) apt major package — often wrong patch level vs host (e.g. 580.95.05). + if [[ -n "${major}" ]] && command -v apt-get >/dev/null 2>&1; then + echo "Trying apt libnvidia-gl-${major} (host driver ${ver}) ..." >&2 + apt-get install -y -qq "libnvidia-gl-${major}" >/dev/null 2>&1 || true + lib="$(find_libglx_nvidia)" + if [[ -n "${lib}" && -e "${lib}" ]]; then + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (apt)" >&2 + return 0 + fi + echo "apt libnvidia-gl-${major} installed but vulkaninfo still fails (version skew?)" >&2 + fi + fi + + # 3) Exact driver .run extract — required on many Community compute-only hosts. + if [[ -z "${ver}" ]]; then + echo "error: cannot determine NVIDIA driver version" >&2 + exit "${EXIT_COMPUTE_ONLY}" + fi + if ! extract_nvidia_run_userspace "${ver}"; then + echo "error: failed to install matching NVIDIA GL userspace for ${ver}" >&2 + exit "${EXIT_COMPUTE_ONLY}" + fi + lib="${WORK_DIR}/nvidia-gl/libGLX_nvidia.so.0" + write_nvidia_icd "${lib}" + if vulkan_has_nvidia; then + echo "Vulkan NVIDIA adapter OK (.run extract ${ver})" >&2 + return 0 + fi + + echo "error: vulkaninfo still sees no NVIDIA GPU after .run extract." >&2 + vulkaninfo --summary 2>&1 | tail -n 40 >&2 || true + echo "exit ${EXIT_COMPUTE_ONLY}: bad Vulkan host — sweep should retry" >&2 + exit "${EXIT_COMPUTE_ONLY}" +} + +ensure_rust() { + if command -v cargo >/dev/null 2>&1; then + return 0 + fi + echo "Installing Rust toolchain (rustup) ..." >&2 + # rustup writes the welcome banner to stdout — must not leak into $(download_miner). + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain stable >&2 + # shellcheck disable=SC1091 + source "${HOME}/.cargo/env" + if ! command -v cargo >/dev/null 2>&1; then + echo "error: cargo not available after rustup install" >&2 + exit 1 + fi +} + +ensure_build_deps() { + if ! command -v apt-get >/dev/null 2>&1; then + return 0 + fi + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/dev/null 2>&1 || true + apt-get install -y -qq \ + git curl ca-certificates build-essential pkg-config libssl-dev \ + >/dev/null 2>&1 || \ + apt-get install -y -qq git curl build-essential pkg-config libssl-dev || true +} + +download_miner_release() { + local dest="${BIN_DIR}/quantus-miner" + if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" ]]; then + echo "Using existing ${dest}" >&2 + printf '%s\n' "${dest}" + return + fi + echo "Downloading miner release: ${MINER_URL}" >&2 + curl -fL "${MINER_URL}" -o "${dest}" + chmod +x "${dest}" + printf '%s\n' "${dest}" +} + +# Clone MINER_BRANCH from MINER_REPO and cargo build -p miner-cli --release. +build_miner_from_git() { + local dest="${BIN_DIR}/quantus-miner" + local src="${WORK_DIR}/quantus-miner-src" + local rev_file="${BIN_DIR}/quantus-miner.rev" + local built_rev="" + + ensure_build_deps + require_cmd git + ensure_rust + # shellcheck disable=SC1091 + [[ -f "${HOME}/.cargo/env" ]] && source "${HOME}/.cargo/env" + + echo "Miner source: ${MINER_REPO} @ ${MINER_BRANCH}" >&2 + if [[ -d "${src}/.git" ]]; then + git -C "${src}" remote set-url origin "${MINER_REPO}" + git -C "${src}" fetch --depth 1 origin "${MINER_BRANCH}" >&2 + git -C "${src}" checkout -f FETCH_HEAD >&2 + else + rm -rf "${src}" + if ! git clone --depth 1 --branch "${MINER_BRANCH}" "${MINER_REPO}" "${src}" >&2; then + echo "error: git clone failed for ${MINER_REPO} branch ${MINER_BRANCH}" >&2 + echo "Push the branch to origin, or set MINER_BRANCH / MINER_REPO." >&2 + exit 1 + fi + fi + + built_rev="$(git -C "${src}" rev-parse HEAD)" + if [[ -x "${dest}" && "${FORCE_MINER_BUILD}" != "1" && -f "${rev_file}" ]]; then + if [[ "$(cat "${rev_file}")" == "${built_rev}" ]]; then + echo "Using existing ${dest} (rev ${built_rev})" >&2 + printf '%s\n' "${dest}" + return + fi + fi + + echo "Building quantus-miner (release, rev ${built_rev}) ..." >&2 + ( + cd "${src}" + cargo build -p miner-cli --release >&2 + ) + if [[ ! -x "${src}/target/release/quantus-miner" ]]; then + echo "error: cargo build did not produce target/release/quantus-miner" >&2 + exit 1 + fi + mkdir -p "${BIN_DIR}" + cp -f "${src}/target/release/quantus-miner" "${dest}" + chmod +x "${dest}" + echo "${built_rev}" >"${rev_file}" + echo "Built ${dest}" >&2 + # Sole stdout line — consumed by MINER_BIN="$(download_miner)" + printf '%s\n' "${dest}" +} + +download_miner() { + case "${MINER_SOURCE}" in + git | build | source) + build_miner_from_git + ;; + release | binary) + download_miner_release + ;; + *) + echo "error: unknown MINER_SOURCE=${MINER_SOURCE} (use git or release)" >&2 + exit 1 + ;; + esac +} + +ensure_nvidia_vulkan +MINER_BIN="$(download_miner | tail -n 1)" +if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: miner binary missing or not executable: '${MINER_BIN}'" >&2 + exit 1 +fi + +echo "GPU:" +nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv || nvidia-smi || true + +# Smoke-test GPU path before spending minutes on a batch sweep. +echo "Smoke-testing benchmark (5s) ..." +set +e +smoke_log="$(mktemp)" +env \ + VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ + NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" \ + __GLX_VENDOR_LIBRARY_NAME=nvidia \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" \ + XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" \ + "${MINER_BIN}" benchmark \ + --cpu-workers 0 \ + --gpu-devices "${GPU_DEVICES}" \ + --gpu-batch-size 1000000 \ + --duration 5 \ + >"${smoke_log}" 2>&1 +smoke_rc=$? +set -e +if [[ "${smoke_rc}" -ne 0 ]]; then + echo "error: benchmark smoke test failed; log:" >&2 + cat "${smoke_log}" >&2 + if grep -qiE 'No usable GPU|llvmpipe|Vulkan|Failed to initialize GPU' "${smoke_log}" 2>/dev/null; then + echo "hint: need NVIDIA Vulkan ICD + NVIDIA_DRIVER_CAPABILITIES=all (not Mesa llvmpipe)" >&2 + rm -f "${smoke_log}" + exit "${EXIT_COMPUTE_ONLY}" + fi + rm -f "${smoke_log}" + exit 1 +fi +rm -f "${smoke_log}" + +NOTE_ARGS=() +if [[ -n "${NOTES}" ]]; then + NOTE_ARGS=(--notes "${NOTES}") +fi + +cd "${WORK_DIR}" +echo "Batch-size sweep: ${BATCH_SIZES} (${DURATION}s each, job_interval=${JOB_INTERVAL})" +EXTRA_ARGS=() +if awk -v j="${JOB_INTERVAL}" 'BEGIN { exit !(j+0 > 0) }'; then + EXTRA_ARGS+=(--job-interval "${JOB_INTERVAL}") +fi +if [[ -n "${DIFFICULTY}" ]]; then + EXTRA_ARGS+=(--difficulty "${DIFFICULTY}") +fi +export MINER_BIN +env \ + VK_ICD_FILENAMES="${VK_ICD_FILENAMES:-}" \ + NVIDIA_DRIVER_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES:-all}" \ + __GLX_VENDOR_LIBRARY_NAME=nvidia \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}" \ + XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/runtime-root}" \ + MINER_BIN="${MINER_BIN}" \ + ./record.sh --benchmark \ + --provider "${PROVIDER}" \ + --cost-per-hour "${COST_PER_HOUR}" \ + --duration "${DURATION}" \ + --gpu-devices "${GPU_DEVICES}" \ + --batch-sizes "${BATCH_SIZES}" \ + "${EXTRA_ARGS[@]}" \ + "${NOTE_ARGS[@]}" + +echo "Done. Results: ${RESULTS_CSV}" +tail -n 8 "${RESULTS_CSV}" || true diff --git a/gpu-bench/results.csv b/gpu-bench/results.csv new file mode 100644 index 0000000..0eaf64e --- /dev/null +++ b/gpu-bench/results.csv @@ -0,0 +1,71 @@ +timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes +2026-08-01T08:16:22Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,7237290.633333,71.30,0.18,0.0000500000,144745812666.660034,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=4exbkxuw781hte;ssh=direct +2026-08-01T08:18:38Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.126.09,6613452.233333,61.40,0.22,0.0000611111,108220127454.540009,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=w40modnpa8t6na;ssh=direct +2026-08-01T08:21:18Z,runpod,NVIDIA GeForce RTX 3090 Ti,24564,84,580.65.06,7323359.689655,73.28,0.27,0.0000750000,97644795862.066666,60,gpuTypeId=NVIDIA GeForce RTX 3090 Ti;pod=lflvz4kxqkxsol;ssh=direct +2026-08-01T08:28:49Z,runpod,NVIDIA RTX A2000,6138,26,550.127.05,3161622.600000,84.47,0.12,0.0000333333,94848678000.000000,60,gpuTypeId=NVIDIA RTX A2000;pod=d5ahfwcfoluhz0;ssh=direct +2026-08-01T08:31:16Z,runpod,NVIDIA RTX A4500,20470,56,550.107.02,5767049.900000,52.57,0.19,0.0000527778,109270419157.894745,60,gpuTypeId=NVIDIA RTX A4500;pod=hzmwz9h5b58cbo;ssh=direct +2026-08-01T09:40:00Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7800565.793103,53.76,0.2,0.0000555556,140410184275.854004,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=m2ydq18u6onk8r;ssh=direct +2026-08-01T10:39:32Z,runpod,NVIDIA GeForce RTX 3080,10240,68,580.95.05,6604543.833333,72.87,0.17,0.0000472222,139860928235.287048,60,gpuTypeId=NVIDIA GeForce RTX 3080;pod=l6v6j20c2bf0jr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:43:30Z,runpod,NVIDIA GeForce RTX 3080 Ti,12288,80,580.65.06,6196787.448276,63.76,0.18,0.0000500000,123935748965.520020,60,gpuTypeId=NVIDIA GeForce RTX 3080 Ti;pod=b0dzcmo8tarsno;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-01T10:47:53Z,runpod,NVIDIA GeForce RTX 3090,24576,82,580.65.06,6834865.566667,53.37,0.22,0.0000611111,111843254727.278183,60,gpuTypeId=NVIDIA GeForce RTX 3090;pod=tdmmxojjx85zbn;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T03:55:55Z,runpod,NVIDIA RTX A4500,20470,56,580.126.18,5587581.033333,67.63,0.19,0.0000527778,105869956421.046310,60,gpuTypeId=NVIDIA RTX A4500;pod=nk7hljiu43i8gc;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:02:04Z,runpod,NVIDIA RTX A5000,24564,64,550.144.03,6510972.633333,65.13,0.16,0.0000444444,146496884249.992493,60,gpuTypeId=NVIDIA RTX A5000;pod=sgeizrvavai3lt;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:09:46Z,runpod,NVIDIA RTX 4000 Ada Generation,20475,48,560.35.03,7669824.448276,53.41,0.2,0.0000555556,138056840068.967987,60,gpuTypeId=NVIDIA RTX 4000 Ada Generation;pod=2o26qc0dczed6h;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-02T04:15:33Z,runpod,NVIDIA RTX 4000 SFF Ada Generation,20475,48,560.35.05,6308176.620690,62.55,0.18,0.0000500000,126163532413.800018,60,gpuTypeId=NVIDIA RTX 4000 SFF Ada Generation;pod=cxdke7crdl03qr;ssh=direct;miner=git@illuzen/gpu-bench +2026-08-03T11:17:05Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.195.03,13750571.900000,36.00,0.39,0.0001083333,126928356000.000000,20, +2026-08-03T12:39:22Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,42248835.662009,98.93,0.99,0.0002750000,153632129680.032715,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:39:53Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,42082385.647841,99.87,0.99,0.0002750000,153026856901.239990,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:40:24Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,39295472.691030,95.40,0.99,0.0002750000,142892627967.381836,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:40:55Z,runpod,NVIDIA GeForce RTX 5090,32607,170,570.211.01,37085894.820191,99.93,0.99,0.0002750000,134857799346.149094,30,gpuTypeId=NVIDIA GeForce RTX 5090;pod=clyr8n3qak5g1y;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T12:49:27Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,6038487.060385,93.33,0.24,0.0000666667,90577305905.774994,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:49:58Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,5448962.558294,99.40,0.24,0.0000666667,81734438374.410004,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:50:29Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,5134779.381443,98.93,0.24,0.0000666667,77021690721.645004,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:51:00Z,runpod,NVIDIA RTX 2000 Ada Generation,16380,22,550.127.08,4896074.708171,98.87,0.24,0.0000666667,73441120622.564987,30,gpuTypeId=NVIDIA RTX 2000 Ada Generation;pod=7bkrl9p7aqk6fe;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T12:55:05Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,24866844.207723,92.40,0.84,0.0002333333,106572189461.669998,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T12:55:35Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,22867880.851064,93.60,0.84,0.0002333333,98005203647.417145,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T12:56:06Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,22001993.094290,94.00,0.84,0.0002333333,94294256118.385712,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T12:56:37Z,runpod,NVIDIA RTX 6000 Ada Generation,49140,142,550.127.05,21368759.764860,94.20,0.84,0.0002333333,91580398992.257141,30,gpuTypeId=NVIDIA RTX 6000 Ada Generation;pod=x6tv4b17px5z08;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:01:04Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,16039933.444260,99.53,0.57,0.0001583333,101304842805.852631,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:01:35Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,14665398.601399,99.93,0.57,0.0001583333,92623570114.098953,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:02:05Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,13948466.910542,93.33,0.57,0.0001583333,88095580487.633698,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:02:36Z,runpod,NVIDIA RTX PRO 4000 Blackwell,24467,48,580.167.08,13271364.007910,100.00,0.57,0.0001583333,83819141102.589478,30,gpuTypeId=NVIDIA RTX PRO 4000 Blackwell;pod=m40bgym04hz61z;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:06:42Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,21930116.472546,99.00,0.74,0.0002055556,106687053109.683243,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:07:13Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,19058694.792703,99.93,0.74,0.0002055556,92717974667.203781,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:07:43Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,18066915.838304,97.67,0.74,0.0002055556,87893104078.235687,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:08:14Z,runpod,NVIDIA RTX PRO 4500 Blackwell,32623,80,580.126.20,17284602.725158,100.00,0.74,0.0002055556,84087256500.768646,30,gpuTypeId=NVIDIA RTX PRO 4500 Blackwell;pod=mh1w82ngb7p2zo;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:13:21Z,runpod,NVIDIA L4,23034,60,550.127.05,9290709.290709,93.33,0.39,0.0001083333,85760393452.698456,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:13:51Z,runpod,NVIDIA L4,23034,60,550.127.05,8610822.781457,99.07,0.39,0.0001083333,79484517982.679993,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:14:22Z,runpod,NVIDIA L4,23034,60,550.127.05,7973439.265814,100.00,0.39,0.0001083333,73600977838.283066,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:14:54Z,runpod,NVIDIA L4,23034,60,550.127.05,7591500.452489,98.73,0.39,0.0001083333,70075388792.206146,30,gpuTypeId=NVIDIA L4;pod=gmgguw6ldf1zbc;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:31:40Z,runpod,NVIDIA L40S,46068,142,570.124.06,31713810.316140,92.27,0.99,0.0002750000,115322946604.145447,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:32:10Z,runpod,NVIDIA L40S,46068,142,570.124.06,29362911.214333,92.80,0.99,0.0002750000,106774222597.574539,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:32:41Z,runpod,NVIDIA L40S,46068,142,570.124.06,27740105.820106,96.67,0.99,0.0002750000,100873112073.112717,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:33:13Z,runpod,NVIDIA L40S,46068,142,570.124.06,26507780.381830,93.87,0.99,0.0002750000,96391928661.199997,30,gpuTypeId=NVIDIA L40S;pod=dn18gblf3qbgmq;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:37:33Z,runpod,NVIDIA A40,46068,84,570.195.03,8873379.860419,93.20,0.44,0.0001222222,72600380676.155457,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:38:04Z,runpod,NVIDIA A40,46068,84,570.195.03,8407904.830759,93.33,0.44,0.0001222222,68791948615.300919,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:38:35Z,runpod,NVIDIA A40,46068,84,570.195.03,8261925.147735,93.80,0.44,0.0001222222,67597569390.559090,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:39:06Z,runpod,NVIDIA A40,46068,84,570.195.03,8186670.136630,98.73,0.44,0.0001222222,66981846572.427277,30,gpuTypeId=NVIDIA A40;pod=g5ldkbgrop5k79;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T13:42:54Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,8009305.417082,93.33,1.49,0.0004138889,19351341947.312214,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T13:43:26Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6875908.196721,93.33,1.49,0.0004138889,16612932555.835972,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T13:43:58Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6713034.571063,100.00,1.49,0.0004138889,16219412386.460939,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T13:44:29Z,runpod,NVIDIA A100-SXM4-80GB,81920,108,580.126.16,6613882.785808,98.13,1.49,0.0004138889,15979851026.113289,30,gpuTypeId=NVIDIA A100-SXM4-80GB;pod=ragyrp8qvknlgn;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:09:24Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,17382617.382617,93.27,2.99,0.0008305556,20928903872.047222,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:09:55Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,12549446.808511,93.33,2.99,0.0008305556,15109701843.023277,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:10:26Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,10544727.224611,93.47,2.99,0.0008305556,12695992645.016586,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:10:57Z,runpod,NVIDIA H100 80GB HBM3,81559,132,580.126.09,9875405.101373,95.13,2.99,0.0008305556,11890119854.495918,30,gpuTypeId=NVIDIA H100 80GB HBM3;pod=mynvbwnu0nxpyy;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:21:12Z,runpod,NVIDIA H200,143771,132,550.144.03,16516816.516817,93.33,4.39,0.0012194444,13544542018.346514,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:21:43Z,runpod,NVIDIA H200,143771,132,550.144.03,11705034.418605,96.07,4.39,0.0012194444,9598661482.227335,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:22:14Z,runpod,NVIDIA H200,143771,132,550.144.03,9956804.747774,99.33,4.39,0.0012194444,8165033506.147243,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:22:45Z,runpod,NVIDIA H200,143771,132,550.144.03,9227197.411841,96.67,4.39,0.0012194444,7566722251.168017,30,gpuTypeId=NVIDIA H200;pod=8xnkg5j4ui2vhm;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:27:14Z,runpod,NVIDIA B200,183359,160,580.105.08,19182180.851064,92.40,5.89,0.0016361111,11724253151.753889,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:27:45Z,runpod,NVIDIA B200,183359,160,580.105.08,13615163.696588,92.93,5.89,0.0016361111,8321662021.683668,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:28:16Z,runpod,NVIDIA B200,183359,160,580.105.08,11096042.328042,93.27,5.89,0.0016361111,6781961355.000205,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:28:47Z,runpod,NVIDIA B200,183359,160,580.105.08,10390062.059974,93.27,5.89,0.0016361111,6350462379.610595,30,gpuTypeId=NVIDIA B200;pod=0s9oojzm3zh1cq;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T14:33:46Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,19733777.038270,93.13,7.39,0.0020527778,9613206676.288500,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T14:34:18Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,13939195.746095,92.86,7.39,0.0020527778,6790406588.084168,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T14:34:49Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,11441547.837658,93.27,7.39,0.0020527778,5573690421.592531,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T14:35:21Z,runpod,NVIDIA B300 SXM6 AC,275040,160,580.126.09,10758073.741584,93.07,7.39,0.0020527778,5240739576.414398,30,gpuTypeId=NVIDIA B300 SXM6 AC;pod=9datu7s1wmfpkh;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 +2026-08-03T15:00:16Z,runpod,NVIDIA L4,23034,60,580.159.04,9066755.230820,93.33,0.39,0.0001083333,83693125207.569229,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=1000000 +2026-08-03T15:00:46Z,runpod,NVIDIA L4,23034,60,580.159.04,8044630.687831,100.00,0.39,0.0001083333,74258129426.132309,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=4194304 +2026-08-03T15:01:17Z,runpod,NVIDIA L4,23034,60,580.159.04,7447958.434725,100.00,0.39,0.0001083333,68750385551.307678,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=8388608 +2026-08-03T15:01:48Z,runpod,NVIDIA L4,23034,60,580.159.04,7134570.101407,100.00,0.39,0.0001083333,65857570166.833839,30,gpuTypeId=NVIDIA L4;pod=nqpvotonng7rya;ssh=direct;miner=git@illuzen/gpu-bench;batch=16777216 diff --git a/gpu-bench/runpod-shell.sh b/gpu-bench/runpod-shell.sh new file mode 100755 index 0000000..1b9db7c --- /dev/null +++ b/gpu-bench/runpod-shell.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Spin up one RunPod GPU, upload bench scripts, print SSH — leave it running +# so you can iterate on Vulkan / miner commands without recreate-delete cycles. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=runpod-sweep.sh +source "${SCRIPT_DIR}/runpod-sweep.sh" + +STATE_FILE="${STATE_FILE:-${SCRIPT_DIR}/sweep-out/last-shell-pod.env}" + +usage() { + cat <<'EOF' +Usage: + ./runpod-shell.sh "NVIDIA L4" Create pod, upload scripts, print SSH + ./runpod-shell.sh --delete [POD_ID] Terminate pod (default: last created) + ./runpod-shell.sh --ssh ssh into last pod + ./runpod-shell.sh --status Show last pod id / SSH line + +Same env as runpod-sweep.sh (RUNPOD_API_KEY, IMAGE_NAME, CLOUD_TYPE, SSH_KEY, …). + +On the pod, typical debug loop: + cd /workspace/quantus-gpu-bench + bash -x ./remote-run.sh --cost-per-hour 0.39 --duration 30 + + # or step through (see printed hints) +EOF +} + +write_state() { + local pod_id="$1" mode="$2" host="$3" port="$4" cost="$5" gpu_type="$6" + mkdir -p "$(dirname "${STATE_FILE}")" + # Quote values so GPU names with spaces survive `source`. + { + printf "POD_ID=%q\n" "${pod_id}" + printf "SSH_MODE=%q\n" "${mode}" + printf "SSH_HOST=%q\n" "${host}" + printf "SSH_PORT=%q\n" "${port}" + printf "COST_PER_HOUR=%q\n" "${cost}" + printf "GPU_TYPE=%q\n" "${gpu_type}" + printf "SSH_KEY=%q\n" "${SSH_KEY}" + printf "SSH_USER=%q\n" "${SSH_USER}" + printf "REMOTE_DIR=%q\n" "${REMOTE_DIR}" + } >"${STATE_FILE}" +} + +load_state() { + if [[ ! -f "${STATE_FILE}" ]]; then + echo "error: no saved pod state at ${STATE_FILE}" >&2 + echo "Create one with: ./runpod-shell.sh \"NVIDIA L4\"" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "${STATE_FILE}" +} + +print_ssh_hint() { + local mode="$1" host="$2" port="$3" cost="$4" pod_id="$5" + echo + echo "======== interactive pod ready ========" + echo "Pod id: ${pod_id}" + echo "costPerHr: ${cost}" + echo "state file: ${STATE_FILE}" + echo + if [[ "${mode}" == "direct" ]]; then + echo "ssh -i ${SSH_KEY} -p ${port} ${SSH_USER}@${host}" + else + echo "ssh -i ${SSH_KEY} ${host}@ssh.runpod.io" + fi + echo + echo "Re-attach: ./runpod-shell.sh --ssh" + echo "Tear down: ./runpod-shell.sh --delete" + echo + echo "On the pod:" + cat </dev/null || true + vulkaninfo --summary 2>&1 | head -n 60 + + # After editing scripts locally, re-upload from your laptop: + # ./runpod-shell.sh --upload +EOF + echo "=======================================" +} + +cmd_create() { + local gpu_type="$1" + + require_cmd curl + require_cmd ssh + require_cmd python3 + + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + if [[ ! -f "${SSH_KEY}" ]]; then + echo "error: SSH private key not found: ${SSH_KEY}" >&2 + exit 1 + fi + + echo "======== shell: ${gpu_type} ========" >&2 + local create_json pod_id create_rc=0 + set +e + create_json="$(create_pod "${gpu_type}")" + create_rc=$? + set -e + if [[ "${create_rc}" -eq 2 ]]; then + echo "error: no instances available for ${gpu_type} (Community + Secure)." >&2 + echo "tip: try another GPU, or CLOUD_TYPE=SECURE ./runpod-shell.sh \"${gpu_type}\"" >&2 + exit 2 + fi + if [[ "${create_rc}" -ne 0 ]]; then + echo "error: failed to create Pod for ${gpu_type}" >&2 + exit 1 + fi + # create_pod already printed JSON to stdout via create_pod_with_cloud; capture it. + # (create_json from $(create_pod) only has stdout — errors went to stderr.) + pod_id="$(json_pod_field "${create_json}" id)" + if [[ -z "${pod_id}" ]]; then + echo "error: create Pod response missing id:" >&2 + echo "${create_json}" >&2 + exit 1 + fi + echo "Pod id: ${pod_id}" >&2 + + local ready_line tag mode host port cost + if ! ready_line="$(wait_ssh "${pod_id}")"; then + echo "error: SSH never came up — deleting ${pod_id}" >&2 + delete_pod "${pod_id}" + exit 1 + fi + IFS='|' read -r tag mode host port cost <<<"${ready_line}" + if [[ "${tag}" != "ready" || -z "${host}" || -z "${port}" ]]; then + echo "error: bad wait_ssh result: ${ready_line}" >&2 + delete_pod "${pod_id}" + exit 1 + fi + [[ -n "${cost}" ]] || cost="0" + + ssh_upload_files "${mode}" "${host}" "${port}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" \ + "${SCRIPT_DIR}/batch-tune.sh" + ssh_cmd "${mode}" "${host}" "${port}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' '${REMOTE_DIR}/batch-tune.sh'" + + write_state "${pod_id}" "${mode}" "${host}" "${port}" "${cost}" "${gpu_type}" + print_ssh_hint "${mode}" "${host}" "${port}" "${cost}" "${pod_id}" +} + +cmd_upload() { + load_state + echo "Uploading scripts to ${POD_ID} (${REMOTE_DIR}) ..." >&2 + ssh_upload_files "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" \ + "${SCRIPT_DIR}/batch-tune.sh" + ssh_cmd "${SSH_MODE}" "${SSH_HOST}" "${SSH_PORT}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' '${REMOTE_DIR}/batch-tune.sh'" + echo "Done." >&2 +} + +cmd_ssh() { + load_state + local ssh_opts=( + -i "${SSH_KEY}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o PreferredAuthentications=publickey + ) + if [[ "${SSH_MODE}" == "direct" ]]; then + exec ssh "${ssh_opts[@]}" -p "${SSH_PORT}" "${SSH_USER}@${SSH_HOST}" + else + exec ssh "${ssh_opts[@]}" "${SSH_HOST}@ssh.runpod.io" + fi +} + +cmd_status() { + load_state + echo "POD_ID=${POD_ID}" + echo "GPU_TYPE=${GPU_TYPE:-}" + echo "COST_PER_HOUR=${COST_PER_HOUR}" + if [[ "${SSH_MODE}" == "direct" ]]; then + echo "ssh -i ${SSH_KEY} -p ${SSH_PORT} ${SSH_USER}@${SSH_HOST}" + else + echo "ssh -i ${SSH_KEY} ${SSH_HOST}@ssh.runpod.io" + fi + if [[ -n "${RUNPOD_API_KEY:-}" ]]; then + local info status + if info="$(api GET "/pods/${POD_ID}?includeMachine=true" 2>/dev/null)"; then + status="$(json_pod_field "${info}" desiredStatus || true)" + echo "desiredStatus=${status}" + fi + fi +} + +cmd_delete() { + local pod_id="${1:-}" + if [[ -z "${pod_id}" ]]; then + load_state + pod_id="${POD_ID}" + fi + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + delete_pod "${pod_id}" + if [[ -f "${STATE_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${STATE_FILE}" + if [[ "${POD_ID:-}" == "${pod_id}" ]]; then + rm -f "${STATE_FILE}" + fi + fi +} + +case "${1:-}" in + -h | --help | "") + usage + [[ -n "${1:-}" ]] || exit 1 + exit 0 + ;; + --delete) + shift + cmd_delete "${1:-}" + ;; + --ssh) + cmd_ssh + ;; + --upload) + cmd_upload + ;; + --status) + cmd_status + ;; + --*) + echo "error: unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + cmd_create "$1" + ;; +esac diff --git a/gpu-bench/runpod-sweep.sh b/gpu-bench/runpod-sweep.sh new file mode 100755 index 0000000..f11c6f1 --- /dev/null +++ b/gpu-bench/runpod-sweep.sh @@ -0,0 +1,692 @@ +#!/usr/bin/env bash +# Orchestrate RunPod Pods via REST API: for each GPU type, create → SSH → +# remote-run.sh (miner build + benchmark batch-size sweep → CSV) → scp → delete. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_BASE="${RUNPOD_API_BASE:-https://rest.runpod.io/v1}" +OUT_DIR="${OUT_DIR:-${SCRIPT_DIR}/sweep-out}" +# Shared collaborative dataset (tracked in git). Per-pod temps stay in OUT_DIR. +RESULTS_CSV="${RESULTS_CSV:-${SCRIPT_DIR}/results.csv}" +SSH_KEY="${SSH_KEY:-${HOME}/.ssh/id_ed25519}" +SSH_USER="${SSH_USER:-root}" +CLOUD_TYPE="${CLOUD_TYPE:-COMMUNITY}" +# Prefer runpod/base (CUDA + SSH, no PyTorch). Drivers come from the host. +# Use a current Hub tag — stale tags leave Pods RUNNING with runtime=null forever. +# Ubuntu 24.04 base is fine for miner builds; keep NVIDIA_DRIVER_CAPABILITIES=all. +IMAGE_NAME="${IMAGE_NAME:-runpod/base:1.1.0-cuda1281-ubuntu2404}" +TEMPLATE_ID="${TEMPLATE_ID:-}" +# Community hosts often reject large disks ("machine does not have the resources"). +# Cargo build of miner needs more disk than a release binary download. +CONTAINER_DISK_GB="${CONTAINER_DISK_GB:-50}" +VOLUME_GB="${VOLUME_GB:-0}" +DURATION="${DURATION:-30}" +GPU_DEVICES="${GPU_DEVICES:-1}" +BATCH_SIZES="${BATCH_SIZES:-262144 524288 1000000 4194304}" +JOB_INTERVAL="${JOB_INTERVAL:-2}" +DIFFICULTY="${DIFFICULTY:-}" +REMOTE_DIR="${REMOTE_DIR:-/workspace/quantus-gpu-bench}" +MINER_SOURCE="${MINER_SOURCE:-git}" +MINER_REPO="${MINER_REPO:-https://github.com/Quantus-Network/quantus-miner.git}" +MINER_BRANCH="${MINER_BRANCH:-illuzen/gpu-bench}" +KEEP_ON_FAILURE="${KEEP_ON_FAILURE:-0}" +CREATE_RETRIES="${CREATE_RETRIES:-5}" +CREATE_RETRY_SLEEP="${CREATE_RETRY_SLEEP:-15}" +# Retry a new Pod when remote-run exits 42 (compute-only / no Vulkan host). +HOST_RETRIES="${HOST_RETRIES:-3}" +EXIT_COMPUTE_ONLY=42 +# If Community create keeps failing capacity, retry on Secure Cloud. +FALLBACK_SECURE="${FALLBACK_SECURE:-1}" +# Proxy SSH (podId@ssh.runpod.io) is off by default — needs a RunPod-account +# SSH key and is easy to false-trigger while the image is still extracting. +# Direct TCP (after GraphQL shows port 22) is the reliable path. +ALLOW_PROXY_SSH="${ALLOW_PROXY_SSH:-0}" +# Image pull/extract on Community can take several minutes. +SSH_WAIT_SECONDS="${SSH_WAIT_SECONDS:-900}" + +usage() { + cat <<'EOF' +Usage: ./runpod-sweep.sh [gpuTypeId ...] + +Environment: + RUNPOD_API_KEY Required (https://www.runpod.io/console/user/settings) + SSH_KEY Default ~/.ssh/id_ed25519 (public key must be in RunPod) + CLOUD_TYPE COMMUNITY (default) or SECURE + IMAGE_NAME Docker image (default: runpod/base … ubuntu2404 for GLIBC) + TEMPLATE_ID Optional RunPod template id (skips IMAGE_NAME) + DURATION Benchmark seconds per batch size (default 30) + BATCH_SIZES Space-separated gpu-batch-size list (default 256K 512K 1M 4M) + JOB_INTERVAL Simulated NewJob seconds (default 2; 0 = sustained) + DIFFICULTY Optional decimal or max for job simulation + RESULTS_CSV Collaborative dataset to append (default ./results.csv) + OUT_DIR Per-pod temp rows (default ./sweep-out, gitignored) + KEEP_ON_FAILURE=1 Do not delete Pod if remote-run fails + HOST_RETRIES New Pods to try if host lacks Vulkan libs (default 3) + MINER_SOURCE git (default, clone+build) or release + MINER_BRANCH git ref to build (default: illuzen/gpu-bench) + MINER_REPO git URL (default: Quantus-Network/quantus-miner) + CONTAINER_DISK_GB default 50 (cargo build needs room) + +GPU type ids are RunPod strings, e.g.: + "NVIDIA GeForce RTX 4090" + "NVIDIA GeForce RTX 3090" + "NVIDIA RTX A5000" + +Or pass a file via: + ./runpod-sweep.sh --gpus-file gpus.txt + +See README for creating a RunPod template. +EOF +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +# Last failed API response body (for callers to classify errors). +API_LAST_ERROR_BODY="" +API_LAST_HTTP_CODE="" + +# Prints response body to stdout. Exits non-zero on HTTP >= 400. +api() { + local method="$1" + local path="$2" + local data="${3:-}" + local tmp code + API_LAST_ERROR_BODY="" + API_LAST_HTTP_CODE="" + tmp="$(mktemp)" + local args=( + -sS + -X "${method}" + -H "Authorization: Bearer ${RUNPOD_API_KEY}" + -H "Content-Type: application/json" + -o "${tmp}" + -w "%{http_code}" + ) + if [[ -n "${data}" ]]; then + args+=(-d "${data}") + fi + code="$(curl "${args[@]}" "${API_BASE}${path}" || true)" + API_LAST_HTTP_CODE="${code}" + if [[ -z "${code}" || "${code}" == "000" ]]; then + API_LAST_ERROR_BODY="$(cat "${tmp}" 2>/dev/null || true)" + echo "error: RunPod API request failed (network): ${method} ${path}" >&2 + echo "${API_LAST_ERROR_BODY}" >&2 || true + rm -f "${tmp}" + return 1 + fi + if [[ "${code}" -ge 400 ]]; then + API_LAST_ERROR_BODY="$(cat "${tmp}" 2>/dev/null || true)" + echo "error: RunPod API ${method} ${path} -> HTTP ${code}" >&2 + if [[ "${VERBOSE:-0}" == "1" && -n "${data}" ]]; then + echo "request body:" >&2 + echo "${data}" >&2 + fi + echo "response: ${API_LAST_ERROR_BODY}" >&2 + echo >&2 + rm -f "${tmp}" + return 1 + fi + cat "${tmp}" + rm -f "${tmp}" +} + +# True if last API error means "no stock for this GPU" — skip the model. +is_no_capacity_error() { + echo "${API_LAST_ERROR_BODY}" | grep -qiE \ + 'no instances currently available|no .*available|out of capacity|insufficient.*capacity' +} + +# True if gpuTypeIds value is not in the current REST API enum. +is_invalid_gpu_type_error() { + echo "${API_LAST_ERROR_BODY}" | grep -qiE \ + 'gpuTypeIds/items/enum|value must be one of' +} + +# Extract a field from a Pod JSON object. Fails clearly if body is an error list/object. +json_pod_field() { + local body="$1" + local field="$2" + BODY="${body}" FIELD="${field}" python3 - <<'PY' +import json, os, sys +raw = os.environ["BODY"] +field = os.environ["FIELD"] +try: + data = json.loads(raw) +except json.JSONDecodeError as e: + print(f"error: invalid JSON from RunPod: {e}", file=sys.stderr) + print(raw[:2000], file=sys.stderr) + sys.exit(1) +if isinstance(data, list): + print("error: RunPod returned a list (usually an API error), not a Pod object:", file=sys.stderr) + print(json.dumps(data, indent=2)[:4000], file=sys.stderr) + sys.exit(1) +if not isinstance(data, dict): + print(f"error: unexpected RunPod JSON type: {type(data).__name__}", file=sys.stderr) + print(raw[:2000], file=sys.stderr) + sys.exit(1) +if field == "ssh_port": + m = data.get("portMappings") or {} + val = m.get("22") or m.get(22) or "" + if not val: + for p in (data.get("runtime") or {}).get("ports") or []: + if str(p.get("privatePort")) == "22" and p.get("publicPort"): + val = p.get("publicPort") + break +elif field == "publicIp": + val = data.get("publicIp") or "" + if not val: + for p in (data.get("runtime") or {}).get("ports") or []: + if str(p.get("privatePort")) == "22" and p.get("ip"): + val = p.get("ip") + break +elif field == "cost": + val = data.get("costPerHr") or data.get("adjustedCostPerHr") or "" +else: + val = data.get(field) or "" +print(val if val is not None else "") +PY +} + +# SSH helpers. MODE=direct|proxy. For proxy, HOST is pod id and PORT is unused. +ssh_cmd() { + local mode="$1" + local host="$2" + local port="$3" + shift 3 + if [[ -z "${host}" ]]; then + echo "error: ssh_cmd called with empty host (mode=${mode})" >&2 + return 2 + fi + local ssh_opts=( + -i "${SSH_KEY}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o GlobalKnownHostsFile=/dev/null + -o ConnectTimeout=20 + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o LogLevel=ERROR + ) + if [[ "${mode}" == "direct" ]]; then + if [[ -z "${port}" ]]; then + echo "error: direct ssh_cmd missing port" >&2 + return 2 + fi + ssh "${ssh_opts[@]}" -p "${port}" "${SSH_USER}@${host}" "$@" + else + # Basic/proxy SSH — no scp, but command + stdin/stdout pipes work. + ssh "${ssh_opts[@]}" "${host}@ssh.runpod.io" "$@" + fi +} + +# Upload local files into REMOTE_DIR (works for direct and proxy SSH). +ssh_upload_files() { + local mode="$1" + local host="$2" + local port="$3" + shift 3 + ssh_cmd "${mode}" "${host}" "${port}" "mkdir -p '${REMOTE_DIR}'" + local f + for f in "$@"; do + local base + base="$(basename "${f}")" + ssh_cmd "${mode}" "${host}" "${port}" "cat > '${REMOTE_DIR}/${base}'" <"${f}" + done +} + +ssh_download() { + local mode="$1" + local host="$2" + local port="$3" + local remote_path="$4" + local local_path="$5" + ssh_cmd "${mode}" "${host}" "${port}" "cat '${remote_path}'" >"${local_path}" +} + +# When sourced by runpod-shell.sh, only helpers below are used. +build_pod_payload() { + local gpu_type="$1" + local name="$2" + local disk_gb="$3" + local volume_gb="$4" + local cloud_type="$5" + GPU_TYPE="${gpu_type}" \ + CLOUD_TYPE="${cloud_type}" \ + IMAGE_NAME="${IMAGE_NAME}" \ + TEMPLATE_ID="${TEMPLATE_ID}" \ + POD_NAME="${name}" \ + CONTAINER_DISK_GB="${disk_gb}" \ + VOLUME_GB="${volume_gb}" \ + python3 - <<'PY' +import json, os +volume = int(os.environ["VOLUME_GB"]) +# graphics+utility so NVIDIA Container Toolkit mounts Vulkan/GL libs +# (compute-only pods only get CUDA → WGPU sees llvmpipe and fails). +body = { + "name": os.environ["POD_NAME"], + "cloudType": os.environ["CLOUD_TYPE"], + "computeType": "GPU", + "gpuTypeIds": [os.environ["GPU_TYPE"]], + "gpuCount": 1, + "gpuTypePriority": "availability", + "supportPublicIp": True, + "containerDiskInGb": int(os.environ["CONTAINER_DISK_GB"]), + "volumeMountPath": "/workspace", + "ports": ["22/tcp"], + "volumeInGb": volume, + "env": { + "NVIDIA_DRIVER_CAPABILITIES": "all", + "NVIDIA_VISIBLE_DEVICES": "all", + }, +} +template = os.environ.get("TEMPLATE_ID") or "" +if template: + body["templateId"] = template +else: + body["imageName"] = os.environ["IMAGE_NAME"] +print(json.dumps(body)) +PY +} + +create_pod_with_cloud() { + local gpu_type="$1" + local cloud_type="$2" + local name="qbench-$(echo "${gpu_type}" | tr -c 'A-Za-z0-9' '-' | cut -c1-40)-$(date +%s)" + local disk="${CONTAINER_DISK_GB}" + local volume="${VOLUME_GB}" + local attempt payload body_file api_rc + + body_file="$(mktemp)" + for ((attempt = 1; attempt <= CREATE_RETRIES; attempt++)); do + payload="$(build_pod_payload "${gpu_type}" "${name}-${attempt}" "${disk}" "${volume}" "${cloud_type}")" + echo "Creating Pod for ${gpu_type} on ${cloud_type} (attempt ${attempt}/${CREATE_RETRIES}, disk=${disk}G volume=${volume}G) ..." >&2 + if [[ "${VERBOSE:-0}" == "1" ]]; then + echo "payload: ${payload}" >&2 + fi + # Call api in this shell (not $(api …)) so API_LAST_ERROR_BODY is preserved. + set +e + api POST /pods "${payload}" >"${body_file}" + api_rc=$? + set -e + if [[ "${api_rc}" -eq 0 ]]; then + cat "${body_file}" + rm -f "${body_file}" + return 0 + fi + # No stock — shrinking disk will not help; stop this cloud tier. + if is_no_capacity_error; then + echo "no capacity for ${gpu_type} on ${cloud_type}" >&2 + rm -f "${body_file}" + return 2 + fi + # Stale gpuTypeId vs current RunPod schema — skip (update gpus.all.txt). + if is_invalid_gpu_type_error; then + echo "skip: invalid gpuTypeId for API schema: ${gpu_type}" >&2 + echo "tip: refresh gpu-bench/gpus.all.txt from the RunPod /pods enum" >&2 + rm -f "${body_file}" + return 2 + fi + if [[ "${volume}" -gt 0 ]]; then + volume=0 + elif [[ "${disk}" -gt 15 ]]; then + disk=15 + elif [[ "${disk}" -gt 10 ]]; then + disk=10 + fi + if [[ "${attempt}" -lt "${CREATE_RETRIES}" ]]; then + echo "retrying in ${CREATE_RETRY_SLEEP}s (next disk=${disk}G volume=${volume}G) ..." >&2 + sleep "${CREATE_RETRY_SLEEP}" + fi + done + rm -f "${body_file}" + return 1 +} + +# 0 = created (JSON on stdout), 2 = skip (no capacity on any tried tier), 1 = other failure +create_pod() { + local gpu_type="$1" + local rc=0 + + set +e + create_pod_with_cloud "${gpu_type}" "${CLOUD_TYPE}" + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + return 0 + fi + + # Community out of stock → try Secure (same as resource exhaustion). + if [[ "${FALLBACK_SECURE}" == "1" && "${CLOUD_TYPE}" == "COMMUNITY" ]]; then + echo "Community unavailable for ${gpu_type}; trying SECURE cloud ..." >&2 + set +e + create_pod_with_cloud "${gpu_type}" "SECURE" + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + return 0 + fi + fi + + if [[ "${rc}" -eq 2 ]]; then + return 2 + fi + echo "error: exhausted create retries for ${gpu_type}" >&2 + return 1 +} + +graphql_pod_ssh() { + # Prints one line to stdout: "IP PORT" when TCP/22 is mapped, else nothing. + # Quiet unless VERBOSE=1. + local pod_id="$1" + local query payload + query="$(printf 'query { pod(input: {podId: "%s"}) { id desiredStatus costPerHr runtime { ports { ip isIpPublic privatePort publicPort type } } } }' "${pod_id}")" + payload="$(POD_QUERY="${query}" python3 - <<'PY' +import json, os +print(json.dumps({"query": os.environ["POD_QUERY"]})) +PY +)" + local resp + resp="$(curl -sS -X POST \ + -H "content-type: application/json" \ + --url "https://api.runpod.io/graphql?api_key=${RUNPOD_API_KEY}" \ + -d "${payload}" || true)" + BODY="${resp}" VERBOSE="${VERBOSE:-0}" python3 - <<'PY' +import json, os, sys +raw = os.environ["BODY"] +verbose = os.environ.get("VERBOSE") == "1" +try: + data = json.loads(raw) +except Exception as e: + if verbose: + print(f"graphql parse error: {e}", file=sys.stderr) + sys.exit(0) +if data.get("errors"): + if verbose: + print("graphql errors:", json.dumps(data["errors"])[:500], file=sys.stderr) + sys.exit(0) +pod = ((data.get("data") or {}).get("pod")) or {} +ports = ((pod.get("runtime") or {}).get("ports")) or [] +best = None +for p in ports: + if int(p.get("privatePort") or 0) != 22: + continue + if not p.get("ip") or not p.get("publicPort"): + continue + if p.get("isIpPublic"): + best = p + break + if best is None: + best = p +if best: + print(f"{best['ip']} {best['publicPort']}") +elif verbose: + print(f" graphql: status={pod.get('desiredStatus')} ports={len(ports)}", file=sys.stderr) +PY +} + +# Prints one stdout line: ready|||| +# Do not print anything else to stdout (stderr is OK). +wait_ssh() { + local pod_id="$1" + if [[ -z "${pod_id}" ]]; then + echo "error: wait_ssh: empty pod id" >&2 + return 1 + fi + local deadline=$((SECONDS + SSH_WAIT_SECONDS)) + local last_heartbeat=0 + local spun_at=$SECONDS + + echo " waiting for image extract + SSH (up to ${SSH_WAIT_SECONDS}s) ..." >&2 + + while (( SECONDS < deadline )); do + local info ip port status cost g_ip g_port elapsed + elapsed=$((SECONDS - spun_at)) + if ! info="$(api GET "/pods/${pod_id}?includeMachine=true")"; then + sleep 10 + continue + fi + ip="$(json_pod_field "${info}" publicIp)" || return 1 + port="$(json_pod_field "${info}" ssh_port)" || return 1 + status="$(json_pod_field "${info}" desiredStatus)" || return 1 + cost="$(json_pod_field "${info}" cost)" || return 1 + [[ -n "${cost}" ]] || cost="0" + + g_ip="" + g_port="" + if read -r g_ip g_port <<<"$(graphql_pod_ssh "${pod_id}")"; then + if [[ -n "${g_ip}" && -n "${g_port}" ]]; then + ip="${g_ip}" + port="${g_port}" + fi + fi + + # Only attempt SSH after port mappings exist (container finished starting). + if [[ "${status}" == "RUNNING" && -n "${ip}" && -n "${port}" ]]; then + echo " ports up (${elapsed}s) — probing ${SSH_USER}@${ip}:${port}" >&2 + if ssh_cmd direct "${ip}" "${port}" nvidia-smi >/dev/null 2>&1; then + printf 'ready|direct|%s|%s|%s\n' "${ip}" "${port}" "${cost}" + return 0 + fi + echo " SSH daemon not ready yet, retrying ..." >&2 + elif (( elapsed - last_heartbeat >= 30 )); then + last_heartbeat=$elapsed + if [[ -n "${ip}" && -n "${port}" ]]; then + echo " still waiting ${elapsed}s (tcp=${ip}:${port}, ssh not ready)" >&2 + else + echo " still waiting ${elapsed}s (image extract / ports pending)" >&2 + fi + fi + + # Optional proxy — only after ports exist, and only if explicitly enabled. + if [[ "${ALLOW_PROXY_SSH}" == "1" && "${status}" == "RUNNING" && -n "${ip}" && -n "${port}" ]]; then + if ssh_cmd proxy "${pod_id}" "22" nvidia-smi >/dev/null 2>&1; then + printf 'ready|proxy|%s|22|%s\n' "${pod_id}" "${cost}" + return 0 + fi + fi + + sleep 10 + done + echo "error: timed out after ${SSH_WAIT_SECONDS}s waiting for Pod ${pod_id}" >&2 + echo "tip: console logs still extracting? increase SSH_WAIT_SECONDS=1200" >&2 + return 1 +} + +delete_pod() { + local pod_id="$1" + echo "Deleting Pod ${pod_id} ..." >&2 + api DELETE "/pods/${pod_id}" >/dev/null || true +} + +run_one_attempt() { + local gpu_type="$1" + local create_json pod_id create_rc=0 + set +e + create_json="$(create_pod "${gpu_type}")" + create_rc=$? + set -e + if [[ "${create_rc}" -eq 2 ]]; then + echo "skip: no instances available for ${gpu_type}" >&2 + return 2 + fi + if [[ "${create_rc}" -ne 0 ]]; then + echo "error: failed to create Pod for ${gpu_type}" >&2 + return 1 + fi + if ! pod_id="$(json_pod_field "${create_json}" id)"; then + echo "error: create Pod response missing id:" >&2 + echo "${create_json}" >&2 + return 1 + fi + if [[ -z "${pod_id}" ]]; then + echo "error: empty Pod id. Full response:" >&2 + echo "${create_json}" >&2 + return 1 + fi + echo "Pod id: ${pod_id}" >&2 + + local ready_line tag mode host port cost + if ! ready_line="$(wait_ssh "${pod_id}")"; then + delete_pod "${pod_id}" + return 1 + fi + IFS='|' read -r tag mode host port cost <<<"${ready_line}" + if [[ "${tag}" != "ready" || -z "${mode}" || -z "${host}" || -z "${port}" ]]; then + echo "error: bad wait_ssh result: ${ready_line}" >&2 + delete_pod "${pod_id}" + return 1 + fi + if [[ -z "${cost}" ]]; then + cost="0" + fi + if [[ "${mode}" == "direct" ]]; then + echo "SSH ${SSH_USER}@${host} -p ${port} (costPerHr=${cost})" >&2 + else + echo "SSH ${host}@ssh.runpod.io (costPerHr=${cost})" >&2 + fi + + ssh_upload_files "${mode}" "${host}" "${port}" \ + "${SCRIPT_DIR}/remote-run.sh" \ + "${SCRIPT_DIR}/record.sh" + + local remote_rc=0 + set +e + ssh_cmd "${mode}" "${host}" "${port}" \ + "chmod +x '${REMOTE_DIR}/remote-run.sh' '${REMOTE_DIR}/record.sh' && \ + WORK_DIR='${REMOTE_DIR}' \ + MINER_SOURCE='${MINER_SOURCE}' \ + MINER_REPO='${MINER_REPO}' \ + MINER_BRANCH='${MINER_BRANCH}' \ + FORCE_MINER_BUILD='${FORCE_MINER_BUILD:-0}' \ + '${REMOTE_DIR}/remote-run.sh' \ + --provider runpod \ + --cost-per-hour '${cost}' \ + --duration '${DURATION}' \ + --gpu-devices '${GPU_DEVICES}' \ + --batch-sizes '${BATCH_SIZES}' \ + --job-interval '${JOB_INTERVAL}' \ + ${DIFFICULTY:+--difficulty '${DIFFICULTY}'} \ + --miner-source '${MINER_SOURCE}' \ + --miner-repo '${MINER_REPO}' \ + --miner-branch '${MINER_BRANCH}' \ + --notes 'gpuTypeId=${gpu_type};pod=${pod_id};ssh=${mode};miner=${MINER_SOURCE}@${MINER_BRANCH}'" + remote_rc=$? + set -e + + if [[ "${remote_rc}" -eq 0 ]]; then + local local_row="${OUT_DIR}/row-${pod_id}.csv" + ssh_download "${mode}" "${host}" "${port}" \ + "${REMOTE_DIR}/results.csv" "${local_row}" + tail -n +2 "${local_row}" >>"${RESULTS_CSV}" + echo "Appended results to ${RESULTS_CSV}" >&2 + delete_pod "${pod_id}" + return 0 + fi + + echo "error: remote-run failed on ${gpu_type} (pod ${pod_id}, rc=${remote_rc})" >&2 + if [[ "${KEEP_ON_FAILURE}" == "1" ]]; then + echo "KEEP_ON_FAILURE=1 — leaving Pod ${pod_id} up" >&2 + if [[ "${mode}" == "direct" ]]; then + echo " ssh -i ${SSH_KEY} -p ${port} ${SSH_USER}@${host}" >&2 + else + echo " ssh -i ${SSH_KEY} ${host}@ssh.runpod.io" >&2 + fi + return "${remote_rc}" + fi + delete_pod "${pod_id}" + # Propagate compute-only so caller can try another host. + return "${remote_rc}" +} + +run_one() { + local gpu_type="$1" + local attempt rc=1 + for ((attempt = 1; attempt <= HOST_RETRIES; attempt++)); do + if [[ "${attempt}" -gt 1 ]]; then + echo "host retry ${attempt}/${HOST_RETRIES} for ${gpu_type} ..." >&2 + fi + set +e + run_one_attempt "${gpu_type}" + rc=$? + set -e + if [[ "${rc}" -eq 0 || "${rc}" -eq 2 ]]; then + return "${rc}" + fi + if [[ "${rc}" -eq "${EXIT_COMPUTE_ONLY}" && "${attempt}" -lt "${HOST_RETRIES}" ]]; then + echo "compute-only / bad Vulkan host — trying another Pod ..." >&2 + continue + fi + return "${rc}" + done + return "${rc}" +} + +runpod_sweep_main() { + local GPUS=() + while [[ $# -gt 0 ]]; do + case "$1" in + --gpus-file) + while IFS= read -r line || [[ -n "${line}" ]]; do + [[ -z "${line}" || "${line}" =~ ^# ]] && continue + GPUS+=("${line}") + done <"${2:?}" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + GPUS+=("$1") + shift + ;; + esac + done + + if [[ "${#GPUS[@]}" -eq 0 ]]; then + echo "error: pass at least one GPU type id" >&2 + usage >&2 + exit 1 + fi + + require_cmd curl + require_cmd ssh + require_cmd python3 + + if [[ -z "${RUNPOD_API_KEY:-}" ]]; then + echo "error: set RUNPOD_API_KEY" >&2 + exit 1 + fi + if [[ ! -f "${SSH_KEY}" ]]; then + echo "error: SSH private key not found: ${SSH_KEY}" >&2 + exit 1 + fi + + mkdir -p "${OUT_DIR}" + if [[ ! -f "${RESULTS_CSV}" ]]; then + printf '%s\n' \ + "timestamp,cloud_provider,gpu_model,vram_mb,sm_count,driver_version,hashrate,gpu_utilization_pct,cost_per_hour,cost_per_sec,hash_per_dollar,sample_seconds,notes" \ + >"${RESULTS_CSV}" + fi + + local failed=0 + local gpu + for gpu in "${GPUS[@]}"; do + echo "======== ${gpu} ========" >&2 + if ! run_one "${gpu}"; then + failed=1 + fi + done + + echo "Dataset CSV: ${RESULTS_CSV}" >&2 + exit "${failed}" +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + runpod_sweep_main "$@" +fi diff --git a/gpu-bench/setup.sh b/gpu-bench/setup.sh new file mode 100755 index 0000000..e7ae5f7 --- /dev/null +++ b/gpu-bench/setup.sh @@ -0,0 +1,551 @@ +#!/usr/bin/env bash +# Start (or stop) a native Quantus node + GPU miner on this host. +# Same-machine layout for cloud GPU rentals (RunPod, Vast, etc.). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUN_DIR="${SCRIPT_DIR}/.run" +ENV_FILE="${SCRIPT_DIR}/.env" +BIN_DIR="${RUN_DIR}/bin" + +DEFAULT_MINER_DIR="${REPO_ROOT}" +DEFAULT_CHAIN_DIR="$(cd "${REPO_ROOT}/.." && pwd)/chain" +NODE_CONTAINER_NAME="${NODE_CONTAINER_NAME:-quantus-gpu-bench-node}" + +NODE_MODE="native" # native | docker +DEV_MODE=0 # 1 = --dev local chain (no rewards hash / no sync) + +usage() { + cat <<'EOF' +Usage: ./setup.sh [start|stop|status|fetch-node|wormhole] [--docker] [--dev] + + start Download/build binaries, start native node + GPU miner (default) + stop Stop miner and node + status Show run state + fetch-node Download quantus-node into .run/bin/ (no start) + wormhole Print a new wormhole address + inner_hash (for REWARDS_INNER_HASH) + --dev Local --dev chain (no Planck sync, rewards hash optional) + --docker Use Docker for the node instead of a native binary + (not recommended on RunPod / nested-Docker hosts) + +Environment (see .env.example): + REWARDS_INNER_HASH Required for start unless --dev + QUANTUS_NODE_BIN Prebuilt quantus-node (skips download) + MINER_BIN Prebuilt quantus-miner (skips cargo build) + GPU_DEVICES Number of GPUs (default: 1) +EOF +} + +load_env() { + if [[ -f "${ENV_FILE}" ]]; then + # shellcheck disable=SC1090 + set -a + source "${ENV_FILE}" + set +a + fi +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "error: required command not found: $1" >&2 + exit 1 + fi +} + +check_prereqs() { + require_cmd nvidia-smi + if ! nvidia-smi >/dev/null 2>&1; then + echo "error: nvidia-smi failed — NVIDIA driver required for this toolkit" >&2 + exit 1 + fi + if [[ "${NODE_MODE}" == "docker" ]]; then + require_cmd docker + else + require_cmd curl + require_cmd tar + fi +} + +stop_pidfile() { + local pidfile="$1" + local label="$2" + if [[ ! -f "${pidfile}" ]]; then + return 0 + fi + local pid + pid="$(cat "${pidfile}")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Stopping ${label} (pid ${pid}) ..." + kill "${pid}" 2>/dev/null || true + local i + for ((i = 0; i < 20; i++)); do + kill -0 "${pid}" 2>/dev/null || break + sleep 0.5 + done + if kill -0 "${pid}" 2>/dev/null; then + kill -9 "${pid}" 2>/dev/null || true + fi + fi + rm -f "${pidfile}" +} + +resolve_miner_bin() { + if [[ -n "${MINER_BIN:-}" ]]; then + if [[ ! -x "${MINER_BIN}" ]]; then + echo "error: MINER_BIN is not executable: ${MINER_BIN}" >&2 + exit 1 + fi + echo "${MINER_BIN}" + return + fi + + local miner_dir="${QUANTUS_MINER_DIR:-${DEFAULT_MINER_DIR}}" + if [[ ! -d "${miner_dir}" ]]; then + echo "error: quantus-miner not found at ${miner_dir}" >&2 + echo "Set QUANTUS_MINER_DIR or MINER_BIN." >&2 + exit 1 + fi + + local bin="${miner_dir}/target/release/quantus-miner" + if [[ -x "${bin}" ]]; then + echo "${bin}" + return + fi + + require_cmd cargo + echo "Building quantus-miner (release) in ${miner_dir} ..." >&2 + ( + cd "${miner_dir}" + cargo build -p miner-cli --release + ) >&2 + if [[ ! -x "${bin}" ]]; then + echo "error: expected binary missing: ${bin}" >&2 + exit 1 + fi + echo "${bin}" +} + +detect_node_target() { + local os arch + case "$(uname -s)" in + Linux) os="linux" ;; + Darwin) os="macos" ;; + *) + echo "error: unsupported OS for node download: $(uname -s)" >&2 + exit 1 + ;; + esac + case "$(uname -m)" in + x86_64 | amd64) + if [[ "${os}" == "linux" ]]; then + arch="x86_64-unknown-linux-gnu" + else + arch="x86_64-apple-darwin" + fi + ;; + arm64 | aarch64) + if [[ "${os}" == "linux" ]]; then + arch="aarch64-unknown-linux-gnu" + else + arch="aarch64-apple-darwin" + fi + ;; + *) + echo "error: unsupported arch for node download: $(uname -m)" >&2 + exit 1 + ;; + esac + echo "${arch}" +} + +download_node_binary() { + local dest="$1" + local target + target="$(detect_node_target)" + echo "Fetching latest quantus-node release (${target}) ..." >&2 + + local release_json tag asset_url tmp + release_json="$(curl -fsSL https://api.github.com/repos/Quantus-Network/chain/releases/latest)" + tag="$(echo "${release_json}" | grep -o '"tag_name": "[^"]*"' | head -n 1 | cut -d'"' -f4)" + if [[ -z "${tag}" ]]; then + echo "error: could not determine latest chain release tag" >&2 + exit 1 + fi + + asset_url="https://github.com/Quantus-Network/chain/releases/download/${tag}/quantus-node-${tag}-${target}.tar.gz" + echo "Downloading ${asset_url} ..." >&2 + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" RETURN + if ! curl -fL "${asset_url}" -o "${tmp}/node.tar.gz"; then + echo "error: failed to download quantus-node for ${target}" >&2 + echo "Set QUANTUS_NODE_BIN to a local binary, or build from the chain repo." >&2 + exit 1 + fi + tar -xzf "${tmp}/node.tar.gz" -C "${tmp}" + if [[ ! -f "${tmp}/quantus-node" ]]; then + echo "error: archive did not contain quantus-node" >&2 + exit 1 + fi + mkdir -p "$(dirname "${dest}")" + mv "${tmp}/quantus-node" "${dest}" + chmod +x "${dest}" + echo "Installed quantus-node -> ${dest}" >&2 +} + +resolve_node_bin() { + if [[ -n "${QUANTUS_NODE_BIN:-}" ]]; then + if [[ ! -x "${QUANTUS_NODE_BIN}" ]]; then + echo "error: QUANTUS_NODE_BIN is not executable: ${QUANTUS_NODE_BIN}" >&2 + exit 1 + fi + echo "${QUANTUS_NODE_BIN}" + return + fi + + if command -v quantus-node >/dev/null 2>&1; then + command -v quantus-node + return + fi + + local chain_dir="${QUANTUS_CHAIN_DIR:-${DEFAULT_CHAIN_DIR}}" + if [[ -x "${chain_dir}/target/release/quantus-node" ]]; then + echo "${chain_dir}/target/release/quantus-node" + return + fi + + local cached="${BIN_DIR}/quantus-node" + if [[ -x "${cached}" ]]; then + echo "${cached}" + return + fi + + download_node_binary "${cached}" + echo "${cached}" +} + +ensure_rewards() { + local rewards="${REWARDS_INNER_HASH:-}" + if [[ -z "${rewards}" || "${rewards}" == "0xyour_inner_hash_here" ]]; then + echo "error: set REWARDS_INNER_HASH in ${ENV_FILE} or the environment" >&2 + echo "Generate with (after node binary is available):" >&2 + echo " quantus-node key quantus --scheme wormhole" >&2 + exit 1 + fi +} + +ensure_node_key() { + local node_bin="$1" + local key_file="${RUN_DIR}/node-keys/key_node" + mkdir -p "$(dirname "${key_file}")" + if [[ ! -f "${key_file}" ]]; then + echo "Generating node key at ${key_file} ..." >&2 + "${node_bin}" key generate-node-key --file "${key_file}" + fi + echo "${key_file}" +} + +start_node_native() { + local node_bin + node_bin="$(resolve_node_bin)" + + local rpc_port="${RPC_PORT:-9944}" + local prom_port="${PROMETHEUS_PORT:-9615}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + local base_path="${RUN_DIR}/node-data" + mkdir -p "${base_path}" + + stop_pidfile "${RUN_DIR}/node.pid" "node" + + local log_file="${RUN_DIR}/node.log" + echo "Starting native quantus-node: ${node_bin}" + + if [[ "${DEV_MODE}" -eq 1 ]]; then + echo " --dev --miner-listen-port ${miner_listen_port}" + nohup "${node_bin}" \ + --dev \ + --base-path "${base_path}" \ + --rpc-port "${rpc_port}" \ + --prometheus-port "${prom_port}" \ + --prometheus-external \ + --miner-listen-port "${miner_listen_port}" \ + --rpc-cors all \ + >"${log_file}" 2>&1 & + else + ensure_rewards + local key_file + key_file="$(ensure_node_key "${node_bin}")" + local chain="${CHAIN:-planck}" + local node_name="${NODE_NAME:-gpu-bench-node}" + local p2p_port="${P2P_PORT:-30333}" + echo " --miner-listen-port ${miner_listen_port} --chain ${chain}" + nohup "${node_bin}" \ + --validator \ + --base-path "${base_path}" \ + --chain "${chain}" \ + --node-key-file "${key_file}" \ + --rewards-inner-hash "${REWARDS_INNER_HASH}" \ + --name "${node_name}" \ + --port "${p2p_port}" \ + --rpc-port "${rpc_port}" \ + --prometheus-port "${prom_port}" \ + --prometheus-external \ + --miner-listen-port "${miner_listen_port}" \ + --wasm-execution compiled \ + --db-cache 2048 \ + --rpc-cors all \ + --max-blocks-per-request 64 \ + >"${log_file}" 2>&1 & + fi + + local pid=$! + echo "${pid}" >"${RUN_DIR}/node.pid" + echo "native" >"${RUN_DIR}/node.mode" + echo "${node_bin}" >"${RUN_DIR}/node.bin" + echo "Node pid ${pid}; miner QUIC on 127.0.0.1:${miner_listen_port}/udp" + echo "Logs: ${log_file}" +} + +start_node_docker() { + ensure_rewards + + if docker ps -a --format '{{.Names}}' | grep -qx "${NODE_CONTAINER_NAME}"; then + echo "Removing existing container ${NODE_CONTAINER_NAME} ..." + docker rm -f "${NODE_CONTAINER_NAME}" >/dev/null + fi + + local node_version="${NODE_VERSION:-latest}" + local chain="${CHAIN:-planck}" + local node_name="${NODE_NAME:-gpu-bench-node}" + local p2p_port="${P2P_PORT:-30333}" + local rpc_port="${RPC_PORT:-9944}" + local prom_port="${PROMETHEUS_PORT:-9615}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + + mkdir -p "${RUN_DIR}/node-keys" "${RUN_DIR}/node-data" + + echo "Starting quantus-node via Docker (${NODE_CONTAINER_NAME}) ..." + docker run -d \ + --name "${NODE_CONTAINER_NAME}" \ + --restart unless-stopped \ + --platform linux/amd64 \ + -v "${SCRIPT_DIR}/init-node.sh:/init-node.sh:ro" \ + -v "${RUN_DIR}/node-keys:/node-keys" \ + -v "${RUN_DIR}/node-data:/var/lib/quantus" \ + -p "${p2p_port}:30333" \ + -p "${rpc_port}:9944" \ + -p "${prom_port}:9615" \ + -p "${miner_listen_port}:9833/udp" \ + --entrypoint /init-node.sh \ + "ghcr.io/quantus-network/quantus-node:${node_version}" \ + --validator \ + --base-path /var/lib/quantus \ + --chain "${chain}" \ + --node-key-file /node-keys/key_node \ + --rewards-inner-hash "${REWARDS_INNER_HASH}" \ + --name "${node_name}" \ + --wasm-execution compiled \ + --db-cache 2048 \ + --rpc-cors all \ + --prometheus-external \ + --miner-listen-port 9833 \ + >/dev/null + + echo "${NODE_CONTAINER_NAME}" >"${RUN_DIR}/node.container" + echo "docker" >"${RUN_DIR}/node.mode" + echo "Node listening for miners on 127.0.0.1:${miner_listen_port}/udp" +} + +wait_for_miner_port() { + local port="$1" + local attempts=30 + local i + for ((i = 1; i <= attempts; i++)); do + if curl -sf "http://127.0.0.1:${port}/metrics" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "warning: miner metrics not ready on :${port} after ${attempts}s (continuing)" >&2 +} + +start_miner() { + local miner_bin + miner_bin="$(resolve_miner_bin)" + + local gpu_devices="${GPU_DEVICES:-1}" + local metrics_port="${METRICS_PORT:-9900}" + local miner_listen_port="${HOST_MINER_LISTEN_PORT:-9833}" + local miner_log="${MINER_LOG:-info}" + + stop_pidfile "${RUN_DIR}/miner.pid" "miner" + + mkdir -p "${RUN_DIR}" + local log_file="${RUN_DIR}/miner.log" + echo "Starting GPU miner: ${miner_bin}" + echo " --node-addr 127.0.0.1:${miner_listen_port} --gpu-devices ${gpu_devices} --cpu-workers 0" + + RUST_LOG="${miner_log}" nohup "${miner_bin}" serve \ + --node-addr "127.0.0.1:${miner_listen_port}" \ + --gpu-devices "${gpu_devices}" \ + --cpu-workers 0 \ + --metrics-port "${metrics_port}" \ + >"${log_file}" 2>&1 & + + local pid=$! + echo "${pid}" >"${RUN_DIR}/miner.pid" + echo "${miner_bin}" >"${RUN_DIR}/miner.bin" + echo "${metrics_port}" >"${RUN_DIR}/metrics.port" + + wait_for_miner_port "${metrics_port}" + echo "Miner pid ${pid}; metrics http://127.0.0.1:${metrics_port}/metrics" + echo "Logs: ${log_file}" +} + +do_fetch_node() { + load_env + require_cmd curl + require_cmd tar + mkdir -p "${BIN_DIR}" + local bin + bin="$(resolve_node_bin)" + echo "quantus-node ready: ${bin}" + echo "Generate rewards hash with: ./setup.sh wormhole" +} + +do_wormhole() { + load_env + require_cmd curl + require_cmd tar + mkdir -p "${BIN_DIR}" + local bin + bin="$(resolve_node_bin)" + echo "Using ${bin}" >&2 + echo "Save the inner_hash into .env as REWARDS_INNER_HASH" >&2 + echo >&2 + "${bin}" key quantus --scheme wormhole +} + +do_start() { + load_env + check_prereqs + mkdir -p "${RUN_DIR}" + + if [[ "${NODE_MODE}" == "docker" ]]; then + start_node_docker + else + start_node_native + fi + + sleep 2 + start_miner + echo + echo "Stack is up (same host). Next:" + echo " ./record.sh --provider --cost-per-hour " + echo "Or hardware-only (no node): ./record.sh --benchmark --provider --cost-per-hour " +} + +do_stop() { + load_env + stop_pidfile "${RUN_DIR}/miner.pid" "miner" + stop_pidfile "${RUN_DIR}/node.pid" "node" + + local container="${NODE_CONTAINER_NAME}" + if [[ -f "${RUN_DIR}/node.container" ]]; then + container="$(cat "${RUN_DIR}/node.container")" + fi + if command -v docker >/dev/null 2>&1 && docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "${container}"; then + echo "Removing node container ${container} ..." + docker rm -f "${container}" >/dev/null + fi + rm -f "${RUN_DIR}/node.container" "${RUN_DIR}/node.mode" + echo "Stopped." +} + +do_status() { + load_env + echo "Run dir: ${RUN_DIR}" + local mode="unknown" + if [[ -f "${RUN_DIR}/node.mode" ]]; then + mode="$(cat "${RUN_DIR}/node.mode")" + fi + echo "Node mode: ${mode}" + + if [[ -f "${RUN_DIR}/node.pid" ]]; then + local pid + pid="$(cat "${RUN_DIR}/node.pid")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Node: running (pid ${pid})" + else + echo "Node: not running (stale pid ${pid})" + fi + elif [[ -f "${RUN_DIR}/node.container" ]]; then + local c + c="$(cat "${RUN_DIR}/node.container")" + if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' | grep -qx "${c}"; then + echo "Node: running (docker ${c})" + else + echo "Node: not running (recorded docker ${c})" + fi + else + echo "Node: not started via setup.sh" + fi + + if [[ -f "${RUN_DIR}/miner.pid" ]]; then + local pid + pid="$(cat "${RUN_DIR}/miner.pid")" + if kill -0 "${pid}" 2>/dev/null; then + echo "Miner: running (pid ${pid})" + else + echo "Miner: not running (stale pid ${pid})" + fi + else + echo "Miner: not started via setup.sh" + fi + + local metrics_port="${METRICS_PORT:-9900}" + if [[ -f "${RUN_DIR}/metrics.port" ]]; then + metrics_port="$(cat "${RUN_DIR}/metrics.port")" + fi + if curl -sf "http://127.0.0.1:${metrics_port}/metrics" >/dev/null 2>&1; then + echo "Metrics: ok on :${metrics_port}" + else + echo "Metrics: not reachable on :${metrics_port}" + fi +} + +cmd="start" +while [[ $# -gt 0 ]]; do + case "$1" in + start | stop | status | fetch-node | wormhole) + cmd="$1" + shift + ;; + --docker) + NODE_MODE="docker" + shift + ;; + --dev) + DEV_MODE=1 + shift + ;; + -h | --help | help) + usage + exit 0 + ;; + *) + echo "error: unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +case "${cmd}" in + start) do_start ;; + stop) do_stop ;; + status) do_status ;; + fetch-node) do_fetch_node ;; + wormhole) do_wormhole ;; +esac