diff --git a/README.md b/README.md index fbc3d03..6fa1327 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,17 @@ Storage keys are derived from pallet and item names, and every value is decoded Metadata and the live values are read at the block being bitten (`--rc-bite-at` / a para's `bite_at`), so they match the state being imported. Parachains use a default public endpoint when no `rpc_endpoint` is configured; if it can't be reached, the bite still runs with a warning and those overrides go unverified. Custom parachains are only verified when their config supplies an `rpc_endpoint`. +#### One artifact, restored elsewhere + +`pack` puts everything a spawn needs into a single file — chain-specs, db snapshots, `config.toml`, the overrides that were applied, `manifest.json`, `ready.json` and any carried upgrade blob: + +```sh +zombie-bite pack -d /tmp/base_path -s bite # -> /tmp/base_path/bite-bundle.tgz +zombie-bite spawn -d /other/path --bundle bite-bundle.tgz +``` + +`spawn` re-points the spec and snapshot paths at wherever the bundle was unpacked, so the artifacts do not have to land in the directory they were produced in. + #### Publishing bootnodes A published chain-spec ships with `bootNodes: []` — that is what keeps a fork from dialing the network it was forked from, but it also means a node this process did not start has no way to find the fork. `--publish-bootnodes` fills the list with the fork's own nodes, in the artifacts generated at teardown (the `bite` bundle is left untouched): diff --git a/src/bundle.rs b/src/bundle.rs new file mode 100644 index 0000000..7e73bdf --- /dev/null +++ b/src/bundle.rs @@ -0,0 +1,168 @@ +//! Pack a step's artifacts into one file and restore it elsewhere. +//! +//! A bite is often produced in CI and consumed hours later on another machine, +//! so everything needed to spawn has to travel together: the chain-specs, the +//! db snapshots, the config, the overrides that were applied, the manifest, and +//! any runtime carried as an authorized upgrade. `spawn` re-points the spec and +//! snapshot paths at wherever the bundle was unpacked, so the artifacts do not +//! have to land in the same directory they were produced in. + +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, bail}; +use flate2::{read::GzDecoder, write::GzEncoder, Compression}; +use tar::Archive; +use tokio::fs; +use tracing::info; + +use crate::{config::Step, manifest::MANIFEST_FILE}; + +/// Files that live in the base dir rather than the step dir, and are part of the +/// bundle when present. +const BASE_FILES: [&str; 3] = [MANIFEST_FILE, "ready.json", "ports.json"]; + +fn default_bundle_name(step: Step) -> String { + format!("{}-bundle.tgz", step.dir()) +} + +/// Pack `/` plus the base-level files into a single `.tgz`. +pub async fn pack( + base_path: &Path, + step: Step, + out: Option, +) -> Result { + let step_dir = base_path.join(step.dir()); + if !fs::try_exists(&step_dir).await? { + bail!( + "nothing to pack: {} does not exist", + step_dir.to_string_lossy() + ); + } + + let out = out.unwrap_or_else(|| base_path.join(default_bundle_name(step))); + let file = std::fs::File::create(&out) + .map_err(|e| anyhow!("can't create {}: {e}", out.to_string_lossy()))?; + let mut encoder = GzEncoder::new(file, Compression::fast()); + { + let mut archive = tar::Builder::new(&mut encoder); + // Paths inside the archive are relative to the base dir, so unpacking + // into any directory reproduces the same layout. + archive.append_dir_all(step.dir(), &step_dir)?; + for name in BASE_FILES { + let path = base_path.join(name); + if fs::try_exists(&path).await? { + archive.append_path_with_name(&path, name)?; + } + } + // Runtimes carried as an authorized upgrade. + let mut entries = fs::read_dir(base_path).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().to_string_lossy().to_string(); + if name.ends_with("-upgrade.wasm") { + archive.append_path_with_name(entry.path(), &name)?; + } + } + archive.finish()?; + } + encoder.finish()?; + + info!("📦 bundle written to {}", out.to_string_lossy()); + Ok(out) +} + +/// Unpack a bundle into `base_path`. +pub async fn unpack(bundle: &Path, base_path: &Path) -> Result<(), anyhow::Error> { + if !fs::try_exists(bundle).await? { + bail!("bundle {} does not exist", bundle.to_string_lossy()); + } + fs::create_dir_all(base_path).await?; + + let file = std::fs::File::open(bundle) + .map_err(|e| anyhow!("can't open {}: {e}", bundle.to_string_lossy()))?; + Archive::new(GzDecoder::new(file)).unpack(base_path)?; + + info!( + "📦 bundle {} unpacked into {}", + bundle.to_string_lossy(), + base_path.to_string_lossy() + ); + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn pack_then_unpack_reproduces_the_layout() { + let root = std::env::temp_dir().join("zb-bundle-test"); + let (from, to) = (root.join("from"), root.join("to")); + let _ = fs::remove_dir_all(&root).await; + fs::create_dir_all(from.join("bite")).await.unwrap(); + fs::create_dir_all(&to).await.unwrap(); + + // step dir: spec, snapshot, config and the overrides that were applied + for (name, content) in [ + ("kusama-spec.json", "{}"), + ("kusama-snap.tgz", "snap"), + ("config.toml", "[relaychain]"), + ("rc_overrides.json", r#"{"overrides":{}}"#), + ] { + fs::write(from.join("bite").join(name), content) + .await + .unwrap(); + } + // base dir files + fs::write(from.join(MANIFEST_FILE), r#"{"created_at":1}"#) + .await + .unwrap(); + fs::write(from.join("ready.json"), r#"{"rc_start_block":7}"#) + .await + .unwrap(); + fs::write(from.join("kusama-upgrade.wasm"), "wasm") + .await + .unwrap(); + // not part of the bundle + fs::write(from.join("unrelated.log"), "noise") + .await + .unwrap(); + + let bundle = pack(&from, Step::Bite, None).await.unwrap(); + unpack(&bundle, &to).await.unwrap(); + + for name in [ + "kusama-spec.json", + "kusama-snap.tgz", + "config.toml", + "rc_overrides.json", + ] { + assert!( + fs::try_exists(to.join("bite").join(name)).await.unwrap(), + "missing {name}" + ); + } + assert_eq!( + fs::read_to_string(to.join("ready.json")).await.unwrap(), + r#"{"rc_start_block":7}"# + ); + assert!(fs::try_exists(to.join(MANIFEST_FILE)).await.unwrap()); + assert!(fs::try_exists(to.join("kusama-upgrade.wasm")) + .await + .unwrap()); + assert!(!fs::try_exists(to.join("unrelated.log")).await.unwrap()); + + fs::remove_dir_all(&root).await.unwrap(); + } + + #[tokio::test] + async fn packing_a_missing_step_dir_fails() { + let dir = std::env::temp_dir().join("zb-bundle-empty"); + let _ = fs::remove_dir_all(&dir).await; + fs::create_dir_all(&dir).await.unwrap(); + + let err = pack(&dir, Step::Bite, None).await.unwrap_err().to_string(); + assert!(err.contains("nothing to pack"), "got: {err}"); + + fs::remove_dir_all(&dir).await.unwrap(); + } +} diff --git a/src/cli.rs b/src/cli.rs index 2f318d3..89c355f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -117,6 +117,22 @@ pub enum Commands { /// works on the same host. #[arg(long, num_args = 0..=1, default_missing_value = "127.0.0.1", verbatim_doc_comment)] publish_bootnodes: Option, + /// Bundle produced by 'pack' to restore into the base path before + /// spawning, so a bite from another machine can be spawned here. + #[arg(long, verbatim_doc_comment)] + bundle: Option, + }, + /// Pack a step's artifacts (specs, snapshots, overrides, manifest) into a single file. + Pack { + /// Base path holding the artifacts. + #[arg(long, short = 'd', verbatim_doc_comment)] + base_path: Option, + /// Step to pack. + #[arg(short = 's', value_parser = clap::builder::PossibleValuesParser::new(["bite", "spawn", "post"]), default_value="bite")] + step: String, + /// Where to write the bundle. Defaults to '/-bundle.tgz'. + #[arg(long, short = 'o', verbatim_doc_comment)] + out: Option, }, /// [Helper] Generate artifacts to be used by the next step (only 'spawn' and 'post' allowed) GenerateArtifacts { diff --git a/src/doppelganger.rs b/src/doppelganger.rs index a8be6fc..f450c1e 100644 --- a/src/doppelganger.rs +++ b/src/doppelganger.rs @@ -697,6 +697,12 @@ pub async fn clean_up_dir_for_step( let mut needed_files: Vec = vec!["config.toml".to_string(), rc_spec.clone()]; + // The overrides that were applied are part of the bundle: without them a + // restored bite cannot show what was changed in the state it carries. + if step == Step::Bite { + needed_files.push("rc_overrides.json".to_string()); + } + // Add parachain files dynamically for para in paras { let para_chain_name = para.as_chain_string(&rc.as_chain_string()); @@ -704,6 +710,9 @@ pub async fn clean_up_dir_for_step( let para_snap = format!("{}-snap.tgz", para_chain_name); needed_files.push(para_spec); needed_files.push(para_snap); + if step == Step::Bite { + needed_files.push(format!("{}_overrides.json", para.id())); + } } if step == Step::Bite { @@ -712,6 +721,20 @@ pub async fn clean_up_dir_for_step( needed_files.push(alice_snap); } + // Overrides are only there when this step generated them; a missing + // spec or snapshot below is still a hard error. + let mut present = vec![]; + for file in needed_files { + if file.ends_with("_overrides.json") + && !fs::try_exists(format!("{debug_path}/{file}")).await? + { + warn!("{file} not found, it will not be part of the bundle"); + continue; + } + present.push(file); + } + let needed_files = present; + for file in &needed_files { let from = format!("{debug_path}/{file}"); let to = format!("{step_path}/{file}"); diff --git a/src/main.rs b/src/main.rs index 659b803..1e09c0b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ use tracing_subscriber::EnvFilter; use zombienet_sdk::{LocalFileSystem, Network, NetworkNode}; mod bootnodes; +mod bundle; mod cli; mod config; mod doppelganger; @@ -270,6 +271,7 @@ async fn main() -> Result<(), anyhow::Error> { step, apply_upgrade, publish_bootnodes, + bundle, } => { let resolved_config = resolve_spawn_config( config, @@ -279,6 +281,11 @@ async fn main() -> Result<(), anyhow::Error> { publish_bootnodes, )?; let step: Step = step.into(); + + if let Some(bundle) = bundle { + bundle::unpack(Path::new(&bundle), resolved_config.base_path.as_path()).await?; + } + let base_path_str = resolved_config.base_path.to_string_lossy(); if !fs::try_exists(format!("{base_path_str}/{}", step.dir_from())) @@ -322,6 +329,15 @@ async fn main() -> Result<(), anyhow::Error> { ) .await?; } + Commands::Pack { + base_path, + step, + out, + } => { + let base_path = get_base_path(base_path); + let step: Step = step.into(); + bundle::pack(&base_path, step, out.map(PathBuf::from)).await?; + } Commands::GenerateArtifacts { relay, base_path,