Pack a step's artifacts into one restorable bundle

README.md

@@ -114,6 +114,17 @@ Storage keys are derived from pallet and item names, and every value is decoded
 
 Metadata and the live values are read at the block being bitten (`--rc-bite-at` / a para's `bite_at`), so they match the state being imported. Parachains use a default public endpoint when no `rpc_endpoint` is configured; if it can't be reached, the bite still runs with a warning and those overrides go unverified. Custom parachains are only verified when their config supplies an `rpc_endpoint`.
 
+#### One artifact, restored elsewhere
+
+`pack` puts everything a spawn needs into a single file — chain-specs, db snapshots, `config.toml`, the overrides that were applied, `manifest.json`, `ready.json` and any carried upgrade blob:
+
+```sh
+zombie-bite pack -d /tmp/base_path -s bite            # -> /tmp/base_path/bite-bundle.tgz
+zombie-bite spawn -d /other/path --bundle bite-bundle.tgz
+```
+
+`spawn` re-points the spec and snapshot paths at wherever the bundle was unpacked, so the artifacts do not have to land in the directory they were produced in.
+
 #### Publishing bootnodes
 
 A published chain-spec ships with `bootNodes: []` — that is what keeps a fork from dialing the network it was forked from, but it also means a node this process did not start has no way to find the fork. `--publish-bootnodes` fills the list with the fork's own nodes, in the artifacts generated at teardown (the `bite` bundle is left untouched):

src/bundle.rs

@@ -0,0 +1,168 @@
+//! Pack a step's artifacts into one file and restore it elsewhere.
+//!
+//! A bite is often produced in CI and consumed hours later on another machine,
+//! so everything needed to spawn has to travel together: the chain-specs, the
+//! db snapshots, the config, the overrides that were applied, the manifest, and
+//! any runtime carried as an authorized upgrade. `spawn` re-points the spec and
+//! snapshot paths at wherever the bundle was unpacked, so the artifacts do not
+//! have to land in the same directory they were produced in.
+
+use std::path::{Path, PathBuf};
+
+use anyhow::{anyhow, bail};
+use flate2::{read::GzDecoder, write::GzEncoder, Compression};
+use tar::Archive;
+use tokio::fs;
+use tracing::info;
+
+use crate::{config::Step, manifest::MANIFEST_FILE};
+
+/// Files that live in the base dir rather than the step dir, and are part of the
+/// bundle when present.
+const BASE_FILES: [&str; 3] = [MANIFEST_FILE, "ready.json", "ports.json"];
+
+fn default_bundle_name(step: Step) -> String {
+    format!("{}-bundle.tgz", step.dir())
+}
+
+/// Pack `<base>/<step>` plus the base-level files into a single `.tgz`.
+pub async fn pack(
+    base_path: &Path,
+    step: Step,
+    out: Option<PathBuf>,
+) -> Result<PathBuf, anyhow::Error> {
+    let step_dir = base_path.join(step.dir());
+    if !fs::try_exists(&step_dir).await? {
+        bail!(
+            "nothing to pack: {} does not exist",
+            step_dir.to_string_lossy()
+        );
+    }
+
+    let out = out.unwrap_or_else(|| base_path.join(default_bundle_name(step)));
+    let file = std::fs::File::create(&out)
+        .map_err(|e| anyhow!("can't create {}: {e}", out.to_string_lossy()))?;
+    let mut encoder = GzEncoder::new(file, Compression::fast());
+    {
+        let mut archive = tar::Builder::new(&mut encoder);
+        // Paths inside the archive are relative to the base dir, so unpacking
+        // into any directory reproduces the same layout.
+        archive.append_dir_all(step.dir(), &step_dir)?;
+        for name in BASE_FILES {
+            let path = base_path.join(name);
+            if fs::try_exists(&path).await? {
+                archive.append_path_with_name(&path, name)?;
+            }
+        }
+        // Runtimes carried as an authorized upgrade.
+        let mut entries = fs::read_dir(base_path).await?;
+        while let Some(entry) = entries.next_entry().await? {
+            let name = entry.file_name().to_string_lossy().to_string();
+            if name.ends_with("-upgrade.wasm") {
+                archive.append_path_with_name(entry.path(), &name)?;
+            }
+        }
+        archive.finish()?;
+    }
+    encoder.finish()?;
+
+    info!("📦 bundle written to {}", out.to_string_lossy());
+    Ok(out)
+}
+
+/// Unpack a bundle into `base_path`.
+pub async fn unpack(bundle: &Path, base_path: &Path) -> Result<(), anyhow::Error> {
+    if !fs::try_exists(bundle).await? {
+        bail!("bundle {} does not exist", bundle.to_string_lossy());
+    }
+    fs::create_dir_all(base_path).await?;
+
+    let file = std::fs::File::open(bundle)
+        .map_err(|e| anyhow!("can't open {}: {e}", bundle.to_string_lossy()))?;
+    Archive::new(GzDecoder::new(file)).unpack(base_path)?;
+
+    info!(
+        "📦 bundle {} unpacked into {}",
+        bundle.to_string_lossy(),
+        base_path.to_string_lossy()
+    );
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn pack_then_unpack_reproduces_the_layout() {
+        let root = std::env::temp_dir().join("zb-bundle-test");
+        let (from, to) = (root.join("from"), root.join("to"));
+        let _ = fs::remove_dir_all(&root).await;
+        fs::create_dir_all(from.join("bite")).await.unwrap();
+        fs::create_dir_all(&to).await.unwrap();
+
+        // step dir: spec, snapshot, config and the overrides that were applied
+        for (name, content) in [
+            ("kusama-spec.json", "{}"),
+            ("kusama-snap.tgz", "snap"),
+            ("config.toml", "[relaychain]"),
+            ("rc_overrides.json", r#"{"overrides":{}}"#),
+        ] {
+            fs::write(from.join("bite").join(name), content)
+                .await
+                .unwrap();
+        }
+        // base dir files
+        fs::write(from.join(MANIFEST_FILE), r#"{"created_at":1}"#)
+            .await
+            .unwrap();
+        fs::write(from.join("ready.json"), r#"{"rc_start_block":7}"#)
+            .await
+            .unwrap();
+        fs::write(from.join("kusama-upgrade.wasm"), "wasm")
+            .await
+            .unwrap();
+        // not part of the bundle
+        fs::write(from.join("unrelated.log"), "noise")
+            .await
+            .unwrap();
+
+        let bundle = pack(&from, Step::Bite, None).await.unwrap();
+        unpack(&bundle, &to).await.unwrap();
+
+        for name in [
+            "kusama-spec.json",
+            "kusama-snap.tgz",
+            "config.toml",
+            "rc_overrides.json",
+        ] {
+            assert!(
+                fs::try_exists(to.join("bite").join(name)).await.unwrap(),
+                "missing {name}"
+            );
+        }
+        assert_eq!(
+            fs::read_to_string(to.join("ready.json")).await.unwrap(),
+            r#"{"rc_start_block":7}"#
+        );
+        assert!(fs::try_exists(to.join(MANIFEST_FILE)).await.unwrap());
+        assert!(fs::try_exists(to.join("kusama-upgrade.wasm"))
+            .await
+            .unwrap());
+        assert!(!fs::try_exists(to.join("unrelated.log")).await.unwrap());
+
+        fs::remove_dir_all(&root).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn packing_a_missing_step_dir_fails() {
+        let dir = std::env::temp_dir().join("zb-bundle-empty");
+        let _ = fs::remove_dir_all(&dir).await;
+        fs::create_dir_all(&dir).await.unwrap();
+
+        let err = pack(&dir, Step::Bite, None).await.unwrap_err().to_string();
+        assert!(err.contains("nothing to pack"), "got: {err}");
+
+        fs::remove_dir_all(&dir).await.unwrap();
+    }
+}

src/cli.rs

@@ -117,6 +117,22 @@ pub enum Commands {
         /// works on the same host.
         #[arg(long, num_args = 0..=1, default_missing_value = "127.0.0.1", verbatim_doc_comment)]
         publish_bootnodes: Option<String>,
+        /// Bundle produced by 'pack' to restore into the base path before
+        /// spawning, so a bite from another machine can be spawned here.
+        #[arg(long, verbatim_doc_comment)]
+        bundle: Option<String>,
+    },
+    /// Pack a step's artifacts (specs, snapshots, overrides, manifest) into a single file.
+    Pack {
+        /// Base path holding the artifacts.
+        #[arg(long, short = 'd', verbatim_doc_comment)]
+        base_path: Option<String>,
+        /// Step to pack.
+        #[arg(short = 's', value_parser = clap::builder::PossibleValuesParser::new(["bite", "spawn", "post"]), default_value="bite")]
+        step: String,
+        /// Where to write the bundle. Defaults to '<base_path>/<step>-bundle.tgz'.
+        #[arg(long, short = 'o', verbatim_doc_comment)]
+        out: Option<String>,
     },
     /// [Helper] Generate artifacts to be used by the next step (only 'spawn' and 'post' allowed)
     GenerateArtifacts {

src/doppelganger.rs

@@ -697,13 +697,22 @@ pub async fn clean_up_dir_for_step(
 
     let mut needed_files: Vec<String> = vec!["config.toml".to_string(), rc_spec.clone()];
 
+    // The overrides that were applied are part of the bundle: without them a
+    // restored bite cannot show what was changed in the state it carries.
+    if step == Step::Bite {
+        needed_files.push("rc_overrides.json".to_string());
+    }
+
     // Add parachain files dynamically
     for para in paras {
         let para_chain_name = para.as_chain_string(&rc.as_chain_string());
         let para_spec = format!("{}-spec.json", para_chain_name);
         let para_snap = format!("{}-snap.tgz", para_chain_name);
         needed_files.push(para_spec);
         needed_files.push(para_snap);
+        if step == Step::Bite {
+            needed_files.push(format!("{}_overrides.json", para.id()));
+        }
     }
 
     if step == Step::Bite {
@@ -712,6 +721,20 @@ pub async fn clean_up_dir_for_step(
         needed_files.push(alice_snap);
     }
 
+    // Overrides are only there when this step generated them; a missing
+    // spec or snapshot below is still a hard error.
+    let mut present = vec![];
+    for file in needed_files {
+        if file.ends_with("_overrides.json")
+            && !fs::try_exists(format!("{debug_path}/{file}")).await?
+        {
+            warn!("{file} not found, it will not be part of the bundle");
+            continue;
+        }
+        present.push(file);
+    }
+    let needed_files = present;
+
     for file in &needed_files {
         let from = format!("{debug_path}/{file}");
         let to = format!("{step_path}/{file}");

src/main.rs

@@ -11,6 +11,7 @@ use tracing_subscriber::EnvFilter;
 use zombienet_sdk::{LocalFileSystem, Network, NetworkNode};
 
 mod bootnodes;
+mod bundle;
 mod cli;
 mod config;
 mod doppelganger;
@@ -270,6 +271,7 @@ async fn main() -> Result<(), anyhow::Error> {
             step,
             apply_upgrade,
             publish_bootnodes,
+            bundle,
         } => {
             let resolved_config = resolve_spawn_config(
                 config,
@@ -279,6 +281,11 @@ async fn main() -> Result<(), anyhow::Error> {
                 publish_bootnodes,
             )?;
             let step: Step = step.into();
+
+            if let Some(bundle) = bundle {
+                bundle::unpack(Path::new(&bundle), resolved_config.base_path.as_path()).await?;
+            }
+
             let base_path_str = resolved_config.base_path.to_string_lossy();
 
             if !fs::try_exists(format!("{base_path_str}/{}", step.dir_from()))
@@ -322,6 +329,15 @@ async fn main() -> Result<(), anyhow::Error> {
             )
             .await?;
         }
+        Commands::Pack {
+            base_path,
+            step,
+            out,
+        } => {
+            let base_path = get_base_path(base_path);
+            let step: Step = step.into();
+            bundle::pack(&base_path, step, out.map(PathBuf::from)).await?;
+        }
         Commands::GenerateArtifacts {
             relay,
             base_path,