From 9d29f9eb261b7979e9a9b3d004feb83f3384dd01 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 14:38:14 +0000 Subject: [PATCH 1/6] fix: re-wire the tinycloud CLI dispatch (TC-1442) main.rs and lib.rs lost `pub mod cli` and the `cli::run` dispatch in the PR #145 merge (aa6b161), so `tinycloud node ...`, `serve`, `--help` and `--version` started the server. Restore the dispatch for those tokens only. No arguments and `--validate-config` keep the existing server start path (container entrypoint, deploy preflight). Adds tests/node_cli.rs. --- tinycloud-node-server/src/lib.rs | 1 + tinycloud-node-server/src/main.rs | 61 ++++++++++- tinycloud-node-server/tests/node_cli.rs | 135 ++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 tinycloud-node-server/tests/node_cli.rs diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index d9f12c4e..4d30a3dc 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -27,6 +27,7 @@ use std::{ pub mod allow_list; pub mod auth_guards; pub mod authorization; +pub mod cli; pub mod config; pub mod dstack; pub mod egress; diff --git a/tinycloud-node-server/src/main.rs b/tinycloud-node-server/src/main.rs index c2132cf3..296d9008 100644 --- a/tinycloud-node-server/src/main.rs +++ b/tinycloud-node-server/src/main.rs @@ -49,9 +49,38 @@ fn install_panic_hook() { })); } -#[rocket::main] -async fn main() { +/// Whether argv selects the clap CLI (`serve`, `node ...`, `--help`, +/// `--version`). Everything else keeps the original server start: the +/// container entrypoint is `tinycloud` with no arguments, and the deploy +/// preflight runs `tinycloud --validate-config`, which clap would reject. +fn is_cli_invocation(args: impl IntoIterator) -> bool { + matches!( + args.into_iter() + .nth(1) + .as_deref() + .and_then(|arg| arg.to_str()), + Some("serve" | "node" | "--help" | "-h" | "--version" | "-V") + ) +} + +fn main() { install_panic_hook(); + if is_cli_invocation(std::env::args_os()) { + if let Err(e) = tinycloud::cli::run() { + eprintln!("\n✗ tinycloud-node failed:\n"); + for cause in e.chain() { + eprintln!(" {cause}"); + } + std::process::exit(1); + } + return; + } + // The runtime `#[rocket::main]` used to set up: worker and blocking + // thread counts come from ROCKET_WORKERS / ROCKET_MAX_BLOCKING. + rocket::execute(start_server()); +} + +async fn start_server() { let config = build_config_figment(); // That's just for easy access to ROCKET_LOG_LEVEL let tinycloud_config = config.extract::().unwrap(); @@ -298,4 +327,32 @@ mod tests { assert_eq!(cfg.share_email.readiness_max_age_seconds, 42); } + + #[test] + fn only_cli_tokens_leave_the_server_start_path() { + let argv = |args: &[&str]| { + std::iter::once("tinycloud") + .chain(args.iter().copied()) + .map(std::ffi::OsString::from) + .collect::>() + }; + for args in [ + &["node", "status", "--json"][..], + &["serve"], + &["--help"], + &["-h"], + &["--version"], + &["-V"], + ] { + assert!(is_cli_invocation(argv(args)), "{args:?} must reach the CLI"); + } + // Container entrypoint and the deploy preflight must keep starting + // (or validating) the server through the original path. + for args in [&[][..], &["--validate-config"], &["--unknown"]] { + assert!( + !is_cli_invocation(argv(args)), + "{args:?} must keep the server start path" + ); + } + } } diff --git a/tinycloud-node-server/tests/node_cli.rs b/tinycloud-node-server/tests/node_cli.rs new file mode 100644 index 00000000..2b77beb4 --- /dev/null +++ b/tinycloud-node-server/tests/node_cli.rs @@ -0,0 +1,135 @@ +//! The released `tinycloud` binary must route `node ...`, `serve`, `--help` +//! and `--version` to the clap CLI instead of booting the server (TC-1442). +//! +//! Every invocation runs with a scrubbed environment and an empty HOME, so a +//! binary that falls through to the server start fails fast on the missing key +//! source rather than binding a socket. The timeout turns any accidental +//! server start into a test failure instead of a hang. + +use std::{ + io::Read, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; + +struct Run { + code: Option, + stdout: String, + stderr: String, +} + +fn run_tinycloud(args: &[&str]) -> Run { + let home = tempfile::tempdir().unwrap(); + let config_root = home.path().join("node-config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_tinycloud")) + .args(args) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("HOME", home.path()) + .env("XDG_CONFIG_HOME", home.path().join("xdg-config")) + .env("XDG_DATA_HOME", home.path().join("xdg-data")) + .env("XDG_STATE_HOME", home.path().join("xdg-state")) + .env("TINYCLOUD_NODE_CONFIG_ROOT", &config_root) + .current_dir(home.path()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let deadline = Instant::now() + Duration::from_secs(20); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break Some(status); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + break None; + } + std::thread::sleep(Duration::from_millis(25)); + }; + + let mut stdout = String::new(); + let mut stderr = String::new(); + child + .stdout + .take() + .unwrap() + .read_to_string(&mut stdout) + .unwrap(); + child + .stderr + .take() + .unwrap() + .read_to_string(&mut stderr) + .unwrap(); + let run = Run { + code: status.and_then(|status| status.code()), + stdout, + stderr, + }; + assert!( + status.is_some(), + "tinycloud {args:?} did not exit within 20s (server started?)\nstdout: {}\nstderr: {}", + run.stdout, + run.stderr + ); + assert!( + !run.stderr.contains("Failed to start tinycloud-node"), + "tinycloud {args:?} fell through to the server start\nstderr: {}", + run.stderr + ); + run +} + +#[test] +fn node_help_lists_subcommands_without_starting_the_server() { + let run = run_tinycloud(&["node", "--help"]); + assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); + for subcommand in [ + "service", "status", "logs", "doctor", "key", "link", "tunnel", + ] { + assert!( + run.stdout + .lines() + .any(|line| line.trim_start().starts_with(subcommand)), + "`tinycloud node --help` does not list `{subcommand}`:\n{}", + run.stdout + ); + } +} + +#[test] +fn top_level_help_lists_serve_and_node() { + let run = run_tinycloud(&["--help"]); + assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); + for subcommand in ["serve", "node"] { + assert!( + run.stdout + .lines() + .any(|line| line.trim_start().starts_with(subcommand)), + "`tinycloud --help` does not list `{subcommand}`:\n{}", + run.stdout + ); + } +} + +#[test] +fn version_prints_the_package_version_without_starting_the_server() { + let run = run_tinycloud(&["--version"]); + assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); + assert_eq!( + run.stdout.trim(), + format!("tinycloud {}", env!("CARGO_PKG_VERSION")) + ); +} + +#[test] +fn node_status_json_on_an_empty_home_fails_fast_without_starting_the_server() { + // No service is installed under the empty HOME: the command reports that + // through the CLI error path (exit 1, nothing on stdout) and returns. + let run = run_tinycloud(&["node", "status", "--json"]); + assert_eq!(run.code, Some(1), "stderr: {}", run.stderr); + assert!(run.stdout.is_empty(), "stdout: {}", run.stdout); +} From bb04778f2b24eeee3e4a4c917e45fb948fb394ad Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 15:01:54 +0000 Subject: [PATCH 2/6] fix: route only no-arg and --validate-config to the server (TC-1442) Every other argv goes to clap, so help, -v, -x -h, --config x, servee and --validate-config --help no longer boot a server. Drop the CLI's no-subcommand arm and run_legacy_server, unreachable from main now. Tests drive the built binary: exact usage errors, node status meaning, --validate-config valid and invalid, no-arg server path. --- tinycloud-node-server/src/cli.rs | 12 +- tinycloud-node-server/src/main.rs | 53 ++------- tinycloud-node-server/tests/node_cli.rs | 152 +++++++++++++++++++----- 3 files changed, 140 insertions(+), 77 deletions(-) diff --git a/tinycloud-node-server/src/cli.rs b/tinycloud-node-server/src/cli.rs index a129401e..c3adf1bf 100644 --- a/tinycloud-node-server/src/cli.rs +++ b/tinycloud-node-server/src/cli.rs @@ -19,7 +19,7 @@ use crate::{ #[command(name = "tinycloud", version, disable_help_subcommand = true)] pub struct Cli { #[command(subcommand)] - command: Option, + command: Commands, } #[derive(Debug, Subcommand)] @@ -184,9 +184,8 @@ struct BackupArgs { pub fn run() -> Result<()> { let cli = Cli::parse(); match cli.command { - None => block_on(run_legacy_server()), - Some(Commands::Serve(args)) => block_on(run_serve(args)), - Some(Commands::Node(args)) => run_node(args), + Commands::Serve(args) => block_on(run_serve(args)), + Commands::Node(args) => run_node(args), } } @@ -198,11 +197,6 @@ fn block_on(future: impl Future>) -> Result { .block_on(future) } -async fn run_legacy_server() -> Result<()> { - let config_path = std::env::current_dir()?.join("tinycloud.toml"); - runtime::launch_with_figment(runtime::legacy_config_figment(), config_path).await -} - async fn run_serve(args: ServeArgs) -> Result<()> { let config_path = args .config diff --git a/tinycloud-node-server/src/main.rs b/tinycloud-node-server/src/main.rs index 296d9008..a4c3a17a 100644 --- a/tinycloud-node-server/src/main.rs +++ b/tinycloud-node-server/src/main.rs @@ -49,23 +49,22 @@ fn install_panic_hook() { })); } -/// Whether argv selects the clap CLI (`serve`, `node ...`, `--help`, -/// `--version`). Everything else keeps the original server start: the -/// container entrypoint is `tinycloud` with no arguments, and the deploy -/// preflight runs `tinycloud --validate-config`, which clap would reject. -fn is_cli_invocation(args: impl IntoIterator) -> bool { - matches!( - args.into_iter() - .nth(1) - .as_deref() - .and_then(|arg| arg.to_str()), - Some("serve" | "node" | "--help" | "-h" | "--version" | "-V") - ) +/// The server start path is only for the two invocations that exist outside +/// the CLI: no arguments (the container entrypoint, `cargo run`) and exactly +/// `--validate-config` (the deploy preflight). Every other argv goes to clap, +/// which also rejects anything it does not recognise instead of booting. +fn starts_server(args: &[std::ffi::OsString]) -> bool { + match args { + [] => true, + [only] => only == "--validate-config", + _ => false, + } } fn main() { install_panic_hook(); - if is_cli_invocation(std::env::args_os()) { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if !starts_server(&args) { if let Err(e) = tinycloud::cli::run() { eprintln!("\n✗ tinycloud-node failed:\n"); for cause in e.chain() { @@ -327,32 +326,4 @@ mod tests { assert_eq!(cfg.share_email.readiness_max_age_seconds, 42); } - - #[test] - fn only_cli_tokens_leave_the_server_start_path() { - let argv = |args: &[&str]| { - std::iter::once("tinycloud") - .chain(args.iter().copied()) - .map(std::ffi::OsString::from) - .collect::>() - }; - for args in [ - &["node", "status", "--json"][..], - &["serve"], - &["--help"], - &["-h"], - &["--version"], - &["-V"], - ] { - assert!(is_cli_invocation(argv(args)), "{args:?} must reach the CLI"); - } - // Container entrypoint and the deploy preflight must keep starting - // (or validating) the server through the original path. - for args in [&[][..], &["--validate-config"], &["--unknown"]] { - assert!( - !is_cli_invocation(argv(args)), - "{args:?} must keep the server start path" - ); - } - } } diff --git a/tinycloud-node-server/tests/node_cli.rs b/tinycloud-node-server/tests/node_cli.rs index 2b77beb4..86426719 100644 --- a/tinycloud-node-server/tests/node_cli.rs +++ b/tinycloud-node-server/tests/node_cli.rs @@ -1,10 +1,14 @@ -//! The released `tinycloud` binary must route `node ...`, `serve`, `--help` -//! and `--version` to the clap CLI instead of booting the server (TC-1442). +//! Argument routing of the released `tinycloud` binary (TC-1442). //! -//! Every invocation runs with a scrubbed environment and an empty HOME, so a -//! binary that falls through to the server start fails fast on the missing key -//! source rather than binding a socket. The timeout turns any accidental -//! server start into a test failure instead of a hang. +//! Only two invocations may reach the server start: no arguments (container +//! entrypoint) and exactly `--validate-config` (deploy preflight). Everything +//! else is the clap CLI, which must never boot a server. These tests run the +//! built binary with default features. +//! +//! Every run uses a scrubbed environment and an empty HOME, so a binary that +//! falls through to the server start fails fast on the missing key source +//! rather than binding a socket. The timeout turns an accidental server start +//! into a test failure instead of a hang. use std::{ io::Read, @@ -12,15 +16,17 @@ use std::{ time::{Duration, Instant}, }; +/// Throwaway 32-byte secret for config validation; never used as a real key. +const TEST_SECRET: &str = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY"; + struct Run { code: Option, stdout: String, stderr: String, } -fn run_tinycloud(args: &[&str]) -> Run { +fn run_tinycloud(args: &[&str], env: &[(&str, &str)]) -> Run { let home = tempfile::tempdir().unwrap(); - let config_root = home.path().join("node-config"); let mut child = Command::new(env!("CARGO_BIN_EXE_tinycloud")) .args(args) .env_clear() @@ -29,7 +35,11 @@ fn run_tinycloud(args: &[&str]) -> Run { .env("XDG_CONFIG_HOME", home.path().join("xdg-config")) .env("XDG_DATA_HOME", home.path().join("xdg-data")) .env("XDG_STATE_HOME", home.path().join("xdg-state")) - .env("TINYCLOUD_NODE_CONFIG_ROOT", &config_root) + .env( + "TINYCLOUD_NODE_CONFIG_ROOT", + home.path().join("node-config"), + ) + .envs(env.iter().copied()) .current_dir(home.path()) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -64,28 +74,32 @@ fn run_tinycloud(args: &[&str]) -> Run { .unwrap() .read_to_string(&mut stderr) .unwrap(); - let run = Run { - code: status.and_then(|status| status.code()), - stdout, - stderr, - }; assert!( status.is_some(), - "tinycloud {args:?} did not exit within 20s (server started?)\nstdout: {}\nstderr: {}", - run.stdout, - run.stderr + "tinycloud {args:?} did not exit within 20s (server started?)\nstdout: {stdout}\nstderr: {stderr}" ); + Run { + code: status.and_then(|status| status.code()), + stdout, + stderr, + } +} + +/// The run was answered by the CLI, not by the server start path. +fn assert_cli_handled(args: &[&str], run: &Run) { assert!( - !run.stderr.contains("Failed to start tinycloud-node"), + !run.stderr.contains("Failed to start tinycloud-node") + && !run.stderr.contains("runtime configuration is"), "tinycloud {args:?} fell through to the server start\nstderr: {}", run.stderr ); - run } #[test] fn node_help_lists_subcommands_without_starting_the_server() { - let run = run_tinycloud(&["node", "--help"]); + let args = ["node", "--help"]; + let run = run_tinycloud(&args, &[]); + assert_cli_handled(&args, &run); assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); for subcommand in [ "service", "status", "logs", "doctor", "key", "link", "tunnel", @@ -102,7 +116,9 @@ fn node_help_lists_subcommands_without_starting_the_server() { #[test] fn top_level_help_lists_serve_and_node() { - let run = run_tinycloud(&["--help"]); + let args = ["--help"]; + let run = run_tinycloud(&args, &[]); + assert_cli_handled(&args, &run); assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); for subcommand in ["serve", "node"] { assert!( @@ -116,8 +132,10 @@ fn top_level_help_lists_serve_and_node() { } #[test] -fn version_prints_the_package_version_without_starting_the_server() { - let run = run_tinycloud(&["--version"]); +fn version_prints_the_package_version() { + let args = ["--version"]; + let run = run_tinycloud(&args, &[]); + assert_cli_handled(&args, &run); assert_eq!(run.code, Some(0), "stderr: {}", run.stderr); assert_eq!( run.stdout.trim(), @@ -126,10 +144,90 @@ fn version_prints_the_package_version_without_starting_the_server() { } #[test] -fn node_status_json_on_an_empty_home_fails_fast_without_starting_the_server() { - // No service is installed under the empty HOME: the command reports that - // through the CLI error path (exit 1, nothing on stdout) and returns. - let run = run_tinycloud(&["node", "status", "--json"]); +fn node_status_json_on_an_empty_home_reports_that_no_service_is_installed() { + // The empty HOME has no service manifest. The CLI reports that on stderr + // and exits 1 with nothing on stdout, so a caller parsing stdout as JSON + // sees an empty document, never a half-started node. + let args = ["node", "status", "--json"]; + let run = run_tinycloud(&args, &[]); + assert_cli_handled(&args, &run); assert_eq!(run.code, Some(1), "stderr: {}", run.stderr); assert!(run.stdout.is_empty(), "stdout: {}", run.stdout); + assert!( + run.stderr.contains("service is not installed"), + "stderr: {}", + run.stderr + ); +} + +#[test] +fn unrecognised_arguments_are_a_usage_error_and_never_start_a_server() { + for args in [ + &["help"][..], + &["-v"], + &["-x", "-h"], + &["--config", "x"], + &["servee"], + &["--validate-config", "--help"], + &["--bogus"], + ] { + let run = run_tinycloud(args, &[]); + assert_cli_handled(args, &run); + assert_eq!(run.code, Some(2), "{args:?} stderr: {}", run.stderr); + assert!( + run.stderr.contains("Usage:"), + "{args:?} stderr: {}", + run.stderr + ); + assert!(run.stdout.is_empty(), "{args:?} stdout: {}", run.stdout); + } +} + +#[test] +fn validate_config_still_checks_the_configuration_without_the_cli() { + let args = ["--validate-config"]; + let valid = run_tinycloud( + &args, + &[ + ("TINYCLOUD_KEYS__TYPE", "static"), + ("TINYCLOUD_KEYS__SECRET", TEST_SECRET), + ], + ); + assert_eq!(valid.code, Some(0), "stderr: {}", valid.stderr); + assert!( + valid.stderr.contains("runtime configuration is valid"), + "stderr: {}", + valid.stderr + ); + + // Share email enabled without a trust bundle is rejected before any + // socket or key is opened. + let invalid = run_tinycloud( + &args, + &[ + ("TINYCLOUD_KEYS__TYPE", "static"), + ("TINYCLOUD_KEYS__SECRET", TEST_SECRET), + ("TINYCLOUD_SHARE_EMAIL__ENABLED", "true"), + ], + ); + assert_eq!(invalid.code, Some(1), "stderr: {}", invalid.stderr); + assert!( + invalid.stderr.contains("runtime configuration is invalid"), + "stderr: {}", + invalid.stderr + ); +} + +#[test] +fn no_arguments_keep_the_server_start_path() { + // No key source is configured, so the server start path reports that and + // exits before binding anything. The CLI would answer with a usage error. + let run = run_tinycloud(&[], &[]); + assert_eq!(run.code, Some(1), "stderr: {}", run.stderr); + assert!( + run.stderr.contains("Failed to start tinycloud-node"), + "stderr: {}", + run.stderr + ); + assert!(!run.stderr.contains("Usage:"), "stderr: {}", run.stderr); } From bdfe8f71afb4d287e8fc182e20a0e80369bddf8b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 14:48:57 +0000 Subject: [PATCH 3/6] test: pin startup identity per key source; provider/auto serve fail (TC-1442) --- .../tests/startup_identity.rs | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 tinycloud-node-server/tests/startup_identity.rs diff --git a/tinycloud-node-server/tests/startup_identity.rs b/tinycloud-node-server/tests/startup_identity.rs new file mode 100644 index 00000000..04d5a1e0 --- /dev/null +++ b/tinycloud-node-server/tests/startup_identity.rs @@ -0,0 +1,295 @@ +//! The node identity each key source resolves at startup, observed through +//! the real `tinycloud` binary's `/.well-known/tinycloud/node-keys` (TC-1442). +//! +//! - `node service install` writes `[global.keys] type = "provider"`, and the +//! installed unit runs `serve --config `. That, and the same file +//! without a key section (`type = "Auto"`, the default), must boot with the +//! provider identity, keep it across restarts, and report it to the +//! control plane. +//! - The production key sources must keep resolving the DID they resolve +//! today. The DIDs below are pinned from fixed fixture keys; none of them is +//! a real node key. + +use std::{ + fs::{self, File}, + net::TcpListener, + path::Path, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; +use tinycloud_core::keys::StaticSecret; + +/// The repository's `tinycloud.toml`, which the Docker image copies next to +/// the `/tinycloud` entrypoint. +const SHIPPED_CONFIG: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../tinycloud.toml"); + +/// base64url of the 32 bytes `TC-1442 static-env fixture key!!`. +const STATIC_SECRET: &str = "VEMtMTQ0MiBzdGF0aWMtZW52IGZpeHR1cmUga2V5ISE"; +/// What the node published for `STATIC_SECRET` before TC-1442's provider fix. +const STATIC_DID: &str = "did:key:z6Mkocaf9dnjBPLMQpu5jo4mLRMnqTEugbMCMS1zvb6ahrvH"; + +struct KillOnDrop(Child); + +impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +/// A `tinycloud` command that inherits no node configuration from the test +/// process, so only the environment each test sets applies. +fn tinycloud(cwd: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_tinycloud")); + command + .env_clear() + .env("PATH", "/usr/bin:/bin") + .current_dir(cwd) + .stdin(Stdio::null()); + command +} + +/// Start the node and wait until it serves. Returns the running node and the +/// DID it publishes. +fn start_node(mut command: Command, logs: &Path) -> (KillOnDrop, String) { + let port = free_port(); + let stderr_path = logs.join(format!("stderr-{port}.log")); + let child = command + .env("TINYCLOUD_ADDRESS", "127.0.0.1") + .env("TINYCLOUD_PORT", port.to_string()) + .stdout(File::create(logs.join(format!("stdout-{port}.log"))).unwrap()) + .stderr(File::create(&stderr_path).unwrap()) + .spawn() + .unwrap(); + let mut server = KillOnDrop(child); + + let url = format!("http://127.0.0.1:{port}/.well-known/tinycloud/node-keys"); + let client = reqwest::blocking::Client::new(); + let deadline = Instant::now() + Duration::from_secs(60); + while Instant::now() < deadline { + if let Some(status) = server.0.try_wait().unwrap() { + panic!( + "tinycloud exited before serving ({status}):\n{}", + fs::read_to_string(&stderr_path).unwrap_or_default() + ); + } + if let Ok(response) = client.get(&url).send() { + if response.status().is_success() { + let body: serde_json::Value = response.json().unwrap(); + let did = body["nodeDid"].as_str().unwrap().to_owned(); + return (server, did); + } + } + std::thread::sleep(Duration::from_millis(100)); + } + panic!("tinycloud did not serve within 60s"); +} + +/// The container layout: `tinycloud.toml` in the working directory. +fn shipped_layout() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + fs::copy(SHIPPED_CONFIG, dir.path().join("tinycloud.toml")).unwrap(); + dir +} + +#[test] +fn shipped_config_with_a_static_secret_keeps_its_did() { + let dir = shipped_layout(); + let secret = base64::decode_config(STATIC_SECRET, base64::URL_SAFE_NO_PAD).unwrap(); + assert_eq!(StaticSecret::new(secret).unwrap().node_did(), STATIC_DID); + + // The container entrypoint: `/tinycloud` with no arguments. + let mut legacy = tinycloud(dir.path()); + legacy.env("TINYCLOUD_KEYS__SECRET", STATIC_SECRET); + assert_eq!(start_node(legacy, dir.path()).1, STATIC_DID); + + let mut serve = tinycloud(dir.path()); + serve + .args(["serve", "--config", "tinycloud.toml"]) + .env("TINYCLOUD_KEYS__SECRET", STATIC_SECRET); + assert_eq!(start_node(serve, dir.path()).1, STATIC_DID); +} + +#[cfg(feature = "dstack")] +mod dstack { + use super::*; + use parking_lot::Mutex; + use std::{ + io::{Read, Write}, + os::unix::net::UnixListener, + path::PathBuf, + sync::Arc, + }; + + /// The bytes the stand-in guest agent derives for the node's key path: + /// `TC-1442 dstack GetKey fixture!!!`. + const KEY_HEX: &str = "54432d313434322064737461636b204765744b65792066697874757265212121"; + /// What the node published for `KEY_HEX` before TC-1442's provider fix. + const DSTACK_DID: &str = "did:key:z6MkmRVxeLv23jCfiR5XQMt5yZsyP5rxoS8GD8RkWoo7BnBA"; + /// The derivation path production's KMS key (and so its DID) comes from. + const PRIMARY_KEY_PATH: &str = "tinycloud/keys/primary"; + + /// A dstack guest agent on a Unix socket: answers `GetKey` for the + /// node's key path with `KEY_HEX` and `Info` with fixed values, and + /// records every `GetKey` path it is asked for. + fn guest_agent(socket: PathBuf) -> Arc>> { + let listener = UnixListener::bind(socket).unwrap(); + let paths = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&paths); + std::thread::spawn(move || { + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut request = Vec::new(); + stream.read_to_end(&mut request).unwrap(); + let request = String::from_utf8(request).unwrap(); + let (head, body) = request.split_once("\r\n\r\n").unwrap(); + let (status, reply) = if head.starts_with("POST /GetKey ") { + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + let path = body["path"].as_str().unwrap().to_owned(); + recorded.lock().push(path.clone()); + if path == PRIMARY_KEY_PATH { + ("200 OK", format!("{{\"key\":\"{KEY_HEX}\"}}")) + } else { + ("404 Not Found", "{}".to_owned()) + } + } else if head.starts_with("GET /Info ") { + ( + "200 OK", + r#"{"app_id":"tc1442","compose_hash":"tc1442","instance_id":"tc1442"}"# + .to_owned(), + ) + } else { + ("404 Not Found", "{}".to_owned()) + }; + let _ = stream.write_all( + format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n{reply}", + reply.len() + ) + .as_bytes(), + ); + } + }); + paths + } + + /// Production runs the image's `tinycloud.toml` (`type = "Static"`) with + /// `TINYCLOUD_KEYS_TYPE=Dstack` and the guest agent's socket. + #[test] + fn production_dstack_config_keeps_its_did() { + let dir = shipped_layout(); + let key = hex::decode(KEY_HEX).unwrap(); + assert_eq!(StaticSecret::new(key).unwrap().node_did(), DSTACK_DID); + let socket = dir.path().join("dstack.sock"); + let requested = guest_agent(socket.clone()); + + let mut legacy = tinycloud(dir.path()); + legacy + .env("TINYCLOUD_KEYS_TYPE", "Dstack") + .env("DSTACK_SIMULATOR_ENDPOINT", &socket); + assert_eq!(start_node(legacy, dir.path()).1, DSTACK_DID); + + let requested = requested.lock(); + assert!( + !requested.is_empty(), + "the node never asked dstack for a key" + ); + assert!( + requested.iter().all(|path| path == PRIMARY_KEY_PATH), + "the node asked dstack for {requested:?}" + ); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn installed_provider_config_serves_a_stable_identity() { + installed_node_serves_a_stable_identity(|config| config); +} + +#[cfg(target_os = "linux")] +#[test] +fn installed_config_without_a_key_section_serves_a_stable_identity() { + installed_node_serves_a_stable_identity(|config| { + let auto = config.replace("\n[global.keys]\ntype = \"provider\"\n", ""); + assert_ne!(auto, config); + auto + }); +} + +/// `node service install` on a fresh Linux user account, `edit` applied to the +/// config it writes, then the command its systemd unit runs. Linux only: +/// macOS stores the provider key in the keychain. +#[cfg(target_os = "linux")] +fn installed_node_serves_a_stable_identity(edit: fn(String) -> String) { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let home = dir.path().join("home"); + let bin = dir.path().join("bin"); + fs::create_dir_all(&home).unwrap(); + fs::create_dir_all(&bin).unwrap(); + // `install` enables the unit through `systemctl --user`. + let systemctl = bin.join("systemctl"); + fs::write(&systemctl, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(&systemctl, fs::Permissions::from_mode(0o755)).unwrap(); + + let user_command = || { + let mut command = tinycloud(dir.path()); + command + .env("PATH", format!("{}:/usr/bin:/bin", bin.display())) + .env("HOME", &home) + .env("XDG_CONFIG_HOME", home.join("xdg-config")) + .env("XDG_DATA_HOME", home.join("xdg-data")) + .env("XDG_STATE_HOME", home.join("xdg-state")); + command + }; + + let installed = user_command() + .args(["node", "service", "install"]) + .output() + .unwrap(); + assert!( + installed.status.success(), + "install failed: {}", + String::from_utf8_lossy(&installed.stderr) + ); + let config = home.join("xdg-config/tinycloud-node/tinycloud.toml"); + let written = fs::read_to_string(&config).unwrap(); + assert!(written.contains("\n[global.keys]\ntype = \"provider\"\n")); + fs::write(&config, edit(written)).unwrap(); + + let serve = || { + let mut command = user_command(); + command.arg("serve").arg("--config").arg(&config); + command + }; + + // First boot generates the identity; the control plane must report it. + let (server, did) = start_node(serve(), dir.path()); + assert!(did.starts_with("did:key:"), "{did}"); + let status = user_command() + .args(["node", "status", "--json"]) + .output() + .unwrap(); + assert!( + status.status.success(), + "node status failed: {}", + String::from_utf8_lossy(&status.stderr) + ); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + assert_eq!(status["identityReady"], true, "{status}"); + assert_eq!(status["nodeDid"].as_str(), Some(did.as_str()), "{status}"); + drop(server); + + // A restart loads the stored identity instead of making a new one. + assert_eq!(start_node(serve(), dir.path()).1, did); +} From 5b9ebf226364209cc8ebc3ef05abe331fe3023ff Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 15:01:08 +0000 Subject: [PATCH 4/6] fix: serve resolves provider and Auto node identities again (TC-1442) aa6b161 replaced app's key_provider::resolve_identity_state(.., Serve) with resolve_keys, whose Provider arm bails and whose Auto arm bails when no TEE is present. `node service install` writes type = "provider", so an installed node's `serve` exited 1, and a config without a key section could not start either. resolve_keys now returns None for those cases (Provider, Auto with TINYCLOUD_TEE_MODE=off, Auto with no dstack socket) and app_with_control resolves the node's stored identity through the key provider, generating it on first start, then publishes it to the control plane with set_identity_snapshot so node status reports identityReady. Static and dstack keys resolve exactly as before. node_cli's no-argument test relied on Auto failing; it now makes the server start fail on a share-email config error instead, which is still reported by the server start path before any key or socket is opened. --- tinycloud-node-server/src/lib.rs | 51 ++++++++++++------- tinycloud-node-server/src/routes/node_keys.rs | 3 +- tinycloud-node-server/tests/node_cli.rs | 15 +++--- 3 files changed, 42 insertions(+), 27 deletions(-) diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 4d30a3dc..b7810191 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -107,6 +107,7 @@ use config::{BlockStorage, Config, Keys, StagingStorage}; use hooks::HookRuntime; use invocation_replay::InvocationReplayCache; use node_control::control::ControlPlaneHandle; +use node_control::key_provider::{self, IdentityPurpose}; use policy_v3::{ authorize_delivery as authorize_policy_v3_delivery, challenge as policy_v3_challenge, get_status as get_policy_v3_status, issue_enforcer_binding as policy_v3_enforcer_binding, @@ -554,7 +555,27 @@ pub async fn app_with_control( tc_bench_health, ]); - let key_setup: StaticSecret = resolve_keys(&tinycloud_config.keys).await?; + let key_setup: StaticSecret = match resolve_keys(&tinycloud_config.keys).await? { + Some(key_setup) => key_setup, + None => { + // The node's stored identity, generated on its first start. The + // control plane only probed it at spawn, so publish what serve + // resolved. + let identity = key_provider::resolve_identity_state( + Some(&tinycloud_config.keys), + &tinycloud_config.storage.datadir, + IdentityPurpose::Serve, + )?; + if let Some(control) = control.as_ref() { + control + .set_identity_snapshot(key_provider::identity_snapshot(&identity)) + .await; + } + identity + .secret + .ok_or_else(|| anyhow::anyhow!("node identity is not ready"))? + } + }; let pseudonymizer = std::sync::Arc::new(tinycloud_core::telemetry::Pseudonymizer::new(&key_setup)); let tracing_fairing = tracing::TracingFairing::new( @@ -1207,13 +1228,16 @@ async fn run_retention_sweep( } } -pub(crate) async fn resolve_keys(keys: &Keys) -> Result { +/// The node key from a Static or dstack key source. `None` means the key is +/// the node's stored identity: provider keys, and Auto outside a TEE. +pub(crate) async fn resolve_keys(keys: &Keys) -> Result> { match keys { - Keys::Static(s) => Ok(s.clone().try_into()?), + Keys::Static(s) => Ok(Some(s.clone().try_into()?)), #[cfg(feature = "dstack")] Keys::Dstack => { let key_bytes = dstack::get_key("tinycloud/keys/primary").await?; StaticSecret::new(key_bytes) + .map(Some) .map_err(|v| anyhow::anyhow!("dstack key too short: {} bytes", v.len())) } Keys::Auto => { @@ -1223,36 +1247,25 @@ pub(crate) async fn resolve_keys(keys: &Keys) -> Result { Some("dstack") => { let key_bytes = dstack::get_key("tinycloud/keys/primary").await?; StaticSecret::new(key_bytes) + .map(Some) .map_err(|v| anyhow::anyhow!("dstack key too short: {} bytes", v.len())) } - Some("off") => { - anyhow::bail!( - "TEE mode disabled but no static key configured. \ - Set TINYCLOUD_KEYS_SECRET or configure [keys] in config." - ) - } + Some("off") => Ok(None), _ => { // Auto-detect: check for dstack socket #[cfg(feature = "dstack")] if dstack::is_available() { ::tracing::info!("dstack socket detected, using TEE key derivation"); let key_bytes = dstack::get_key("tinycloud/keys/primary").await?; - return StaticSecret::new(key_bytes).map_err(|v| { + return StaticSecret::new(key_bytes).map(Some).map_err(|v| { anyhow::anyhow!("dstack key too short: {} bytes", v.len()) }); } - anyhow::bail!( - "No key source configured. Either:\n \ - - Set TINYCLOUD_KEYS_SECRET environment variable\n \ - - Configure [keys] section in tinycloud.toml\n \ - - Run inside a dstack TEE (with 'dstack' feature enabled)" - ) + Ok(None) } } } - Keys::Provider => anyhow::bail!( - "provider key mode must be resolved through the node control key-provider path" - ), + Keys::Provider => Ok(None), } } diff --git a/tinycloud-node-server/src/routes/node_keys.rs b/tinycloud-node-server/src/routes/node_keys.rs index e52d4fcf..dfe4254d 100644 --- a/tinycloud-node-server/src/routes/node_keys.rs +++ b/tinycloud-node-server/src/routes/node_keys.rs @@ -191,7 +191,8 @@ mod tests { let resolved = crate::resolve_keys(&crate::config::Keys::Dstack) .await - .expect("dstack key resolution"); + .expect("dstack key resolution") + .expect("dstack yields a key"); responder.await.expect("dstack responder"); let rocket = rocket::build() diff --git a/tinycloud-node-server/tests/node_cli.rs b/tinycloud-node-server/tests/node_cli.rs index 86426719..a54200b7 100644 --- a/tinycloud-node-server/tests/node_cli.rs +++ b/tinycloud-node-server/tests/node_cli.rs @@ -5,10 +5,10 @@ //! else is the clap CLI, which must never boot a server. These tests run the //! built binary with default features. //! -//! Every run uses a scrubbed environment and an empty HOME, so a binary that -//! falls through to the server start fails fast on the missing key source -//! rather than binding a socket. The timeout turns an accidental server start -//! into a test failure instead of a hang. +//! Every run uses a scrubbed environment and an empty HOME. A server start +//! with no key configuration generates a node identity and serves, so the +//! timeout turns an accidental server start into a test failure instead of a +//! hang. use std::{ io::Read, @@ -220,9 +220,10 @@ fn validate_config_still_checks_the_configuration_without_the_cli() { #[test] fn no_arguments_keep_the_server_start_path() { - // No key source is configured, so the server start path reports that and - // exits before binding anything. The CLI would answer with a usage error. - let run = run_tinycloud(&[], &[]); + // Share email enabled without a trust bundle, so the server start path + // refuses the configuration and exits before opening a key or binding + // anything. The CLI would answer with a usage error. + let run = run_tinycloud(&[], &[("TINYCLOUD_SHARE_EMAIL__ENABLED", "true")]); assert_eq!(run.code, Some(1), "stderr: {}", run.stderr); assert!( run.stderr.contains("Failed to start tinycloud-node"), From 570cb45138b58e5f6cc958ce8dda143d164065fa Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 15:31:16 +0000 Subject: [PATCH 5/6] fix: Auto refuses a local identity where a TEE is indicated; log new identities (TC-1442) Auto now creates a local node identity only when nothing indicates a dstack TEE. With TINYCLOUD_TEE_MODE=dstack, tee.mode = "Dstack", TINYCLOUD_KEYS_TYPE=Dstack, DSTACK_SIMULATOR_ENDPOINT, or a dstack socket in a build without dstack, and no dstack key, startup fails with an error naming the indication and writes no identity, so a TEE node can never come up under a new DID. TINYCLOUD_TEE_MODE=off is still checked first and opts out of detection. Provider keys are unaffected. Creating an identity now logs once at WARN, "created a new node identity at ()", replacing the INFO "generated node identity secret on first run". A restart that loads the identity logs nothing. No key material is logged; KeyProvider gains location() for the message. --- tinycloud-node-server/src/lib.rs | 85 ++++++-- .../src/node_control/key_provider.rs | 18 +- tinycloud-node-server/src/routes/node_keys.rs | 11 +- .../tests/startup_identity.rs | 190 ++++++++++++++++-- 4 files changed, 254 insertions(+), 50 deletions(-) diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index b7810191..dd17380f 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -555,27 +555,28 @@ pub async fn app_with_control( tc_bench_health, ]); - let key_setup: StaticSecret = match resolve_keys(&tinycloud_config.keys).await? { - Some(key_setup) => key_setup, - None => { - // The node's stored identity, generated on its first start. The - // control plane only probed it at spawn, so publish what serve - // resolved. - let identity = key_provider::resolve_identity_state( - Some(&tinycloud_config.keys), - &tinycloud_config.storage.datadir, - IdentityPurpose::Serve, - )?; - if let Some(control) = control.as_ref() { - control - .set_identity_snapshot(key_provider::identity_snapshot(&identity)) - .await; + let key_setup: StaticSecret = + match resolve_keys(&tinycloud_config.keys, &tinycloud_config.tee).await? { + Some(key_setup) => key_setup, + None => { + // The node's stored identity, generated on its first start. The + // control plane only probed it at spawn, so publish what serve + // resolved. + let identity = key_provider::resolve_identity_state( + Some(&tinycloud_config.keys), + &tinycloud_config.storage.datadir, + IdentityPurpose::Serve, + )?; + if let Some(control) = control.as_ref() { + control + .set_identity_snapshot(key_provider::identity_snapshot(&identity)) + .await; + } + identity + .secret + .ok_or_else(|| anyhow::anyhow!("node identity is not ready"))? } - identity - .secret - .ok_or_else(|| anyhow::anyhow!("node identity is not ready"))? - } - }; + }; let pseudonymizer = std::sync::Arc::new(tinycloud_core::telemetry::Pseudonymizer::new(&key_setup)); let tracing_fairing = tracing::TracingFairing::new( @@ -1229,8 +1230,14 @@ async fn run_retention_sweep( } /// The node key from a Static or dstack key source. `None` means the key is -/// the node's stored identity: provider keys, and Auto outside a TEE. -pub(crate) async fn resolve_keys(keys: &Keys) -> Result> { +/// the node's stored identity: provider keys, and Auto outside a TEE. Auto +/// refuses when the environment indicates a dstack TEE but no dstack key is +/// available, so a TEE node never silently gets a new local DID. +/// `TINYCLOUD_TEE_MODE=off` is checked first and opts out of TEE detection. +pub(crate) async fn resolve_keys( + keys: &Keys, + tee: &config::TeeConfig, +) -> Result> { match keys { Keys::Static(s) => Ok(Some(s.clone().try_into()?)), #[cfg(feature = "dstack")] @@ -1261,6 +1268,19 @@ pub(crate) async fn resolve_keys(keys: &Keys) -> Result> { anyhow::anyhow!("dstack key too short: {} bytes", v.len()) }); } + if let Some(hint) = dstack_indication(tee) { + anyhow::bail!( + "refusing to create a new node identity: {hint} indicates a dstack TEE, \ + but no dstack key is available (socket {}{}). Start the node with its \ + dstack socket, or remove {hint} if this host is not a TEE.", + dstack::socket_path(), + if cfg!(feature = "dstack") { + "" + } else { + "; this build has no dstack support" + }, + ); + } Ok(None) } } @@ -1269,6 +1289,27 @@ pub(crate) async fn resolve_keys(keys: &Keys) -> Result> { } } +/// Why the environment looks like a dstack deployment, if it does: the TEE +/// mode the node reads, the key type dstack deployments set, an explicit +/// dstack socket, or a dstack socket this build cannot use. +fn dstack_indication(tee: &config::TeeConfig) -> Option<&'static str> { + let env_is_dstack = + |name: &str| std::env::var(name).is_ok_and(|value| value.eq_ignore_ascii_case("dstack")); + if env_is_dstack("TINYCLOUD_TEE_MODE") { + Some("TINYCLOUD_TEE_MODE=dstack") + } else if tee.mode == config::TeeMode::Dstack { + Some("tee.mode = \"Dstack\"") + } else if env_is_dstack("TINYCLOUD_KEYS_TYPE") || env_is_dstack("TINYCLOUD_KEYS__TYPE") { + Some("TINYCLOUD_KEYS_TYPE=Dstack") + } else if std::env::var_os("DSTACK_SIMULATOR_ENDPOINT").is_some() { + Some("DSTACK_SIMULATOR_ENDPOINT") + } else if dstack::is_available() { + Some("the dstack socket") + } else { + None + } +} + /// Ensure local storage directories exist before connecting. /// /// For local resources (SQLite file paths, filesystem dirs), create them diff --git a/tinycloud-node-server/src/node_control/key_provider.rs b/tinycloud-node-server/src/node_control/key_provider.rs index de38de85..07c639b4 100644 --- a/tinycloud-node-server/src/node_control/key_provider.rs +++ b/tinycloud-node-server/src/node_control/key_provider.rs @@ -141,6 +141,8 @@ pub struct BackupResult { pub trait KeyProvider: Send + Sync { fn backend(&self) -> KeyBackend; + /// Where the identity is stored, for operators. Never key material. + fn location(&self) -> String; fn load_secret(&self) -> Result>>; fn store_secret(&self, secret: &[u8]) -> Result<()>; fn delete_secret(&self) -> Result<()>; @@ -417,10 +419,10 @@ fn resolve_provider_state(data_root: &Path, purpose: IdentityPurpose) -> Result< let secret = StaticSecret::new(secret.to_vec()) .expect("generated identity secret is always 32 bytes"); let node_did = secret.node_did(); - tracing::info!( - backend = ?backend, - data_path = %data_root.display(), - "generated node identity secret on first run" + // Once per identity: a restart loads it and logs nothing here. + tracing::warn!( + "created a new node identity at {} ({node_did})", + provider.location() ); Ok(identity_state( IdentitySourceKind::Provider, @@ -702,6 +704,10 @@ impl KeyProvider for EncryptedFileProvider { KeyBackend::EncryptedFile } + fn location(&self) -> String { + self.identity_path.display().to_string() + } + fn load_secret(&self) -> Result>> { if !self.identity_path.exists() { return Ok(None); @@ -840,6 +846,10 @@ impl KeyProvider for MacosKeychainProvider { KeyBackend::MacosKeychain } + fn location(&self) -> String { + format!("macOS keychain item {} / {}", self.service, self.account) + } + fn load_secret(&self) -> Result>> { match self.load_secret_in_tier(MacosKeychainTier::DataProtection)? { MacosLookupOutcome::Found(secret) => return validate_loaded_secret(secret), diff --git a/tinycloud-node-server/src/routes/node_keys.rs b/tinycloud-node-server/src/routes/node_keys.rs index dfe4254d..493efce9 100644 --- a/tinycloud-node-server/src/routes/node_keys.rs +++ b/tinycloud-node-server/src/routes/node_keys.rs @@ -189,10 +189,13 @@ mod tests { socket.display().to_string(), ); - let resolved = crate::resolve_keys(&crate::config::Keys::Dstack) - .await - .expect("dstack key resolution") - .expect("dstack yields a key"); + let resolved = crate::resolve_keys( + &crate::config::Keys::Dstack, + &crate::config::TeeConfig::default(), + ) + .await + .expect("dstack key resolution") + .expect("dstack yields a key"); responder.await.expect("dstack responder"); let rocket = rocket::build() diff --git a/tinycloud-node-server/tests/startup_identity.rs b/tinycloud-node-server/tests/startup_identity.rs index 04d5a1e0..b953704d 100644 --- a/tinycloud-node-server/tests/startup_identity.rs +++ b/tinycloud-node-server/tests/startup_identity.rs @@ -6,6 +6,10 @@ //! without a key section (`type = "Auto"`, the default), must boot with the //! provider identity, keep it across restarts, and report it to the //! control plane. +//! - A node that creates a new identity says so once in its log; a restart +//! that loads it does not. +//! - Auto never creates a local identity where the environment indicates a +//! dstack TEE. //! - The production key sources must keep resolving the DID they resolve //! today. The DIDs below are pinned from fixed fixture keys; none of them is //! a real node key. @@ -13,7 +17,7 @@ use std::{ fs::{self, File}, net::TcpListener, - path::Path, + path::{Path, PathBuf}, process::{Child, Command, Stdio}, time::{Duration, Instant}, }; @@ -28,6 +32,10 @@ const STATIC_SECRET: &str = "VEMtMTQ0MiBzdGF0aWMtZW52IGZpeHR1cmUga2V5ISE"; /// What the node published for `STATIC_SECRET` before TC-1442's provider fix. const STATIC_DID: &str = "did:key:z6Mkocaf9dnjBPLMQpu5jo4mLRMnqTEugbMCMS1zvb6ahrvH"; +/// What a node logs when it creates a new identity. +#[cfg(target_os = "linux")] +const CREATED: &str = "created a new node identity at "; + struct KillOnDrop(Child); impl Drop for KillOnDrop { @@ -57,19 +65,38 @@ fn tinycloud(cwd: &Path) -> Command { command } -/// Start the node and wait until it serves. Returns the running node and the -/// DID it publishes. -fn start_node(mut command: Command, logs: &Path) -> (KillOnDrop, String) { - let port = free_port(); - let stderr_path = logs.join(format!("stderr-{port}.log")); +/// Spawn `command` with stdout and stderr in one new log file under `logs`. +fn spawn_logged(command: &mut Command, logs: &Path) -> (KillOnDrop, PathBuf) { + let log = logs.join(format!("tinycloud-{}.log", uuid::Uuid::new_v4().simple())); + let file = File::create(&log).unwrap(); let child = command - .env("TINYCLOUD_ADDRESS", "127.0.0.1") - .env("TINYCLOUD_PORT", port.to_string()) - .stdout(File::create(logs.join(format!("stdout-{port}.log"))).unwrap()) - .stderr(File::create(&stderr_path).unwrap()) + .stdout(file.try_clone().unwrap()) + .stderr(file) .spawn() .unwrap(); - let mut server = KillOnDrop(child); + (KillOnDrop(child), log) +} + +struct Node { + _server: KillOnDrop, + did: String, + log: PathBuf, +} + +#[cfg(target_os = "linux")] +impl Node { + fn log(&self) -> String { + fs::read_to_string(&self.log).unwrap() + } +} + +/// Start the node and wait until it serves the DID it publishes. +fn start_node(mut command: Command, logs: &Path) -> Node { + let port = free_port(); + command + .env("TINYCLOUD_ADDRESS", "127.0.0.1") + .env("TINYCLOUD_PORT", port.to_string()); + let (mut server, log) = spawn_logged(&mut command, logs); let url = format!("http://127.0.0.1:{port}/.well-known/tinycloud/node-keys"); let client = reqwest::blocking::Client::new(); @@ -78,14 +105,18 @@ fn start_node(mut command: Command, logs: &Path) -> (KillOnDrop, String) { if let Some(status) = server.0.try_wait().unwrap() { panic!( "tinycloud exited before serving ({status}):\n{}", - fs::read_to_string(&stderr_path).unwrap_or_default() + fs::read_to_string(&log).unwrap_or_default() ); } if let Ok(response) = client.get(&url).send() { if response.status().is_success() { let body: serde_json::Value = response.json().unwrap(); let did = body["nodeDid"].as_str().unwrap().to_owned(); - return (server, did); + return Node { + _server: server, + did, + log, + }; } } std::thread::sleep(Duration::from_millis(100)); @@ -109,13 +140,13 @@ fn shipped_config_with_a_static_secret_keeps_its_did() { // The container entrypoint: `/tinycloud` with no arguments. let mut legacy = tinycloud(dir.path()); legacy.env("TINYCLOUD_KEYS__SECRET", STATIC_SECRET); - assert_eq!(start_node(legacy, dir.path()).1, STATIC_DID); + assert_eq!(start_node(legacy, dir.path()).did, STATIC_DID); let mut serve = tinycloud(dir.path()); serve .args(["serve", "--config", "tinycloud.toml"]) .env("TINYCLOUD_KEYS__SECRET", STATIC_SECRET); - assert_eq!(start_node(serve, dir.path()).1, STATIC_DID); + assert_eq!(start_node(serve, dir.path()).did, STATIC_DID); } #[cfg(feature = "dstack")] @@ -195,7 +226,7 @@ mod dstack { legacy .env("TINYCLOUD_KEYS_TYPE", "Dstack") .env("DSTACK_SIMULATOR_ENDPOINT", &socket); - assert_eq!(start_node(legacy, dir.path()).1, DSTACK_DID); + assert_eq!(start_node(legacy, dir.path()).did, DSTACK_DID); let requested = requested.lock(); assert!( @@ -209,6 +240,119 @@ mod dstack { } } +/// Run `command` until it exits. A node still running after 30 s started +/// serving, which fails the test. +#[cfg(target_os = "linux")] +fn run_to_exit(mut command: Command, logs: &Path) -> (Option, String) { + command + .env("TINYCLOUD_ADDRESS", "127.0.0.1") + .env("TINYCLOUD_PORT", free_port().to_string()); + let (mut server, log) = spawn_logged(&mut command, logs); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if let Some(status) = server.0.try_wait().unwrap() { + return (status.code(), fs::read_to_string(&log).unwrap()); + } + assert!( + Instant::now() < deadline, + "tinycloud kept running instead of refusing:\n{}", + fs::read_to_string(&log).unwrap_or_default() + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +/// The log says exactly once that the identity at `location` was created. +#[cfg(target_os = "linux")] +fn assert_created_once(log: &str, location: &str, did: &str) { + let created: Vec<&str> = log.lines().filter(|line| line.contains(CREATED)).collect(); + assert_eq!(created.len(), 1, "{log}"); + assert!( + created[0].contains(&format!("{location} ({did})")), + "{}", + created[0] + ); +} + +/// The container entrypoint with no key configuration and no TEE indicated: +/// the node creates a local identity, says so once, and loads it on restart. +/// Linux only: macOS stores the provider key in the keychain. +#[cfg(target_os = "linux")] +#[test] +fn bare_start_without_a_tee_creates_one_identity_and_reloads_it() { + let dir = tempfile::tempdir().unwrap(); + + let first = start_node(tinycloud(dir.path()), dir.path()); + assert!(dir.path().join("data/keys/identity.key.enc").exists()); + assert_created_once(&first.log(), "data/keys/identity.key.enc", &first.did); + let did = first.did.clone(); + drop(first); + + let restart = start_node(tinycloud(dir.path()), dir.path()); + assert_eq!(restart.did, did); + assert!(!restart.log().contains(CREATED), "{}", restart.log()); +} + +/// Auto inside an environment that indicates a dstack TEE, with no dstack +/// socket: the node exits non-zero and creates no identity. Linux only, so a +/// regression cannot write to a macOS keychain. +#[cfg(target_os = "linux")] +#[test] +fn auto_refuses_to_create_an_identity_where_a_tee_is_indicated() { + type Environment = &'static [(&'static str, &'static str)]; + const MISSING_SOCKET: &str = "/nonexistent/tc-1442/dstack.sock"; + // (environment, the indication the refusal names). `None`: an existing + // path refuses first. Production's own signal, `TINYCLOUD_KEYS_TYPE=Dstack`, + // selects dstack keys (or is rejected by a build without dstack), and a + // dstack build asks the socket named by `TINYCLOUD_TEE_MODE=dstack`. + let cases: [(Environment, Option<&str>); 5] = [ + ( + &[("TINYCLOUD_TEE__MODE", "Dstack")], + Some("tee.mode = \"Dstack\""), + ), + ( + &[ + ("TINYCLOUD_TEE_MODE", "dstack"), + ("TINYCLOUD_TEE__MODE", "Dstack"), + ], + (!cfg!(feature = "dstack")).then_some("TINYCLOUD_TEE_MODE=dstack"), + ), + ( + &[ + ("TINYCLOUD_KEYS_TYPE", "Dstack"), + ("TINYCLOUD_KEYS__TYPE", "auto"), + ], + Some("TINYCLOUD_KEYS_TYPE=Dstack"), + ), + ( + &[("DSTACK_SIMULATOR_ENDPOINT", MISSING_SOCKET)], + Some("DSTACK_SIMULATOR_ENDPOINT"), + ), + (&[("TINYCLOUD_KEYS_TYPE", "Dstack")], None), + ]; + for (env, indication) in cases { + let dir = tempfile::tempdir().unwrap(); + let mut bare = tinycloud(dir.path()); + bare.envs(env.iter().copied()); + let (code, log) = run_to_exit(bare, dir.path()); + + assert!( + matches!(code, Some(code) if code != 0), + "{env:?}: {code:?}\n{log}" + ); + assert!(!dir.path().join("data/keys").exists(), "{env:?}\n{log}"); + assert!(!log.contains(CREATED), "{env:?}\n{log}"); + assert!(log.to_lowercase().contains("dstack"), "{env:?}\n{log}"); + if let Some(indication) = indication { + assert!( + log.contains(&format!( + "refusing to create a new node identity: {indication} indicates a dstack TEE" + )), + "{env:?}\n{log}" + ); + } + } +} #[cfg(target_os = "linux")] #[test] fn installed_provider_config_serves_a_stable_identity() { @@ -273,9 +417,13 @@ fn installed_node_serves_a_stable_identity(edit: fn(String) -> String) { command }; - // First boot generates the identity; the control plane must report it. - let (server, did) = start_node(serve(), dir.path()); + // First boot generates the identity, says so once, and the control plane + // reports it. + let identity = home.join("xdg-data/tinycloud-node/keys/identity.key.enc"); + let first = start_node(serve(), dir.path()); + let did = first.did.clone(); assert!(did.starts_with("did:key:"), "{did}"); + assert_created_once(&first.log(), &identity.display().to_string(), &did); let status = user_command() .args(["node", "status", "--json"]) .output() @@ -288,8 +436,10 @@ fn installed_node_serves_a_stable_identity(edit: fn(String) -> String) { let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); assert_eq!(status["identityReady"], true, "{status}"); assert_eq!(status["nodeDid"].as_str(), Some(did.as_str()), "{status}"); - drop(server); + drop(first); // A restart loads the stored identity instead of making a new one. - assert_eq!(start_node(serve(), dir.path()).1, did); + let restart = start_node(serve(), dir.path()); + assert_eq!(restart.did, did); + assert!(!restart.log().contains(CREATED), "{}", restart.log()); } From 08dc5a13a597621b5c4960a00affd3ea99c919e0 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Sun, 18 Oct 2026 15:51:27 +0000 Subject: [PATCH 6/6] fix: provider keys refuse where a TEE is indicated (TC-1442) keys.type = provider now runs the same dstack_indication check Auto uses and fails startup, writing no identity, when the environment indicates a dstack TEE, including production's TINYCLOUD_KEYS_TYPE=Dstack overridden by TINYCLOUD_KEYS__TYPE=provider. A provider node with no marker starts as before. dstack_indication now matches variable names case-insensitively, as figment's TINYCLOUD_ prefix does, so tinycloud_keys_type=Dstack counts. --- tinycloud-node-server/src/lib.rs | 34 +++++-- .../tests/startup_identity.rs | 90 ++++++++++++++----- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index dd17380f..85f241d5 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -1230,10 +1230,11 @@ async fn run_retention_sweep( } /// The node key from a Static or dstack key source. `None` means the key is -/// the node's stored identity: provider keys, and Auto outside a TEE. Auto -/// refuses when the environment indicates a dstack TEE but no dstack key is -/// available, so a TEE node never silently gets a new local DID. -/// `TINYCLOUD_TEE_MODE=off` is checked first and opts out of TEE detection. +/// the node's stored identity: provider keys, and Auto outside a TEE. Where +/// the environment indicates a dstack TEE, provider keys always refuse and +/// Auto refuses when no dstack key is available, so a TEE node never runs +/// under a local DID. For Auto, `TINYCLOUD_TEE_MODE=off` is checked first and +/// opts out of TEE detection. pub(crate) async fn resolve_keys( keys: &Keys, tee: &config::TeeConfig, @@ -1285,16 +1286,33 @@ pub(crate) async fn resolve_keys( } } } - Keys::Provider => Ok(None), + Keys::Provider => { + if let Some(hint) = dstack_indication(tee) { + anyhow::bail!( + "refusing keys.type = provider: {hint} indicates a dstack TEE, where the node \ + key must come from dstack. Use keys.type = Dstack, or remove {hint} if this \ + host is not a TEE." + ); + } + Ok(None) + } } } /// Why the environment looks like a dstack deployment, if it does: the TEE /// mode the node reads, the key type dstack deployments set, an explicit -/// dstack socket, or a dstack socket this build cannot use. +/// dstack socket, or a dstack socket this build cannot use. Variable names +/// match case-insensitively, as figment's `TINYCLOUD_` prefix does. fn dstack_indication(tee: &config::TeeConfig) -> Option<&'static str> { - let env_is_dstack = - |name: &str| std::env::var(name).is_ok_and(|value| value.eq_ignore_ascii_case("dstack")); + let env_is_dstack = |name: &str| { + std::env::vars_os().any(|(key, value)| { + key.to_str() + .is_some_and(|key| key.eq_ignore_ascii_case(name)) + && value + .to_str() + .is_some_and(|value| value.eq_ignore_ascii_case("dstack")) + }) + }; if env_is_dstack("TINYCLOUD_TEE_MODE") { Some("TINYCLOUD_TEE_MODE=dstack") } else if tee.mode == config::TeeMode::Dstack { diff --git a/tinycloud-node-server/tests/startup_identity.rs b/tinycloud-node-server/tests/startup_identity.rs index b953704d..e4047b7c 100644 --- a/tinycloud-node-server/tests/startup_identity.rs +++ b/tinycloud-node-server/tests/startup_identity.rs @@ -293,6 +293,30 @@ fn bare_start_without_a_tee_creates_one_identity_and_reloads_it() { assert!(!restart.log().contains(CREATED), "{}", restart.log()); } +#[cfg(target_os = "linux")] +const MISSING_SOCKET: &str = "/nonexistent/tc-1442/dstack.sock"; + +/// Bare `tinycloud` under `env`, with no dstack socket, exits non-zero, +/// creates no identity, and (when given) prints `refusal`. +#[cfg(target_os = "linux")] +fn assert_refused(env: &[(&str, &str)], refusal: Option) { + let dir = tempfile::tempdir().unwrap(); + let mut bare = tinycloud(dir.path()); + bare.envs(env.iter().copied()); + let (code, log) = run_to_exit(bare, dir.path()); + + assert!( + matches!(code, Some(code) if code != 0), + "{env:?}: {code:?}\n{log}" + ); + assert!(!dir.path().join("data/keys").exists(), "{env:?}\n{log}"); + assert!(!log.contains(CREATED), "{env:?}\n{log}"); + assert!(log.to_lowercase().contains("dstack"), "{env:?}\n{log}"); + if let Some(refusal) = refusal { + assert!(log.contains(&refusal), "{env:?}\n{log}"); + } +} + /// Auto inside an environment that indicates a dstack TEE, with no dstack /// socket: the node exits non-zero and creates no identity. Linux only, so a /// regression cannot write to a macOS keychain. @@ -300,12 +324,11 @@ fn bare_start_without_a_tee_creates_one_identity_and_reloads_it() { #[test] fn auto_refuses_to_create_an_identity_where_a_tee_is_indicated() { type Environment = &'static [(&'static str, &'static str)]; - const MISSING_SOCKET: &str = "/nonexistent/tc-1442/dstack.sock"; // (environment, the indication the refusal names). `None`: an existing // path refuses first. Production's own signal, `TINYCLOUD_KEYS_TYPE=Dstack`, // selects dstack keys (or is rejected by a build without dstack), and a // dstack build asks the socket named by `TINYCLOUD_TEE_MODE=dstack`. - let cases: [(Environment, Option<&str>); 5] = [ + let cases: [(Environment, Option<&str>); 6] = [ ( &[("TINYCLOUD_TEE__MODE", "Dstack")], Some("tee.mode = \"Dstack\""), @@ -324,6 +347,14 @@ fn auto_refuses_to_create_an_identity_where_a_tee_is_indicated() { ], Some("TINYCLOUD_KEYS_TYPE=Dstack"), ), + // Figment matches the `TINYCLOUD_` prefix case-insensitively. + ( + &[ + ("tinycloud_keys_type", "Dstack"), + ("TINYCLOUD_KEYS__TYPE", "auto"), + ], + Some("TINYCLOUD_KEYS_TYPE=Dstack"), + ), ( &[("DSTACK_SIMULATOR_ENDPOINT", MISSING_SOCKET)], Some("DSTACK_SIMULATOR_ENDPOINT"), @@ -331,28 +362,47 @@ fn auto_refuses_to_create_an_identity_where_a_tee_is_indicated() { (&[("TINYCLOUD_KEYS_TYPE", "Dstack")], None), ]; for (env, indication) in cases { - let dir = tempfile::tempdir().unwrap(); - let mut bare = tinycloud(dir.path()); - bare.envs(env.iter().copied()); - let (code, log) = run_to_exit(bare, dir.path()); + assert_refused( + env, + indication.map(|indication| { + format!( + "refusing to create a new node identity: {indication} indicates a dstack TEE" + ) + }), + ); + } +} - assert!( - matches!(code, Some(code) if code != 0), - "{env:?}: {code:?}\n{log}" +/// Provider keys inside an environment that indicates a dstack TEE: the node +/// exits non-zero and creates no identity, even when production's +/// `TINYCLOUD_KEYS_TYPE=Dstack` is overridden to provider. +#[cfg(target_os = "linux")] +#[test] +fn provider_refuses_where_a_tee_is_indicated() { + for (marker, indication) in [ + ( + ("TINYCLOUD_KEYS_TYPE", "Dstack"), + "TINYCLOUD_KEYS_TYPE=Dstack", + ), + ( + ("TINYCLOUD_TEE_MODE", "Dstack"), + "TINYCLOUD_TEE_MODE=dstack", + ), + (("TINYCLOUD_TEE__MODE", "Dstack"), "tee.mode = \"Dstack\""), + ( + ("DSTACK_SIMULATOR_ENDPOINT", MISSING_SOCKET), + "DSTACK_SIMULATOR_ENDPOINT", + ), + ] { + assert_refused( + &[marker, ("TINYCLOUD_KEYS__TYPE", "provider")], + Some(format!( + "refusing keys.type = provider: {indication} indicates a dstack TEE" + )), ); - assert!(!dir.path().join("data/keys").exists(), "{env:?}\n{log}"); - assert!(!log.contains(CREATED), "{env:?}\n{log}"); - assert!(log.to_lowercase().contains("dstack"), "{env:?}\n{log}"); - if let Some(indication) = indication { - assert!( - log.contains(&format!( - "refusing to create a new node identity: {indication} indicates a dstack TEE" - )), - "{env:?}\n{log}" - ); - } } } + #[cfg(target_os = "linux")] #[test] fn installed_provider_config_serves_a_stable_identity() {