From df2de7cfc1b2e631bcb75e22a246bfe76f640275 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 19 Aug 2026 16:28:23 -0400 Subject: [PATCH 01/42] refactor(config): normalize compute driver field names Signed-off-by: Jesse Jaggars --- crates/openshell-driver-docker/src/lib.rs | 33 +++---- crates/openshell-driver-docker/src/tests.rs | 34 ++++++- crates/openshell-driver-podman/README.md | 6 +- crates/openshell-driver-podman/src/config.rs | 38 +++++++- .../openshell-driver-podman/src/container.rs | 6 +- crates/openshell-driver-podman/src/main.rs | 2 +- crates/openshell-driver-vm/src/driver.rs | 93 ++++++++++++++----- crates/openshell-driver-vm/src/main.rs | 34 ++++++- crates/openshell-gateway/src/vm.rs | 2 +- deploy/docker/gateway.toml | 4 +- docs/reference/gateway-config.mdx | 13 ++- docs/reference/sandbox-compute-drivers.mdx | 4 +- e2e/configs/gateway/docker.toml | 2 +- e2e/configs/gateway/podman.toml | 1 + e2e/with-docker-gateway.sh | 4 +- rfc/0003-gateway-configuration/README.md | 2 +- rfc/0011-multi-player-design/README.md | 15 +-- tasks/scripts/gateway-docker.sh | 2 +- 18 files changed, 220 insertions(+), 75 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index f19560ef97..73a80822f3 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -121,8 +121,9 @@ pub struct DockerComputeConfig { /// Image pull policy for sandbox images. pub image_pull_policy: String, - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, + /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. + #[serde(alias = "sandbox_namespace")] + pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, @@ -170,7 +171,7 @@ impl Default for DockerComputeConfig { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, supervisor_image: None, @@ -197,7 +198,7 @@ pub(crate) struct DockerGuestTlsPaths { struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, - sandbox_namespace: String, + sandbox_label: String, grpc_endpoint: String, network_name: String, gateway_route: DockerGatewayRoute, @@ -541,7 +542,7 @@ impl DockerComputeDriver { config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), + sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, gateway_route, @@ -847,7 +848,7 @@ impl DockerComputeDriver { ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, )); @@ -1238,7 +1239,7 @@ impl DockerComputeDriver { PendingSandboxRecord { sandbox: pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, ), @@ -1293,7 +1294,7 @@ impl DockerComputeDriver { cleanup_sandbox_token_file(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, error_condition(failure.reason, &failure.message), false, ); @@ -1499,7 +1500,7 @@ impl DockerComputeDriver { } async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + let filters = managed_container_label_filters(&self.config.sandbox_label, []); self.docker .list_containers(Some( ListContainersOptionsBuilder::default() @@ -1524,7 +1525,7 @@ impl DockerComputeDriver { } let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + managed_container_label_filters(&self.config.sandbox_label, label_filter_values); let containers = self .docker .list_containers(Some( @@ -1542,7 +1543,7 @@ impl DockerComputeDriver { }; let namespace_matches = labels .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); + .is_some_and(|value| value == &self.config.sandbox_label); let id_matches = sandbox_id.is_empty() || labels .get(LABEL_SANDBOX_ID) @@ -2630,7 +2631,7 @@ fn sandbox_token_host_path_by_id( ) -> Result { openshell_core::driver_utils::sandbox_token_path( "docker-sandbox-tokens", - Some(&config.sandbox_namespace), + Some(&config.sandbox_label), sandbox_id, ) .map_err(|err| { @@ -2987,13 +2988,13 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_WORKSPACE.to_string(), sandbox.workspace.clone(), ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use + // The list/get/find paths filter by `config.sandbox_label`, so use // the same value here. `DriverSandbox.namespace` is unset on the request // path (the gateway elides it), and using it would produce containers // that the driver itself cannot find afterwards. labels.insert( LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), + config.sandbox_label.clone(), ); Ok(ContainerCreateBody { @@ -3595,12 +3596,12 @@ fn label_filters(values: impl IntoIterator) -> HashMap, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_label}"), ]; values.extend(extra_values); label_filters(values) diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 98e7cd1c37..7a623409b7 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -96,7 +96,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { @@ -127,6 +127,34 @@ fn runtime_config() -> DockerDriverRuntimeConfig { } } +#[test] +fn docker_config_uses_canonical_sandbox_label_name() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_label": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["sandbox_label"], "tenant-a"); + assert!(serialized.get("sandbox_namespace").is_none()); +} + +#[test] +fn docker_config_accepts_legacy_sandbox_namespace_alias() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_namespace": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); +} + +#[test] +fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_label": "tenant-a", + "sandbox_namespace": "tenant-b" + })) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); +} + fn json_struct(value: serde_json::Value) -> prost_types::Struct { let serde_json::Value::Object(object) = value else { panic!("expected JSON object"); @@ -2669,10 +2697,10 @@ fn build_container_create_body_uses_runtime_namespace_label() { // runtime config, not from `DriverSandbox.namespace`. The gateway // does not populate `DriverSandbox.namespace`, so a container created // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_namespace`), leaking + // queries (which filter on `config.sandbox_label`), leaking // sandboxes that the driver itself cannot observe. let mut config = runtime_config(); - config.sandbox_namespace = "tenant-a".to_string(); + config.sandbox_label = "tenant-a".to_string(); let mut sandbox = test_sandbox(); sandbox.namespace = "ignored-by-driver".to_string(); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index e53ddc9f3f..dfccb47fae 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -275,9 +275,9 @@ Podman follows the same end-to-end contract as the Kubernetes and VM drivers for the in-container SSH relay: gateway config to `PodmanComputeConfig` to sandbox environment to supervisor session registration on that path. -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. +1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into + `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process + driver. The field defaults to `/run/openshell/ssh.sock` when omitted. 2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that value, alongside required vars such as `OPENSHELL_ENDPOINT` and `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 42571f00f1..04ae3fe5e3 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -90,7 +90,8 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - pub sandbox_ssh_socket_path: String, + #[serde(alias = "sandbox_ssh_socket_path")] + pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. pub network_name: String, @@ -536,7 +537,7 @@ impl Default for PodmanComputeConfig { image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - sandbox_ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, @@ -569,7 +570,7 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("image_pull_policy", &self.image_pull_policy.as_str()) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) - .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) + .field("ssh_socket_path", &self.ssh_socket_path) .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) @@ -605,6 +606,37 @@ impl std::fmt::Debug for PodmanComputeConfig { mod tests { use super::*; + #[test] + fn config_uses_canonical_ssh_socket_path_name() { + let config: PodmanComputeConfig = + serde_json::from_value(serde_json::json!({ "ssh_socket_path": "/run/test.sock" })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["ssh_socket_path"], "/run/test.sock"); + assert!(serialized.get("sandbox_ssh_socket_path").is_none()); + } + + #[test] + fn config_accepts_legacy_sandbox_ssh_socket_path_alias() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_ssh_socket_path": "/run/test.sock" + })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + } + + #[test] + fn config_rejects_canonical_and_legacy_ssh_socket_path_names_together() { + let error = serde_json::from_value::(serde_json::json!({ + "ssh_socket_path": "/run/canonical.sock", + "sandbox_ssh_socket_path": "/run/legacy.sock" + })) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); + } + #[test] fn default_config_sets_health_check_interval() { let cfg = PodmanComputeConfig::default(); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..1743015e26 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -526,7 +526,7 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.sandbox_ssh_socket_path.clone(), + config.ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) @@ -1181,7 +1181,7 @@ pub fn build_container_spec_for_image( "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.sandbox_ssh_socket_path, + config.ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], @@ -2407,7 +2407,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 4c42ba9699..8f223ed802 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -193,7 +193,7 @@ async fn main() -> Result<()> { host_gateway_ip: args .host_gateway_ip .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, supervisor_image: args diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index af55998a2a..0f442f78f2 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -237,7 +237,8 @@ enum GuestImagePayloadSource { #[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct VmDriverConfig { - pub openshell_endpoint: String, + #[serde(alias = "openshell_endpoint")] + pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, pub default_image: String, @@ -333,7 +334,7 @@ pub struct VmDriverConfig { impl std::fmt::Debug for VmDriverConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("VmDriverConfig") - .field("openshell_endpoint", &self.openshell_endpoint) + .field("grpc_endpoint", &self.grpc_endpoint) .field("state_dir", &self.state_dir) .field("launcher_bin", &self.launcher_bin) .field("default_image", &self.default_image) @@ -367,7 +368,7 @@ pub const DEFAULT_SANDBOX_UID: u32 = 10001; impl Default for VmDriverConfig { fn default() -> Self { Self { - openshell_endpoint: String::new(), + grpc_endpoint: String::new(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -453,7 +454,7 @@ impl VmDriverConfig { } fn requires_tls_materials(&self) -> bool { - self.openshell_endpoint.starts_with("https://") + self.grpc_endpoint.starts_with("https://") } fn tls_paths(&self) -> Result, String> { @@ -593,10 +594,10 @@ impl VmDriver { .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; config.validate_proxy_config()?; - if config.openshell_endpoint.trim().is_empty() { + if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } - validate_openshell_endpoint(&config.openshell_endpoint)?; + validate_openshell_endpoint(&config.grpc_endpoint)?; let _ = config.tls_paths()?; #[cfg(target_os = "linux")] @@ -1066,7 +1067,7 @@ impl VmDriver { let endpoint_override = if plan.backend == VmBackend::Qemu { plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) + guest_visible_openshell_endpoint_for_tap(&self.config.grpc_endpoint, host_ip) }) } else { None @@ -1866,7 +1867,7 @@ impl VmDriver { "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + plan.gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); } // The corporate-proxy host-loopback recipe is a libkrun/gvproxy @@ -2019,7 +2020,7 @@ impl VmDriver { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + let gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) @@ -4721,7 +4722,7 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { /// Rewrites loopback host references in a gateway URL to a hostname the guest /// can reach via gvproxy. /// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which +/// The driver receives the gateway endpoint from `--grpc-endpoint`, which /// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is /// useless inside the guest because the guest's loopback interface is its own, /// not the host's. Inside the guest we need a name that gvproxy will translate @@ -4832,7 +4833,7 @@ fn build_guest_environment( endpoint_override: Option<&str>, ) -> Vec { let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), + || guest_visible_openshell_endpoint(&config.grpc_endpoint), String::from, ); // 1. User-supplied environment (lowest priority). @@ -6196,6 +6197,52 @@ mod tests { assert_eq!(attempts.load(Ordering::Relaxed), 3); } + #[test] + fn vm_config_uses_canonical_grpc_endpoint_name() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let serialized = serde_json::to_value(&config).unwrap(); + assert_eq!(serialized["grpc_endpoint"], "http://127.0.0.1:8080"); + assert!(serialized.get("openshell_endpoint").is_none()); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_accepts_legacy_openshell_endpoint_alias() { + let config = VmDriverConfig::default(); + let mut serialized = serde_json::to_value(config).unwrap(); + let fields = serialized.as_object_mut().unwrap(); + fields.remove("grpc_endpoint"); + fields.insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:8080"), + ); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_rejects_canonical_and_legacy_endpoint_names_together() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let mut serialized = serde_json::to_value(config).unwrap(); + serialized.as_object_mut().unwrap().insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:9090"), + ); + + let error = serde_json::from_value::(serialized) + .expect_err("canonical and legacy names must not both be accepted"); + assert!(error.to_string().contains("duplicate field")); + } + struct TestTracing { exporter: opentelemetry_sdk::trace::InMemorySpanExporter, _provider: opentelemetry_sdk::trace::SdkTracerProvider, @@ -7640,7 +7687,7 @@ mod tests { #[test] fn build_guest_environment_sets_supervisor_defaults() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7675,7 +7722,7 @@ mod tests { #[test] fn persisted_legacy_sandbox_without_command_uses_scratch_main() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; // Requests persisted before the canonical-main contract have a @@ -7709,7 +7756,7 @@ mod tests { #[test] fn build_guest_environment_preserves_main_command_spaces() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8080".to_string(), + grpc_endpoint: "https://127.0.0.1:8080".to_string(), ..Default::default() }; let command = vec![ @@ -7746,7 +7793,7 @@ mod tests { #[test] fn build_guest_environment_uses_token_file_without_raw_token_env() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7778,7 +7825,7 @@ mod tests { #[test] fn build_guest_environment_strips_gateway_tls_server_name() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7815,7 +7862,7 @@ mod tests { )], || { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7854,7 +7901,7 @@ mod tests { #[test] fn build_guest_environment_clears_unsupported_network_capabilities() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -7884,7 +7931,7 @@ mod tests { #[test] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8086,7 +8133,7 @@ mod tests { #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), guest_tls_cert: Some(PathBuf::from("/host/tls.crt")), guest_tls_key: Some(PathBuf::from("/host/tls.key")), @@ -8108,7 +8155,7 @@ mod tests { #[test] fn vm_driver_config_requires_tls_materials_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), ..Default::default() }; let err = config @@ -8643,7 +8690,7 @@ mod tests { let (events, _) = broadcast::channel(WATCH_BUFFER); VmDriver { config: VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), vcpus: 2, mem_mib: 2048, gpu_vcpus: 8, @@ -9006,7 +9053,7 @@ mod tests { ca_bundle: Option<&str>, ) -> VmDriverConfig { VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), https_proxy: https_proxy.map(ToString::to_string), proxy_auth_file: auth_file.map(ToString::to_string), proxy_auth_allow_insecure: auth_file.map(|_| true), diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index b8788e4fc9..c875768bfb 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -91,8 +91,12 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, + #[arg( + long = "grpc-endpoint", + alias = "openshell-endpoint", + env = "OPENSHELL_GRPC_ENDPOINT" + )] + grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] default_image: String, @@ -235,8 +239,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint + grpc_endpoint: args + .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, state_dir: args.state_dir.clone(), launcher_bin: None, @@ -764,6 +768,28 @@ mod tests { assert!(err.contains("--bind-socket is required")); } + #[test] + fn accepts_canonical_grpc_endpoint_flag() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--grpc-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + + #[test] + fn accepts_legacy_openshell_endpoint_flag_alias() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index f52bd9ddfa..e7ff24808b 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -536,7 +536,7 @@ pub async fn spawn( command.arg("--log-level").arg(gateway_log_level); append_otlp_args(&mut command, otlp_config, gateway_name); command - .arg("--openshell-endpoint") + .arg("--grpc-endpoint") .arg(&vm_config.grpc_endpoint); command.arg("--state-dir").arg(&vm_config.state_dir); if !vm_config.default_image.trim().is_empty() { diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index 4fe84d633a..b2b649e0ff 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -41,8 +41,8 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. image_pull_policy = "IfNotPresent" -# Prefix applied to sandbox container names. -sandbox_namespace = "openshell" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. # The Docker driver replaces the host with host.openshell.internal and the # port with the gateway's own bind port (8080). Only the scheme survives. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 1c846c29a8..ea12afb469 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -607,7 +607,8 @@ socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "docker-dev" # Empty auto-detects https://host.openshell.internal: when guest TLS is set. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. @@ -629,6 +630,10 @@ enable_bind_mounts = false sandbox_pids_limit = 2048 ``` +Use `sandbox_label` for new Docker configurations. The legacy +`sandbox_namespace` key remains accepted as a compatibility alias. Do not set +both keys in the same driver table. + ### Podman Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. @@ -657,7 +662,7 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -sandbox_ssh_socket_path = "/run/openshell/ssh.sock" +ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" @@ -763,6 +768,10 @@ health_check_interval_secs = 10 # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" ``` +Use `ssh_socket_path` for new Podman configurations. The legacy +`sandbox_ssh_socket_path` key remains accepted as a compatibility alias. Do not +set both keys in the same driver table. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 987e66b0d9..b49870bedf 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -161,7 +161,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -239,7 +239,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 59baed1d7d..c498693063 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -23,5 +23,5 @@ ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "IfNotPresent" -sandbox_namespace = "openshell-e2e" +sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..35b4005243 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -25,4 +25,5 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "missing" network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" +ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 0a767576dd..90d6dc2ba8 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -514,7 +514,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -532,7 +532,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index d6d4750f25..be406296df 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -130,7 +130,7 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override diff --git a/rfc/0011-multi-player-design/README.md b/rfc/0011-multi-player-design/README.md index 32a2f584e1..5fcb67fe51 100644 --- a/rfc/0011-multi-player-design/README.md +++ b/rfc/0011-multi-player-design/README.md @@ -755,13 +755,14 @@ use the workspace to select the target Kubernetes namespace instead of encoding it in the resource name. The label-based lookup and annotation patterns established here carry over unchanged. -**Docker and Podman drivers.** The Docker driver's `sandbox_namespace` label -provides a foundation for workspace mapping, but the driver currently uses a -single configured namespace rather than per-sandbox values. The driver contract -must be updated so that workspace flows through `DriverSandbox` and the driver -applies it as the container label filter. The same applies to Podman and other -local drivers — workspace isolation is enforced at the gateway level and does -not require Kubernetes. +**Docker and Podman drivers.** The Docker driver's `sandbox_label` +configuration value is stored in the `openshell.sandbox_namespace` container +label and provides a foundation for workspace mapping, but the driver currently +uses a single configured value rather than per-sandbox values. The driver +contract must be updated so that workspace flows through `DriverSandbox` and +the driver applies it as the container label filter. The same applies to Podman +and other local drivers — workspace isolation is enforced at the gateway level +and does not require Kubernetes. ### Compute Driver Trust Model diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 6826829fd8..280cbbfc0c 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -231,7 +231,7 @@ ttl_secs = 3600 [openshell.drivers.docker] default_image = "${SANDBOX_IMAGE}" image_pull_policy = "${SANDBOX_IMAGE_PULL_POLICY}" -sandbox_namespace = "${SANDBOX_NAMESPACE}" +sandbox_label = "${SANDBOX_NAMESPACE}" grpc_endpoint = "${GRPC_ENDPOINT}" supervisor_bin = "${SUPERVISOR_BIN}" EOF From c4e6f76b4b46870cbbefee4406ee537e25bd37f7 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 20 Aug 2026 13:25:00 -0400 Subject: [PATCH 02/42] refactor(config): introduce canonical gateway fields Signed-off-by: Jesse Jaggars --- architecture/compute-runtimes.md | 2 +- architecture/gateway.md | 25 +- crates/openshell-core/src/config.rs | 43 +++- crates/openshell-driver-vm/README.md | 4 +- crates/openshell-gateway/src/lib.rs | 2 +- crates/openshell-server/src/cli.rs | 16 ++ crates/openshell-server/src/config_file.rs | 228 +++++++++++++++++- crates/openshell-server/src/lib.rs | 6 +- deploy/docker/gateway.toml | 2 +- .../openshell/templates/gateway-config.yaml | 8 +- .../openshell/tests/gateway_config_test.yaml | 12 + .../tests/sandbox_namespace_test.yaml | 14 +- deploy/rpm/CONFIGURATION.md | 8 +- deploy/rpm/TROUBLESHOOTING.md | 2 +- deploy/rpm/gateway.toml.default | 2 +- docs/about/installation.mdx | 2 +- docs/reference/gateway-config.mdx | 26 +- docs/reference/sandbox-compute-drivers.mdx | 27 ++- docs/security/best-practices.mdx | 2 +- e2e/configs/gateway/docker.toml | 2 +- e2e/configs/gateway/podman.toml | 2 +- e2e/run.sh | 9 +- e2e/rust/e2e-vm.sh | 2 +- e2e/with-podman-gateway.sh | 4 +- examples/aws-s3-sts.md | 2 +- .../podman/README.md | 2 +- .../spiffe-token-exchange-demo/podman/demo.sh | 2 +- .../podman/start-gateway.sh | 2 +- rfc/0003-gateway-configuration/README.md | 23 +- skills/debug-openshell-cluster/SKILL.md | 2 +- tasks/scripts/gateway-docker.sh | 2 +- tasks/scripts/gateway-podman.sh | 4 +- tasks/scripts/gateway-vm.sh | 2 +- tasks/scripts/gateway.sh | 2 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 2 +- 35 files changed, 398 insertions(+), 97 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..fadd479bc9 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -251,7 +251,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. diff --git a/architecture/gateway.md b/architecture/gateway.md index ba325ccc2c..11c1ebcf27 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -690,9 +690,11 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. See -`docs/reference/gateway-config.mdx` for worked per-driver examples and RFC -0003 for the full schema. +Driver implementation settings live in the TOML driver tables. The canonical +selector is the singular `[openshell.gateway] compute_driver`; the legacy +`compute_drivers` list remains accepted and normalizes into the existing +exactly-one-driver runtime validation. See `docs/reference/gateway-config.mdx` +for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. @@ -708,13 +710,16 @@ aliases, network names, and the sandbox JWT issuer. ### Driver inheritance -`[openshell.gateway]` carries a small set of values (`sandbox_namespace`, -`default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, -`host_gateway_ip`, `enable_user_namespaces`) that are inherited into each -driver's `[openshell.drivers.]` table when the driver-specific table -does not override them. The allowlist is per-driver so a gateway-wide -default cannot land in a driver that does not understand it (e.g. +`[openshell.gateway]` carries shared defaults such as `default_image`, +`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, and +`host_gateway_ip`. It also continues to accept the historical +`sandbox_namespace`, `service_account_name`, and `enable_user_namespaces` +locations as compatibility inputs. Canonical Kubernetes configuration places +those values in `[openshell.drivers.kubernetes]` as `namespace`, +`service_account_name`, and `enable_user_namespaces`; canonical Docker +configuration uses `sandbox_label`. Driver-table values take precedence over +compatibility inputs. The allowlist is per-driver so a gateway-wide default +cannot land in a driver that does not understand it (for example, `client_tls_secret_name` is K8s-only). `image_pull_policy` is intentionally **not** inheritable: Kubernetes uses diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3507011120..2da3225252 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -523,11 +523,31 @@ pub struct GatewayJwtConfig { #[serde(default = "default_gateway_id")] pub gateway_id: String, /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. - #[serde(default = "default_sandbox_token_ttl_secs")] + /// intended only for local single-player deployments. Canonical serialized + /// configuration omits the field for that non-expiring behavior; explicit + /// legacy zero remains accepted. + #[serde( + default = "default_sandbox_token_ttl_secs", + skip_serializing_if = "is_default" + )] pub ttl_secs: u64, } +impl GatewayJwtConfig { + /// Effective token lifetime. `None` preserves the established non-expiring + /// behavior represented by an omitted or explicit zero `ttl_secs` value. + pub fn sandbox_token_ttl(&self) -> Option { + (self.ttl_secs != 0).then(|| Duration::from_secs(self.ttl_secs)) + } +} + +fn is_default(value: &T) -> bool +where + T: Default + PartialEq, +{ + value == &T::default() +} + fn default_gateway_id() -> String { "openshell".to_string() } @@ -923,6 +943,25 @@ mod tests { .expect("gateway JWT config should deserialize with default ttl"); assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.sandbox_token_ttl(), None); + + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert!(serialized.get("ttl_secs").is_none()); + } + + #[test] + fn gateway_jwt_positive_ttl_serializes_and_has_effective_duration() { + let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 3600 + })) + .expect("gateway JWT config should deserialize with positive ttl"); + + assert_eq!(cfg.sandbox_token_ttl(), Some(Duration::from_secs(3600))); + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert_eq!(serialized["ttl_secs"], 3600); } #[test] diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 5c61ae1823..1750b0390e 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -116,7 +116,7 @@ cat > .cache/gateway-vm/gateway.toml <, // ── Drivers ────────────────────────────────────────────────────────── - #[serde(default)] + /// Canonical TOML uses the singular `compute_driver = "..."`. The legacy + /// `compute_drivers = ["..."]` form remains accepted and is normalized to + /// this existing vector representation so Rust callers and runtime + /// validation retain their current behavior. + #[serde( + default, + rename = "compute_driver", + alias = "compute_drivers", + deserialize_with = "deserialize_compute_drivers", + serialize_with = "serialize_compute_drivers", + skip_serializing_if = "Option::is_none" + )] pub compute_drivers: Option>, #[serde(default)] pub credential_drivers: Option>, @@ -114,6 +126,9 @@ pub struct GatewayFileSection { pub credential_storage: Option, // ── Sandbox / SSH ──────────────────────────────────────────────────── + /// Compatibility input for Kubernetes `namespace` and Docker + /// `sandbox_label`. Canonical configurations set those driver-owned + /// fields in their respective `[openshell.drivers.]` tables. #[serde(default)] pub sandbox_namespace: Option, #[serde(default)] @@ -142,10 +157,12 @@ pub struct GatewayFileSection { pub supervisor_image: Option, #[serde(default)] pub client_tls_secret_name: Option, + /// Compatibility input for Kubernetes `service_account_name`. #[serde(default)] pub service_account_name: Option, #[serde(default)] pub host_gateway_ip: Option, + /// Compatibility input for Kubernetes `enable_user_namespaces`. #[serde(default)] pub enable_user_namespaces: Option, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet @@ -193,6 +210,62 @@ pub struct GatewayFileSection { pub database_url: Option, } +fn deserialize_compute_drivers<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + struct ComputeDriversVisitor; + + impl<'de> Visitor<'de> for ComputeDriversVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a compute driver name or an array of compute driver names") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + Ok(Some(vec![value.to_string()])) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(Some(vec![value])) + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut drivers = Vec::new(); + while let Some(driver) = sequence.next_element::()? { + drivers.push(driver); + } + Ok(Some(drivers)) + } + } + + deserializer.deserialize_any(ComputeDriversVisitor) +} + +fn serialize_compute_drivers( + drivers: &Option>, + serializer: S, +) -> Result +where + S: Serializer, +{ + match drivers { + Some(drivers) if drivers.len() == 1 => serializer.serialize_str(&drivers[0]), + Some(drivers) => drivers.serialize(serializer), + None => serializer.serialize_none(), + } +} + /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -447,7 +520,7 @@ pub(crate) fn driver_table_with_inherited_keys( }; for key in inheritable_keys { - if merged.contains_key(*key) { + if driver_field_is_present(&merged, key) { continue; } if let Some(value) = gateway_inherited_value(gateway, key) { @@ -458,9 +531,16 @@ pub(crate) fn driver_table_with_inherited_keys( toml::Value::Table(merged) } +fn driver_field_is_present(table: &toml::Table, key: &str) -> bool { + table.contains_key(key) + || (key == "sandbox_label" && table.contains_key("sandbox_namespace")) +} + fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { match key { - "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), + "namespace" | "sandbox_namespace" | "sandbox_label" => { + g.sandbox_namespace.as_deref().map(string_value) + } "default_image" => g.default_image.as_deref().map(string_value), "supervisor_image" => g.supervisor_image.as_deref().map(string_value), "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), @@ -506,6 +586,86 @@ mod tests { assert!(file.openshell.drivers.is_empty()); } + #[test] + fn canonical_compute_driver_scalar_normalizes_to_existing_vector() { + let file: ConfigFile = toml::from_str( + r#" +[openshell.gateway] +compute_driver = "docker" +"#, + ) + .expect("canonical compute driver parses"); + + assert_eq!( + file.openshell.gateway.compute_drivers, + Some(vec!["docker".to_string()]) + ); + } + + #[test] + fn legacy_compute_drivers_list_remains_accepted() { + for (input, expected) in [ + ("compute_drivers = []", Vec::::new()), + ("compute_drivers = [\"docker\"]", vec!["docker".to_string()]), + ( + "compute_drivers = [\"docker\", \"podman\"]", + vec!["docker".to_string(), "podman".to_string()], + ), + ] { + let file: ConfigFile = toml::from_str(&format!("[openshell.gateway]\n{input}\n")) + .expect("legacy compute drivers parse"); + assert_eq!(file.openshell.gateway.compute_drivers, Some(expected)); + } + } + + #[test] + fn compute_driver_rejects_non_string_values_with_a_clear_error() { + let error = toml::from_str::( + r" +[openshell.gateway] +compute_driver = 42 +", + ) + .expect_err("compute driver must be a string or string array"); + + assert!( + error + .to_string() + .contains("a compute driver name or an array of compute driver names") + ); + } + + #[test] + fn canonical_and_legacy_compute_driver_names_are_rejected_together() { + let error = toml::from_str::( + r#" +[openshell.gateway] +compute_driver = "docker" +compute_drivers = ["docker"] +"#, + ) + .expect_err("canonical and legacy names must not both be accepted"); + + assert!(error.to_string().contains("duplicate field")); + } + + #[test] + fn compute_driver_serialization_uses_canonical_scalar_name() { + let file = ConfigFile { + openshell: OpenShellRoot { + gateway: GatewayFileSection { + compute_drivers: Some(vec!["docker".to_string()]), + ..Default::default() + }, + ..Default::default() + }, + }; + + let serialized = toml::to_string(&file).expect("config serializes"); + assert!(serialized.contains("compute_driver = \"docker\"")); + assert!(!serialized.contains("compute_drivers")); + } + #[test] fn parses_full_example() { let toml = r#" @@ -516,7 +676,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] sandbox_namespace = "agents" grpc_rate_limit_requests = 120 @@ -987,11 +1147,11 @@ version = 2 "alpha", &gateway, None, - &["sandbox_namespace", "default_image", "host_gateway_ip"], + &["sandbox_label", "default_image", "host_gateway_ip"], ); let table = merged.as_table().expect("table"); assert_eq!( - table.get("sandbox_namespace").and_then(|v| v.as_str()), + table.get("sandbox_label").and_then(|v| v.as_str()), Some("agents") ); assert_eq!( @@ -1004,6 +1164,54 @@ version = 2 ); } + #[test] + fn canonical_driver_label_overrides_legacy_gateway_default() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("gateway-default".to_string()), + ..Default::default() + }; + let raw = toml::toml! { + sandbox_label = "driver-specific" + }; + let merged = driver_table_with_inherited_keys( + "docker", + &gateway, + Some(&toml::Value::Table(raw)), + &["sandbox_label"], + ); + let table = merged.as_table().expect("table"); + assert_eq!( + table.get("sandbox_label").and_then(toml::Value::as_str), + Some("driver-specific") + ); + assert!(!table.contains_key("sandbox_namespace")); + } + + #[test] + fn legacy_driver_label_suppresses_canonical_gateway_inheritance() { + let gateway = GatewayFileSection { + sandbox_namespace: Some("gateway-default".to_string()), + ..Default::default() + }; + let raw = toml::toml! { + sandbox_namespace = "driver-specific" + }; + let merged = driver_table_with_inherited_keys( + "docker", + &gateway, + Some(&toml::Value::Table(raw)), + &["sandbox_label"], + ); + let table = merged.as_table().expect("table"); + assert_eq!( + table + .get("sandbox_namespace") + .and_then(toml::Value::as_str), + Some("driver-specific") + ); + assert!(!table.contains_key("sandbox_label")); + } + #[test] fn registered_driver_table_can_select_network_defaults() { let gateway = GatewayFileSection { @@ -1108,7 +1316,7 @@ version = 2 /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) /// - accidental addition of a wildcard bind-address override - /// - accidental changes to the compute driver list + /// - accidental changes to the configured compute driver #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -1127,11 +1335,11 @@ version = 2 let drivers = gw .compute_drivers .as_ref() - .expect("compute_drivers must be explicitly set in the RPM default config"); + .expect("compute_driver must be explicitly set in the RPM default config"); assert_eq!( drivers, &["podman".to_string()], - "RPM default must pin compute_drivers to [podman] to prevent unexpected \ + "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8c8afdf08..5058a1d8a7 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -496,7 +496,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - Duration::from_secs(jwt.ttl_secs), + jwt.sandbox_token_ttl().unwrap_or_default(), ) .map_err(Error::config)?, ); @@ -1380,11 +1380,11 @@ async fn build_compute_runtime( if config .gateway_jwt .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) + .is_some_and(|jwt| jwt.sandbox_token_ttl().is_none()) && !driver.is_local_singleplayer(registry) { warn!( - "Gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" + "Gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted or zero); set gateway_jwt.ttl_secs > 0 for shared deployments" ); } diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index b2b649e0ff..da8ef72873 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -30,7 +30,7 @@ version = 1 bind_address = "127.0.0.1:8080" health_bind_address = "127.0.0.1:8081" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.drivers.docker] diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 083748aee3..515417a740 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -47,7 +47,6 @@ data: {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} - sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} @@ -60,9 +59,6 @@ data: {{- if .Values.server.hostGatewayIP }} host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} {{- end }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true {{- else }} @@ -148,10 +144,14 @@ data: {{- end }} [openshell.drivers.kubernetes] + namespace = {{ include "openshell.sandboxNamespace" . | quote }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.enableUserNamespaces }} + enable_user_namespaces = true + {{- end }} {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1380cb18c5..156d8db19d 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -145,6 +145,18 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + - it: renders user namespace enablement under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + server.enableUserNamespaces: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?enable_user_namespaces\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' + - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 864e3a8512..337c991961 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -13,21 +13,27 @@ release: namespace: my-namespace tests: - - it: defaults sandbox_namespace to release namespace in the TOML config + - it: defaults the Kubernetes driver namespace to release namespace template: templates/gateway-config.yaml asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"my-namespace"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - - it: uses explicit sandboxNamespace when set + - it: uses explicit sandboxNamespace for the Kubernetes driver template: templates/gateway-config.yaml set: server.sandboxNamespace: other-ns asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"other-ns"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"other-ns"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - it: defaults NetworkPolicy namespace to release namespace template: templates/networkpolicy.yaml diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 4fc18e6215..aaa97d08d0 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -20,7 +20,7 @@ The defaults are tuned for rootless Podman use: version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" ``` The RPM does not override `bind_address`. The primary listener uses the @@ -28,7 +28,7 @@ built-in `127.0.0.1:17670` default. The Podman driver reports the callback interface it needs, and the gateway adds a separate listener scoped to that interface. This keeps the general API off unrelated host interfaces. -`compute_drivers = ["podman"]` pins the compute driver to Podman. Without +`compute_driver = "podman"` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver selection if Docker is also installed on the host. @@ -215,7 +215,7 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. The legacy `compute_drivers` list remains accepted. | | `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | | `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | @@ -235,7 +235,7 @@ settings: version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" [openshell.drivers.podman] diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 103ce3bf9d..f67b69149b 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -255,7 +255,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_drivers = ["podman"]` under `[openshell.gateway]` | +| `OPENSHELL_DRIVERS=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index cd7e0d99c3..ba76f873b2 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -25,4 +25,4 @@ version = 1 # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. -compute_drivers = ["podman"] +compute_driver = "podman" diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index a733a1b881..9026f939a6 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -30,7 +30,7 @@ Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers -OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_drivers` in the gateway TOML when you need to pin a specific driver. +OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_driver` in the gateway TOML when you need to pin a specific driver. | Compute Driver | How It Is Configured | System Requirements | |---|---|---| diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ea12afb469..f8bd7361a4 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -59,6 +59,8 @@ version = 1 # ... credential-driver-specific settings ... ``` +The canonical gateway selector is `compute_driver = ""`. The legacy `compute_drivers = [""]` list remains accepted for compatibility. An omitted selector or an empty legacy list retains auto-detection; a legacy list with multiple entries retains the existing startup error because only one compute driver can be active. + ## Full Example A complete gateway configuration covering every section. Trim to the fields you need. @@ -78,15 +80,14 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -109,9 +110,7 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:lat # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false sa_token_ttl_secs = 3600 guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" @@ -200,6 +199,11 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] +[openshell.drivers.kubernetes] +namespace = "openshell" +service_account_name = "openshell-sandbox" +enable_user_namespaces = false + [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false @@ -446,6 +450,8 @@ args = [ Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Canonical Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical gateway-level locations remain accepted as compatibility inputs and retain the same lower precedence. Gateway-level `sandbox_namespace` also remains a compatibility default for Docker `sandbox_label`. + ### Kubernetes The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS material for sandboxes is delivered through a Kubernetes Secret rather than host-side file paths. @@ -459,7 +465,7 @@ bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -600,7 +606,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" @@ -645,7 +651,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -784,7 +790,7 @@ version = 1 bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_drivers = ["vm"] +compute_driver = "vm" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -866,7 +872,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index b49870bedf..ef7c93a017 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -42,11 +42,11 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -54,13 +54,15 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_DRIVERS=vm` in the launch environment. + +The legacy `compute_drivers = [""]` list remains accepted for compatibility. Empty legacy lists retain auto-detection, and lists with more than one entry retain the existing startup error because a gateway supports exactly one active compute driver. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. @@ -70,7 +72,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -161,7 +163,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -239,7 +241,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -333,11 +335,11 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. @@ -387,15 +389,16 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical `[openshell.gateway]` locations remain accepted as lower-precedence compatibility inputs. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 1c541f8f8e..53a68364e5 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in the gateway config to enable cluster-wide. | +| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index c498693063..878aee677c 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -7,7 +7,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 35b4005243..2064a081f5 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -7,7 +7,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/e2e/run.sh b/e2e/run.sh index 875186394c..9764fde979 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -142,7 +142,14 @@ if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then fi gateway_driver="$(python3 -c ' import sys, tomllib -print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) +gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] +driver = gateway.get("compute_driver") +if driver is None: + drivers = gateway.get("compute_drivers", []) + driver = drivers[0] if drivers else None +if not driver: + raise SystemExit("gateway config must explicitly select a compute driver") +print(driver) ' "${gateway_config}")" if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 96da2a879f..3081adf988 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -260,7 +260,7 @@ version = 1 [openshell.gateway] bind_address = "127.0.0.1:${HOST_PORT}" -compute_drivers = ["vm"] +compute_driver = "vm" [openshell.gateway.tls] cert_path = "${PKI_DIR}/server/tls.crt" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..089b9923fd 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -452,7 +452,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same TOML # config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine, +# bind_address unset and sets compute_driver = "podman". On Podman Machine, # the driver reserves IPv4 loopback for its callback-only listener, so the # primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. # @@ -519,7 +519,7 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_drivers comes from the RPM template. Override the loopback address + # compute_driver comes from the RPM template. Override the loopback address # and port so Podman Machine can keep its IPv4 callback listener distinct. --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index ed0c204d6b..f6f0b10204 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -90,7 +90,7 @@ if your gateway cache directory differs): version = 1 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" disable_tls = true supervisor_image = "localhost/openshell/supervisor:dev" diff --git a/examples/spiffe-token-exchange-demo/podman/README.md b/examples/spiffe-token-exchange-demo/podman/README.md index b30c4830dd..4405e9c00a 100644 --- a/examples/spiffe-token-exchange-demo/podman/README.md +++ b/examples/spiffe-token-exchange-demo/podman/README.md @@ -62,7 +62,7 @@ to the same Podman network. `START_GATEWAY=1` automates that same-network gateway setup. It mounts the host Podman socket into the gateway container, writes a temporary gateway config with -`compute_drivers = ["podman"]`, and mounts the SPIRE Workload API socket at the +`compute_driver = "podman"`, and mounts the SPIRE Workload API socket at the same absolute host path so the gateway can pass that path to sibling sandbox containers. diff --git a/examples/spiffe-token-exchange-demo/podman/demo.sh b/examples/spiffe-token-exchange-demo/podman/demo.sh index 6b5804a523..3b8bbfaa16 100755 --- a/examples/spiffe-token-exchange-demo/podman/demo.sh +++ b/examples/spiffe-token-exchange-demo/podman/demo.sh @@ -398,7 +398,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/examples/spiffe-token-exchange-demo/podman/start-gateway.sh b/examples/spiffe-token-exchange-demo/podman/start-gateway.sh index fea89a117a..c6f275009f 100755 --- a/examples/spiffe-token-exchange-demo/podman/start-gateway.sh +++ b/examples/spiffe-token-exchange-demo/podman/start-gateway.sh @@ -138,7 +138,7 @@ version = 1 bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index be406296df..41aac552c2 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -72,10 +72,9 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute drivers — list of driver names whose [openshell.drivers.] -# tables should be activated. When empty, the gateway auto-detects a driver -# (kubernetes → podman → docker). VM is never auto-detected. -compute_drivers = ["kubernetes"] +# Compute driver — exactly one driver may be active. When omitted, the gateway +# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. +compute_driver = "kubernetes" # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -113,7 +112,7 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only tables for drivers listed in compute_drivers are activated. +# Only the selected or auto-detected driver's table is activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] @@ -172,7 +171,7 @@ Each `[openshell.drivers.]` table is extracted from the parsed file and ha Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. ### Merge semantics @@ -209,11 +208,11 @@ The following cross-field validations are applied after merging file + env + CLI - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. - When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list remains accepted: an empty list auto-detects, a singleton selects that driver, and multiple entries retain the existing startup error. ### Backwards compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). +The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). Legacy `compute_drivers = [""]` TOML remains accepted, while canonical configurations use the singular `compute_driver = ""`. ### Example: minimal Kubernetes deployment @@ -222,8 +221,8 @@ The existing CLI interface is fully preserved. All flags continue to work exactl version = 1 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_drivers = ["kubernetes"] +bind_address = "0.0.0.0:8080" +compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). # No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). @@ -250,7 +249,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_drivers: ["kubernetes"] + compute_driver: "kubernetes" drivers: kubernetes: namespace: agents @@ -298,5 +297,5 @@ No part of this RFC has shipped yet. The work breaks down as: 1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. 2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. + Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. 3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8e86bc0643..aea67ba528 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -84,7 +84,7 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n 'compute_driver|compute_drivers|socket_path' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 280cbbfc0c..2695566476 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -215,7 +215,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index ab166865ef..2b9d9bc349 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -219,8 +219,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["podman"] -default_image = "${SANDBOX_IMAGE}" +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] @@ -234,6 +233,7 @@ gateway_id = "${GATEWAY_NAME}" ttl_secs = 3600 [openshell.drivers.podman] +default_image = "${SANDBOX_IMAGE}" supervisor_image = "${SUPERVISOR_IMAGE}" image_pull_policy = "$(podman_pull_policy "${SANDBOX_IMAGE_PULL_POLICY}")" EOF diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 3818dca364..226bfb910c 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -340,7 +340,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["vm"] +compute_driver = "vm" disable_tls = ${DISABLE_TLS} [openshell.gateway.auth] diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 019d1b1b63..cffad5ae2b 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -248,7 +248,7 @@ version = 1 [openshell.gateway] name = "${GATEWAY_NAME}" -compute_drivers = ["${DRIVER}"] +compute_driver = "${DRIVER}" default_image = "${SANDBOX_IMAGE}" disable_tls = true diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 7d0b05334d..ab564810de 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -57,7 +57,7 @@ start_gateway() { version = 1 [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" disable_tls = true [openshell.drivers.vm] From 2250ff6efd697e899b2eb815d987d1ff0167165d Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 16:54:17 -0400 Subject: [PATCH 03/42] refactor(config): enforce gateway schema version 2 Signed-off-by: Jesse Jaggars --- .agents/skills/test-release-canary/SKILL.md | 2 +- .github/workflows/release-canary.yml | 8 +- architecture/compute-runtimes.md | 31 +- architecture/gateway.md | 52 +- crates/openshell-core/src/config.rs | 306 ++++++++-- crates/openshell-core/src/container_paths.rs | 11 +- crates/openshell-core/src/driver_utils.rs | 151 +++++ crates/openshell-core/src/lib.rs | 9 +- crates/openshell-driver-docker/README.md | 21 +- crates/openshell-driver-docker/src/lib.rs | 257 ++++++-- crates/openshell-driver-docker/src/tests.rs | 132 +++- crates/openshell-driver-kubernetes/README.md | 13 +- .../openshell-driver-kubernetes/src/config.rs | 197 +++--- .../openshell-driver-kubernetes/src/driver.rs | 84 +-- .../openshell-driver-kubernetes/src/main.rs | 22 +- .../examples/run-mxc-e2e.ps1 | 2 +- crates/openshell-driver-podman/README.md | 13 +- crates/openshell-driver-podman/src/client.rs | 3 + crates/openshell-driver-podman/src/config.rs | 262 +++----- .../openshell-driver-podman/src/container.rs | 45 +- crates/openshell-driver-podman/src/driver.rs | 51 +- crates/openshell-driver-podman/src/main.rs | 52 +- crates/openshell-driver-vm/README.md | 35 +- .../scripts/openshell-vm-sandbox-init.sh | 42 +- crates/openshell-driver-vm/src/driver.rs | 541 ++++++++--------- crates/openshell-driver-vm/src/main.rs | 66 +- crates/openshell-driver-vm/src/rootfs.rs | 37 +- crates/openshell-gateway/src/lib.rs | 33 - crates/openshell-gateway/src/vm.rs | 197 +++--- crates/openshell-server/src/cli.rs | 144 ++--- .../src/compute/driver_config.rs | 111 +++- crates/openshell-server/src/config_file.rs | 572 +++++------------- crates/openshell-server/src/defaults.rs | 4 +- crates/openshell-server/src/lib.rs | 44 +- deploy/docker/gateway.toml | 7 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/ci/values-skaffold.yaml | 4 +- .../openshell/templates/gateway-config.yaml | 22 +- .../openshell/tests/gateway_config_test.yaml | 38 +- deploy/helm/openshell/values.yaml | 13 +- deploy/man/openshell-gateway.8.md | 7 +- deploy/rpm/CONFIGURATION.md | 22 +- deploy/rpm/TROUBLESHOOTING.md | 4 +- deploy/rpm/gateway.toml.default | 7 +- docs/about/container-gateway.mdx | 6 +- docs/reference/gateway-config.mdx | 171 ++++-- docs/reference/sandbox-compute-drivers.mdx | 14 +- e2e/configs/gateway/docker.toml | 6 +- e2e/configs/gateway/podman.toml | 7 +- .../Dockerfile.external-kubernetes-gateway | 6 +- e2e/run.sh | 3 - e2e/rust/e2e-vm.sh | 11 +- e2e/rust/tests/podman_corporate_proxy.rs | 40 +- e2e/support/gateway-common.sh | 2 +- e2e/with-docker-gateway.sh | 20 +- e2e/with-podman-gateway.sh | 41 +- examples/aws-s3-sts.md | 11 +- examples/governance-interceptor/smoke.sh | 5 +- .../podman/README.md | 2 +- .../spiffe-token-exchange-demo/podman/demo.sh | 5 +- .../podman/start-gateway.sh | 5 +- .../smoke.sh | 5 +- python/openshell/release_formula_test.py | 6 +- rfc/0003-gateway-configuration/README.md | 36 +- skills/debug-openshell-cluster/SKILL.md | 10 +- tasks/scripts/gateway-docker.sh | 42 +- tasks/scripts/gateway-podman.sh | 22 +- tasks/scripts/gateway-vm.sh | 42 +- tasks/scripts/gateway.sh | 26 +- tasks/scripts/release.py | 4 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 4 +- 71 files changed, 2414 insertions(+), 1816 deletions(-) diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 2d0fdfe366..7788469c3c 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -131,7 +131,7 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i | Symptom | Likely cause | Where to look | |---|---|---| | `macos`/`ubuntu`/`fedora` job fails on `install.sh` | Latest tagged release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | -| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_DRIVERS` env in the "Ensure …" step. | +| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_COMPUTE_DRIVER` env in the "Ensure …" step. | | `ubuntu-snap` fails after interface connection | The gateway did not recover after Docker became available, or did not become reachable within the 30-second bound. | Failure diagnostics dump Snap service/connection/change state, gateway and snapd journals, Snap logs, and port 17670 listeners. | | `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. | | `kubernetes` job fails on `kubectl wait` | Gateway pod stuck `CrashLoopBackOff` or `ImagePullBackOff`. | Diagnostics dump; check `:dev` image existence at `ghcr.io/nvidia/openshell/gateway`. | diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index ee4074142e..98470584de 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Ensure VM driver run: | - launchctl setenv OPENSHELL_DRIVERS vm + launchctl setenv OPENSHELL_COMPUTE_DRIVER vm launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED" - name: Install and check status @@ -53,7 +53,7 @@ jobs: fi sudo systemctl start docker || sudo service docker start mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" docker info @@ -145,7 +145,7 @@ jobs: bash -s <<'EOF' set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" podman info curl -LsSf "${INSTALL_SH_URL}" | sh @@ -301,7 +301,7 @@ jobs: run: | set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\n' > "${HOME}/.config/openshell/gateway.env" + printf 'OPENSHELL_COMPUTE_DRIVER=docker\n' > "${HOME}/.config/openshell/gateway.env" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh - name: Register kind gateway and check status diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index fadd479bc9..8c7094530e 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -127,11 +127,11 @@ defines the available implementation set, while the runtime consumes a generic registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry -to `run_cli_with_compute_drivers`; factories receive merged driver config and -return either an in-process driver or a gateway-managed remote endpoint. The -server constructs the common runtime adapter and snapshots `GetCapabilities` -for either result. A configured UDS endpoint still takes precedence over a -compiled registration with the same name. +to `run_cli_with_compute_drivers`; factories receive only the selected +`[openshell.drivers.]` table and return either an in-process driver or a +gateway-managed remote endpoint. The server constructs the common runtime +adapter and snapshots `GetCapabilities` for either result. A configured UDS +endpoint still takes precedence over a compiled registration with the same name. The `openshell-gateway` composition crate groups first-party registrations behind the `in-tree-compute-drivers` feature. `openshell-server` has no compute @@ -251,7 +251,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--compute-driver ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -288,10 +288,21 @@ pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise they implement and validate the same complete contract. The capability marker is driver-owned supervisor input and is removed from workload environments. -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Kubernetes, Docker, and Podman share one AppArmor configuration model: +`RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates +that model to its native API and rejects a requested confined profile when its +backend reports AppArmor unavailable. Docker and Podman use explicit +`Unconfined` by default because their runtime-default profiles commonly block +the supervisor's namespace mount setup; the Helm chart uses the same default. + +Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, +and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, +and the explicit cleartext-Basic-auth acknowledgement without allowing +workload environment to override them. Local containers project provider SPIFFE +through a dedicated host UNIX-socket parent mount. A VM cannot safely expose +that host socket: it accepts only a separately operated, concrete TCP listener +when `provider_spiffe_allow_guest_tcp = true` explicitly acknowledges guest +access. Host-only sockets are never implicitly forwarded to VM guests. The Kubernetes deployment packaging has two ownership boundaries. The gateway chart owns the gateway workload, configuration, Services, PKI, and diff --git a/architecture/gateway.md b/architecture/gateway.md index 11c1ebcf27..59a4d42b8e 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -246,10 +246,10 @@ controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing deployments. Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. -The config default is -`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM -gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other -shared deployments should set a positive TTL. +Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local +single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`. +Kubernetes and other shared deployments should set a positive TTL. Explicit +zero is rejected. Gateway JWT signing-key rotation is currently an offline operator action. The runtime loads one active signing key and one matching public verification key @@ -690,10 +690,9 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. The canonical -selector is the singular `[openshell.gateway] compute_driver`; the legacy -`compute_drivers` list remains accepted and normalizes into the existing -exactly-one-driver runtime validation. See `docs/reference/gateway-config.mdx` +Driver implementation settings live exclusively in TOML driver tables. The +selector is the singular `[openshell.gateway] compute_driver`; legacy +`compute_drivers` lists are rejected. See `docs/reference/gateway-config.mdx` for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with @@ -708,29 +707,20 @@ aliases, network names, and the sandbox JWT issuer. `database_url` is env-only and rejected when present in the file (`OPENSHELL_DB_URL` / `--db-url`). -### Driver inheritance - -`[openshell.gateway]` carries shared defaults such as `default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, and -`host_gateway_ip`. It also continues to accept the historical -`sandbox_namespace`, `service_account_name`, and `enable_user_namespaces` -locations as compatibility inputs. Canonical Kubernetes configuration places -those values in `[openshell.drivers.kubernetes]` as `namespace`, -`service_account_name`, and `enable_user_namespaces`; canonical Docker -configuration uses `sandbox_label`. Driver-table values take precedence over -compatibility inputs. The allowlist is per-driver so a gateway-wide default -cannot land in a driver that does not understand it (for example, -`client_tls_secret_name` is K8s-only). - -`image_pull_policy` is intentionally **not** inheritable: Kubernetes uses -`Always | IfNotPresent | Never` (passed verbatim to the K8s API) while -Podman uses the lowercase enum `always | missing | never | newer`. No -value means the same thing in both, so the key lives only under each -driver's own table. - -Driver-specific values that are not part of the inheritance allowlist -(e.g. Podman `socket_path`, VM `vcpus`) only come from the driver's own -table. +### Driver ownership + +`[openshell.gateway]` contains gateway process settings only. Each selected +driver reads its own configuration exclusively from +`[openshell.drivers.]`; values are never inherited from gateway scope. +Kubernetes owns `namespace`, `default_image`, `supervisor_image`, +`client_tls_secret_name`, `service_account_name`, `host_gateway_ip`, +`enable_user_namespaces`, and `sa_token_ttl_secs`. Docker uses +`sandbox_label` instead of the legacy `sandbox_namespace` name. Podman and VM +likewise own their image, endpoint, and runtime settings in their tables. + +`image_pull_policy` uses the shared canonical vocabulary +`always | if_not_present | never | newer`. Drivers translate it to their runtime +APIs; `newer` is supported only by Podman and rejected by Docker and Kubernetes. ### OTLP export diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 2da3225252..3d5c2d0709 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -6,7 +6,9 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; +use std::fmt; use std::net::SocketAddr; +use std::num::NonZeroU64; use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -194,12 +196,9 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, - /// Compute drivers configured for the gateway. - /// - /// The config shape allows multiple drivers so the gateway can evolve - /// toward multi-backend routing. Current releases require exactly one - /// configured driver. - pub compute_drivers: Vec, + /// Explicit compute driver configured for the gateway. + /// `None` enables runtime auto-detection. + pub compute_driver: Option, /// Operator-provided endpoints for named remote compute drivers. /// @@ -503,6 +502,225 @@ const fn default_jwks_ttl_secs() -> u64 { 3600 } +/// Canonical policy controlling when a driver pulls a sandbox image. +/// +/// Backends translate this shared vocabulary to their runtime API. `newer` is +/// supported only by Podman; other backends reject it during configuration. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImagePullPolicy { + /// Always pull, even if a local image is available. + Always, + /// Pull only when a local image is unavailable. + #[default] + IfNotPresent, + /// Never pull; fail when a local image is unavailable. + Never, + /// Pull only when the registry image is newer than the local copy. + Newer, +} + +impl ImagePullPolicy { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Always => "always", + Self::IfNotPresent => "if_not_present", + Self::Never => "never", + Self::Newer => "newer", + } + } +} + +impl fmt::Display for ImagePullPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ImagePullPolicy { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "always" => Ok(Self::Always), + "if_not_present" => Ok(Self::IfNotPresent), + "never" => Ok(Self::Never), + "newer" => Ok(Self::Newer), + other => Err(format!( + "invalid image pull policy '{other}'; expected one of: always, if_not_present, never, newer" + )), + } + } +} + +/// Canonical `AppArmor` confinement requested for a sandbox container. +/// +/// Drivers translate this model to their runtime API. An omitted value leaves +/// the runtime default unchanged; `Unconfined` is explicit because the +/// supervisor needs mount operations that the default Docker/Podman profile +/// commonly denies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppArmorProfile { + RuntimeDefault, + Unconfined, + Localhost(String), +} + +impl AppArmorProfile { + #[must_use] + pub const fn kubernetes_type(&self) -> &'static str { + match self { + Self::RuntimeDefault => "RuntimeDefault", + Self::Unconfined => "Unconfined", + Self::Localhost(_) => "Localhost", + } + } + + #[must_use] + pub fn localhost_profile(&self) -> Option<&str> { + match self { + Self::Localhost(profile) => Some(profile), + Self::RuntimeDefault | Self::Unconfined => None, + } + } + + /// Translate to the OCI `apparmor=` security option. + /// + /// `RuntimeDefault` deliberately returns `None`: omitting an OCI option + /// asks Docker/Podman to apply their runtime default profile. + #[must_use] + pub fn oci_security_opt(&self) -> Option { + match self { + Self::RuntimeDefault => None, + Self::Unconfined => Some("apparmor=unconfined".to_string()), + Self::Localhost(profile) => Some(format!("apparmor={profile}")), + } + } +} + +impl fmt::Display for AppArmorProfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RuntimeDefault => f.write_str("RuntimeDefault"), + Self::Unconfined => f.write_str("Unconfined"), + Self::Localhost(profile) => write!(f, "Localhost/{profile}"), + } + } +} + +impl FromStr for AppArmorProfile { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "RuntimeDefault" => Ok(Self::RuntimeDefault), + "Unconfined" => Ok(Self::Unconfined), + other => match other.strip_prefix("Localhost/") { + Some("") => Err( + "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" + .to_string(), + ), + Some(profile) if !profile.contains(char::is_whitespace) => { + Ok(Self::Localhost(profile.to_string())) + } + Some(_) => { + Err("invalid AppArmor localhost profile; whitespace is not allowed".to_string()) + } + None => Err(format!( + "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" + )), + }, + } + } +} + +impl Serialize for AppArmorProfile { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for AppArmorProfile { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +/// Common local-driver corporate forward-proxy settings. +/// +/// This type is `flatten`ed by local compute-driver tables, preserving the +/// established TOML field names while keeping their safety contract shared. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct UpstreamProxyConfig { + pub https_proxy: Option, + pub no_proxy: Option, + pub proxy_auth_file: Option, + pub proxy_auth_allow_insecure: Option, + pub proxy_connect_by_hostname: Option, +} + +impl UpstreamProxyConfig { + /// Validate relationships that are independent of the container backend. + /// Credential contents are intentionally not read here and are never put + /// in an error message; drivers validate and stage them per sandbox. + pub fn validate(&self) -> Result<(), String> { + use crate::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; + + let proxy_secure = if let Some(url) = self.https_proxy.as_deref() { + parse_upstream_proxy_url(url) + .map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials; supply them with proxy_auth_file so they are not stored in configuration or runtime metadata".to_string(), + err => format!("https_proxy {err}"), + })? + .secure + } else { + false + }; + + if self + .no_proxy + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if self.no_proxy.is_some() && self.https_proxy.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + if let Some(path) = self.proxy_auth_file.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_auth_file must not be empty when set".to_string()); + } + if self.https_proxy.is_none() { + return Err("proxy_auth_file is set but no https_proxy is configured".to_string()); + } + if !proxy_secure && self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy_auth_file sends a cleartext Basic credential to an http:// proxy; set proxy_auth_allow_insecure = true to acknowledge that exposure".to_string()); + } + } else if self.proxy_auth_allow_insecure.is_some() { + return Err( + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), + ); + } + if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + Ok(()) + } +} + /// Gateway-minted sandbox JWT configuration. /// /// Points the gateway at the Ed25519 signing key (produced by `certgen`) @@ -522,40 +740,23 @@ pub struct GatewayJwtConfig { /// `openshell`. #[serde(default = "default_gateway_id")] pub gateway_id: String, - /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. Canonical serialized - /// configuration omits the field for that non-expiring behavior; explicit - /// legacy zero remains accepted. - #[serde( - default = "default_sandbox_token_ttl_secs", - skip_serializing_if = "is_default" - )] - pub ttl_secs: u64, + /// Token lifetime in seconds. Omit the field for a non-expiring token. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_secs: Option, } impl GatewayJwtConfig { - /// Effective token lifetime. `None` preserves the established non-expiring - /// behavior represented by an omitted or explicit zero `ttl_secs` value. + /// Effective token lifetime. `None` represents a non-expiring token. pub fn sandbox_token_ttl(&self) -> Option { - (self.ttl_secs != 0).then(|| Duration::from_secs(self.ttl_secs)) + self.ttl_secs.map(|ttl| Duration::from_secs(ttl.get())) } } -fn is_default(value: &T) -> bool -where - T: Default + PartialEq, -{ - value == &T::default() -} - fn default_gateway_id() -> String { "openshell".to_string() } -const fn default_sandbox_token_ttl_secs() -> u64 { - 0 -} - fn default_roles_claim() -> String { "realm_access.roles".to_string() } @@ -589,7 +790,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), - compute_drivers: vec![], + compute_driver: None, compute_driver_endpoints: BTreeMap::new(), credential_drivers: Vec::new(), default_credential_driver: None, @@ -640,17 +841,10 @@ impl Config { self } - /// Create a new configuration with the configured compute drivers. + /// Create a new configuration with an explicit compute driver. #[must_use] - pub fn with_compute_drivers(mut self, drivers: I) -> Self - where - I: IntoIterator, - D: ToString, - { - self.compute_drivers = drivers - .into_iter() - .map(|driver| driver.to_string()) - .collect(); + pub fn with_compute_driver(mut self, driver: impl ToString) -> Self { + self.compute_driver = Some(driver.to_string()); self } @@ -847,7 +1041,7 @@ mod tests { use super::{ Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, + GatewayProviderProfileSourceConfig, ImagePullPolicy, PolicyValidationFailureMode, normalize_compute_driver_name, }; use std::net::SocketAddr; @@ -942,7 +1136,7 @@ mod tests { })) .expect("gateway JWT config should deserialize with default ttl"); - assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.ttl_secs, None); assert_eq!(cfg.sandbox_token_ttl(), None); let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); @@ -964,6 +1158,34 @@ mod tests { assert_eq!(serialized["ttl_secs"], 3600); } + #[test] + fn gateway_jwt_ttl_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 0 + })) + .expect_err("zero TTL must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); + } + + #[test] + fn image_pull_policy_uses_canonical_vocabulary() { + for (value, expected) in [ + ("always", ImagePullPolicy::Always), + ("if_not_present", ImagePullPolicy::IfNotPresent), + ("never", ImagePullPolicy::Never), + ("newer", ImagePullPolicy::Newer), + ] { + assert_eq!(value.parse::(), Ok(expected)); + assert_eq!(expected.to_string(), value); + assert_eq!(serde_json::to_value(expected).unwrap(), value); + } + assert!("missing".parse::().is_err()); + assert!("IfNotPresent".parse::().is_err()); + } + #[test] fn name_defaults_and_can_be_overridden() { assert_eq!(Config::new(None).name, "openshell"); diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index e26ea53f7a..9cf3022014 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -74,14 +74,6 @@ pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest" /// secrets, so this is the same delivery the per-sandbox JWT already uses. pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; -/// Guest path for the corporate proxy CA bundle in VM sandboxes. -/// -/// A CA certificate is not secret, so unlike the credential this is staged -/// world-readable. The supervisor trusts it for the handshake with an -/// `https://` proxy and for server certificates re-signed by a -/// TLS-intercepting proxy. -pub const VM_GUEST_PROXY_CA_PATH: &str = "/opt/openshell/tls/proxy-ca.pem"; - /// Guest path for the driver-authored supervisor argument list in VM sandboxes. /// /// Podman and Kubernetes build the supervisor's command line directly; the VM @@ -127,10 +119,9 @@ mod tests { VM_GUEST_TLS_CERT_PATH, VM_GUEST_TLS_KEY_PATH, VM_GUEST_SANDBOX_TOKEN_PATH, + VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, VM_GUEST_INIT_DROPIN_DIR, VM_GUEST_INIT_DROPIN_MANIFEST, - VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, - VM_GUEST_PROXY_CA_PATH, VM_GUEST_SUPERVISOR_ARGS_PATH, VM_UMOCI_PATH, VM_SANDBOX_OWNER_NORMALIZED_MARKER, diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 74871751da..03f3adf45a 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -7,6 +7,73 @@ use std::path::{Path, PathBuf}; use crate::proto::compute::v1::DriverSandbox; +/// Built-in sandbox network topologies used to derive a callback endpoint +/// when an operator does not configure a per-driver `grpc_endpoint` override. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GatewayCallbackTopology<'a> { + /// A sandbox pod reaches the gateway through its Kubernetes service. + Kubernetes { namespace: &'a str }, + /// A Docker container reaches the host through Docker's gateway alias. + Docker, + /// A Podman container reaches the host through Podman's gateway alias. + Podman, + /// A libkrun guest reaches the host through gvproxy's gateway alias. + Vm, +} + +/// Build the endpoint a sandbox uses to call its gateway for a known topology. +/// +/// The result is deliberately derived by the gateway rather than baked into +/// individual driver defaults. A configured `grpc_endpoint` remains an +/// operator override for remote or non-standard deployments. +#[must_use] +pub fn gateway_callback_endpoint( + topology: GatewayCallbackTopology<'_>, + gateway_port: u16, + gateway_tls_enabled: bool, +) -> String { + let scheme = if gateway_tls_enabled { "https" } else { "http" }; + let host = match topology { + GatewayCallbackTopology::Kubernetes { namespace } => { + return format!("{scheme}://openshell-gateway.{namespace}.svc:{gateway_port}"); + } + GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", + GatewayCallbackTopology::Podman => "host.containers.internal", + }; + format!("{scheme}://{host}:{gateway_port}") +} + +#[cfg(test)] +mod callback_endpoint_tests { + use super::{GatewayCallbackTopology, gateway_callback_endpoint}; + + #[test] + fn derives_endpoint_for_each_builtin_topology() { + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Docker, 17670, false), + "http://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Podman, 17670, true), + "https://host.containers.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), + "https://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint( + GatewayCallbackTopology::Kubernetes { + namespace: "agents" + }, + 8080, + true, + ), + "https://openshell-gateway.agents.svc:8080" + ); + } +} + // --------------------------------------------------------------------------- // Sandbox container/pod label keys (openshell.ai/ namespace) // --------------------------------------------------------------------------- @@ -663,6 +730,68 @@ pub fn validate_upstream_proxy_settings( /// Container-side directory where the provider SPIFFE Workload API socket is mounted. pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workload-api"; +/// Validate a host UNIX socket selected for provider SPIFFE projection. +/// +/// Local container drivers bind-mount the socket's dedicated parent directory, +/// not a broad host root. TCP endpoints are deliberately rejected here: a +/// container projection must be a filesystem socket, while VM guest TCP +/// exposure has its own explicit acknowledgement contract. +pub fn validate_provider_spiffe_unix_socket(path: &Path) -> Result<(), String> { + let raw = path + .to_str() + .ok_or_else(|| "provider_spiffe_workload_api_socket must be valid UTF-8".to_string())?; + if raw.trim() != raw || raw.is_empty() { + return Err("provider_spiffe_workload_api_socket must not be empty or contain surrounding whitespace".to_string()); + } + if raw.starts_with("tcp:") || raw.starts_with("unix:") { + return Err("provider_spiffe_workload_api_socket must be an absolute host UNIX socket path, not a URI".to_string()); + } + if !path.is_absolute() || path.parent().is_none_or(|parent| parent == Path::new("/")) { + return Err("provider_spiffe_workload_api_socket must be an absolute UNIX socket path below a dedicated parent directory".to_string()); + } + Ok(()) +} + +/// Return the guest/container path for a projected provider SPIFFE socket. +pub fn projected_provider_spiffe_socket_path(path: &Path) -> Result { + validate_provider_spiffe_unix_socket(path)?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .ok_or_else(|| "provider_spiffe_workload_api_socket must name a socket file".to_string())?; + Ok(format!( + "{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}/{file_name}" + )) +} + +/// Validate an explicitly operator-acknowledged guest-reachable SPIFFE TCP endpoint. +/// +/// The `tcp:` spelling is the SPIFFE Workload API endpoint grammar accepted by +/// the client. The address must be concrete; wildcard and host-only UNIX +/// sockets are never silently exposed to VM guests. +pub fn validate_guest_spiffe_tcp_endpoint( + endpoint: &str, + acknowledged: bool, +) -> Result<(), String> { + if endpoint.trim() != endpoint || endpoint.is_empty() { + return Err("provider_spiffe_workload_api_tcp_endpoint must not be empty or contain surrounding whitespace".to_string()); + } + if !acknowledged { + return Err("provider_spiffe_workload_api_tcp_endpoint exposes a Workload API to VM guests; set provider_spiffe_allow_guest_tcp = true only after explicitly acknowledging that exposure".to_string()); + } + let address = endpoint.strip_prefix("tcp:").ok_or_else(|| { + "provider_spiffe_workload_api_tcp_endpoint must use tcp:host:port (for example tcp:192.0.2.10:8081)".to_string() + })?; + let address: std::net::SocketAddr = address.parse().map_err(|_| { + "provider_spiffe_workload_api_tcp_endpoint must use a concrete IP address and non-zero port".to_string() + })?; + if address.ip().is_unspecified() || address.port() == 0 { + return Err("provider_spiffe_workload_api_tcp_endpoint must not use an unspecified address or port 0".to_string()); + } + Ok(()) +} + /// Return the XDG state path for a driver's sandbox JWT token file. /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. @@ -1101,6 +1230,28 @@ mod tests { } #[cfg(unix)] + #[test] + fn projected_spiffe_socket_requires_dedicated_absolute_unix_path() { + assert_eq!( + projected_provider_spiffe_socket_path(Path::new("/run/spire/agent.sock")).unwrap(), + "/spiffe-workload-api/agent.sock" + ); + for path in ["relative.sock", "/agent.sock", "tcp:127.0.0.1:8081"] { + assert!( + validate_provider_spiffe_unix_socket(Path::new(path)).is_err(), + "{path}" + ); + } + } + + #[test] + fn guest_spiffe_tcp_requires_acknowledgement_and_concrete_endpoint() { + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", true).is_ok()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", false).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:0.0.0.0:8081", true).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("unix:/run/spire/agent.sock", true).is_err()); + } + #[test] fn credential_file_rejects_fifo_without_hanging() { // A FIFO with no writer would block a blocking open() forever. The diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 7acb72dd6f..db634101ea 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -52,10 +52,11 @@ pub mod time; pub mod transport_errors; pub use config::{ - Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, - GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, - GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, - PolicyValidationFailureMode, TlsConfig, + AppArmorProfile, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, + GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, + GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, + ImagePullPolicy, MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + UpstreamProxyConfig, }; pub use dynamic_string_allowlist::DynamicStringAllowlist; pub use error::{ComputeDriverError, Error, Result}; diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..31c59bc6f3 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Omit `[openshell.drivers.docker].sandbox_pids_limit` to inherit the Docker/runtime default; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | @@ -187,6 +187,25 @@ mounted into the container and exposed with: HTTP endpoints reject TLS material because the supervisor would not use it. +## Corporate proxy, SPIFFE, and AppArmor + +`https_proxy`, `no_proxy`, and `proxy_auth_file` in +`[openshell.drivers.docker]` are operator-owned supervisor settings. Docker +passes the proxy URL and bypass list on the supervisor command line and mounts +an optional `user:pass` auth file read-only at a root-only path. Credentials +never appear in container environment or Docker labels. An auth file used with +an `http://` proxy requires `proxy_auth_allow_insecure = true`; an `https://` +proxy protects the Basic-auth header in its TLS session. + +Set `provider_spiffe_workload_api_socket` to an absolute host UNIX socket to +project its dedicated parent directory into the supervisor and set +`OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to the guest path. TCP URIs are +rejected for this projection. `app_armor_profile` uses the shared +`RuntimeDefault`, `Unconfined`, or `Localhost/` vocabulary. Docker +uses explicit `Unconfined` by default because the supervisor's namespace mount +setup is incompatible with `docker-default`; requested confined profiles fail +at startup if Docker does not report AppArmor support. + ## Environment Ownership The driver merges template environment and sandbox spec environment first, then diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 73a80822f3..617177f516 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -21,13 +21,14 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackTopology, LABEL_MANAGED_BY, + LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -54,7 +55,9 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Error, Result as CoreResult}; +use openshell_core::{ + AppArmorProfile, Error, ImagePullPolicy, Result as CoreResult, UpstreamProxyConfig, +}; use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -79,6 +82,10 @@ const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = + openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; +const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = + openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; @@ -119,10 +126,9 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: String, + pub image_pull_policy: ImagePullPolicy, /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. - #[serde(alias = "sandbox_namespace")] pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. @@ -156,13 +162,29 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to leave Docker's runtime/default PID limit unchanged. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, + + /// Corporate forward-proxy settings supplied to the supervisor on argv. + /// The flattened fields retain the common `https_proxy`, `no_proxy`, and + /// `proxy_auth_*` gateway TOML contract. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Host UNIX socket to project into sandbox supervisors for provider + /// SPIFFE token exchange. + pub provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` confinement requested for sandbox containers. The explicit + /// default preserves the prior supervisor-compatible Docker behavior. + pub app_armor_profile: Option, } impl Default for DockerComputeConfig { @@ -170,7 +192,7 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), + image_pull_policy: ImagePullPolicy::default(), sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, @@ -181,8 +203,11 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } } @@ -197,7 +222,7 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: String, + image_pull_policy: ImagePullPolicy, sandbox_label: String, grpc_endpoint: String, network_name: String, @@ -211,8 +236,11 @@ struct DockerDriverRuntimeConfig { daemon_version: String, supports_gpu: bool, allow_all_default_gpu: bool, - sandbox_pids_limit: i64, + sandbox_pids_limit: Option, enable_bind_mounts: bool, + upstream_proxy: UpstreamProxyConfig, + provider_spiffe_workload_api_socket: Option, + app_armor_profile: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -505,6 +533,17 @@ impl DockerComputeDriver { let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + validate_image_pull_policy(docker_config.image_pull_policy)?; + docker_config + .upstream_proxy + .validate() + .map_err(Error::config)?; + validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; + if let Some(socket) = docker_config.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } + validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); if gateway_port == 0 { return Err(Error::config( @@ -520,13 +559,13 @@ impl DockerComputeDriver { docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + docker_config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Docker, + gateway_port, + docker_config.guest_tls_ca.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some(), + ); } let grpc_endpoint = docker_container_openshell_endpoint( &docker_config.grpc_endpoint, @@ -541,7 +580,7 @@ impl DockerComputeDriver { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), + image_pull_policy: docker_config.image_pull_policy, sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, @@ -557,6 +596,11 @@ impl DockerComputeDriver { allow_all_default_gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + upstream_proxy: docker_config.upstream_proxy.clone(), + provider_spiffe_workload_api_socket: docker_config + .provider_spiffe_workload_api_socket + .clone(), + app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -1561,9 +1605,8 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - let inspect = match policy.as_str() { - "" | "ifnotpresent" => { + let inspect = match self.config.image_pull_policy { + ImagePullPolicy::IfNotPresent => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -1580,14 +1623,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - "always" => { + ImagePullPolicy::Always => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - "never" => match self.docker.inspect_image(image).await { + ImagePullPolicy::Never => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -1599,15 +1642,15 @@ impl DockerComputeDriver { } Err(err) if is_not_found_error(&err) => { return Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" + "docker image '{image}' is not present locally and image_pull_policy = \"never\"" ))); } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - other => { - return Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))); + ImagePullPolicy::Newer => { + return Err(Status::failed_precondition( + "image_pull_policy = \"newer\" is supported only by the Podman compute driver", + )); } }; @@ -2586,6 +2629,49 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } +/// Verify the configured credential without exposing its contents. Docker +/// bind-mounts the root-owned file directly, unlike Podman which uses a native +/// secret object; this preflight makes a bad file fail before any sandbox is +/// created. +fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { + let Some(path) = config.proxy_auth_file.as_ref() else { + return Ok(()); + }; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( + path.to_str() + .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, + ) + .map_err(Error::config)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; + Ok(()) +} + +/// Build immutable operator-owned proxy arguments. Credentials never appear on +/// argv: only the fixed in-container root-only file path is supplied. +fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = config.https_proxy.as_ref() { + args.extend(["--upstream-proxy".to_string(), url.clone()]); + } + if let Some(no_proxy) = config.no_proxy.as_ref() { + args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); + } + if config.proxy_auth_file.is_some() { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2615,6 +2701,23 @@ fn build_binds( SANDBOX_TOKEN_MOUNT_PATH )); } + if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { + binds.push(format!( + "{}:{}:ro,z", + path.display(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH + )); + } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let parent = socket.parent().ok_or_else(|| { + Status::failed_precondition("provider SPIFFE socket has no parent directory") + })?; + binds.push(format!( + "{}:{}:ro,rbind", + parent.display(), + PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + )); + } Ok(binds) } @@ -2797,6 +2900,15 @@ fn build_environment_for_oci_user( TLS_KEY_MOUNT_PATH.to_string(), ); } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() + && let Ok(path) = + openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) + { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + path, + ); + } environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); @@ -3007,7 +3119,11 @@ fn build_container_create_body_for_image( entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + cmd: { + let mut args = vec!["--workdir".to_string(), workspace_root]; + args.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + Some(args) + }, labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3029,17 +3145,13 @@ fn build_container_create_body_for_image( "SYS_PTRACE".to_string(), "SYSLOG".to_string(), ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), + // The default is explicitly Unconfined because the supervisor + // needs mount operations commonly denied by docker-default. + security_opt: config + .app_armor_profile + .as_ref() + .and_then(AppArmorProfile::oci_security_opt) + .map(|option| vec![option]), network_mode: Some(config.network_name.clone()), extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), ..Default::default() @@ -3324,26 +3436,55 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { +fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { + if value.is_some_and(|limit| limit.get() < 0) { return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", + "docker sandbox_pids_limit must be positive when set", )); } Ok(()) } -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", +fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { + if policy == ImagePullPolicy::Newer { + return Err(Error::config( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", )); } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) + Ok(()) +} + +fn validate_docker_app_armor_profile( + profile: Option<&AppArmorProfile>, + info: &SystemInfo, +) -> CoreResult<()> { + let requires_apparmor = matches!( + profile, + Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) + ); + if !requires_apparmor { + return Ok(()); + } + let available = info.security_options.as_ref().is_some_and(|options| { + options + .iter() + .any(|option| option.to_ascii_lowercase().contains("apparmor")) + }); + if !available { + return Err(Error::config( + "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + )); } + Ok(()) +} + +fn docker_pids_limit(value: Option) -> Result, Status> { + if value.is_some_and(|limit| limit.get() < 0) { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be positive when set", + )); + } + Ok(value.map(std::num::NonZeroI64::get)) } #[allow(clippy::cast_possible_truncation)] @@ -3963,12 +4104,6 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { - docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() -} - pub(crate) fn docker_guest_tls_paths( docker_config: &DockerComputeConfig, ) -> CoreResult> { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 7a623409b7..e04b79f87b 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -95,7 +95,7 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: String::new(), + image_pull_policy: ImagePullPolicy::IfNotPresent, sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), @@ -122,8 +122,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { daemon_version: "28.0.0".to_string(), supports_gpu: false, allow_all_default_gpu: false, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } @@ -139,20 +142,69 @@ fn docker_config_uses_canonical_sandbox_label_name() { } #[test] -fn docker_config_accepts_legacy_sandbox_namespace_alias() { - let config: DockerComputeConfig = - serde_json::from_value(serde_json::json!({ "sandbox_namespace": "tenant-a" })).unwrap(); - assert_eq!(config.sandbox_label, "tenant-a"); +fn docker_config_rejects_legacy_sandbox_namespace() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_namespace": "tenant-a" + })) + .expect_err("legacy sandbox_namespace must be rejected"); + assert!(error.to_string().contains("sandbox_namespace")); } #[test] -fn docker_config_rejects_canonical_and_legacy_sandbox_label_names_together() { - let error = serde_json::from_value::(serde_json::json!({ - "sandbox_label": "tenant-a", - "sandbox_namespace": "tenant-b" +fn docker_config_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 })) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: DockerComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = validate_sandbox_pids_limit(negative.sandbox_pids_limit).unwrap_err(); + assert!(error.to_string().contains("must be positive")); +} + +#[test] +fn docker_rejects_newer_image_pull_policy() { + let error = validate_image_pull_policy(ImagePullPolicy::Newer).unwrap_err(); + assert!(error.to_string().contains("supported only by the Podman")); +} + +#[test] +fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { + let config: DockerComputeConfig = toml::from_str( + r#" +https_proxy = "http://proxy.example:8080" +no_proxy = ".svc" +proxy_auth_file = "/run/secrets/proxy-auth" +proxy_auth_allow_insecure = true +app_armor_profile = "Localhost/openshell-supervisor" +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +"#, + ) + .unwrap(); + assert_eq!( + config.upstream_proxy.https_proxy.as_deref(), + Some("http://proxy.example:8080") + ); + assert_eq!( + config.app_armor_profile, + Some(AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )) + ); + assert!(config.upstream_proxy.validate().is_ok()); + assert!( + openshell_core::driver_utils::validate_provider_spiffe_unix_socket( + config + .provider_spiffe_workload_api_socket + .as_deref() + .unwrap() + ) + .is_ok() + ); } fn json_struct(value: serde_json::Value) -> prost_types::Struct { @@ -557,7 +609,7 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = "unsupported".to_string(); + config.image_pull_policy = ImagePullPolicy::Newer; let driver = test_driver_with_config(config); async { @@ -1211,13 +1263,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { +fn docker_pids_limit_uses_runtime_default_when_omitted() { assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) + docker_pids_limit(std::num::NonZeroI64::new(2048)).unwrap(), + Some(2048) ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); + assert_eq!(docker_pids_limit(None).unwrap(), None); + assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); } #[test] @@ -1227,10 +1279,10 @@ fn docker_compute_config_disables_bind_mounts_by_default() { } #[test] -fn container_create_body_sets_driver_owned_pids_limit() { +fn container_create_body_omits_pids_limit_by_default() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); + assert_eq!(host_config.pids_limit, None); } #[test] @@ -2166,6 +2218,46 @@ fn build_environment_uses_token_file_without_raw_token_env() { ))); } +#[test] +fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { + let mut config = runtime_config(); + config.upstream_proxy = UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }; + config.provider_spiffe_workload_api_socket = Some(PathBuf::from("/run/spire/agent.sock")); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + let command = body.cmd.unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy", "https://proxy.example:8443"]) + ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) + ); + let binds = body.host_config.unwrap().binds.unwrap(); + assert!( + binds + .iter() + .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) + ); + assert!( + binds + .iter() + .any(|bind| bind.contains(PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR)) + ); + let env = body.env.unwrap(); + assert!(env.iter().any(|entry| entry + == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); + assert!(!env.iter().any(|entry| entry.contains("proxy-auth"))); +} + #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..5d6154bd17 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -152,14 +152,13 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. -The driver can request a Kubernetes AppArmor profile through -`app_armor_profile`. - +The driver uses the shared AppArmor model through `app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`. An empty or unset value omits -`securityContext.appArmorProfile`. Helm deployments default sandbox agent -containers to `Unconfined` because runtime/default AppArmor profiles can block -the supervisor's network namespace mount setup on AppArmor-enabled nodes. +`Localhost/`; an empty or unset value omits +`securityContext.appArmorProfile`. Docker and Podman translate the same values +to OCI security options. Helm deployments default sandbox agent containers to +`Unconfined` because runtime/default AppArmor profiles can block the +supervisor's network namespace mount setup on AppArmor-enabled nodes. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 805c0314b0..bfdceff9d4 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,8 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub use openshell_core::AppArmorProfile; pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; -use openshell_core::config; +use openshell_core::{ImagePullPolicy, config}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; #[cfg(test)] @@ -177,83 +178,6 @@ impl KubernetesSidecarConfig { } } -/// Kubernetes `AppArmor` profile requested for the sandbox agent container. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AppArmorProfile { - RuntimeDefault, - Unconfined, - Localhost(String), -} - -impl AppArmorProfile { - #[must_use] - pub fn to_k8s_type(&self) -> &'static str { - match self { - Self::RuntimeDefault => "RuntimeDefault", - Self::Unconfined => "Unconfined", - Self::Localhost(_) => "Localhost", - } - } - - #[must_use] - pub fn localhost_profile(&self) -> Option<&str> { - match self { - Self::Localhost(profile) => Some(profile), - Self::RuntimeDefault | Self::Unconfined => None, - } - } -} - -impl std::fmt::Display for AppArmorProfile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RuntimeDefault => f.write_str("RuntimeDefault"), - Self::Unconfined => f.write_str("Unconfined"), - Self::Localhost(profile) => write!(f, "Localhost/{profile}"), - } - } -} - -impl FromStr for AppArmorProfile { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "RuntimeDefault" => Ok(Self::RuntimeDefault), - "Unconfined" => Ok(Self::Unconfined), - other => match other.strip_prefix("Localhost/") { - Some("") => Err( - "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" - .to_string(), - ), - Some(profile) => Ok(Self::Localhost(profile.to_string())), - None => Err(format!( - "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" - )), - }, - } - } -} - -impl Serialize for AppArmorProfile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for AppArmorProfile { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::from_str(&value).map_err(serde::de::Error::custom) - } -} - fn deserialize_optional_app_armor_profile<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -263,7 +187,8 @@ where let value = Option::::deserialize(deserializer)?; match value.as_deref() { None | Some("") => Ok(None), - Some(value) => AppArmorProfile::from_str(value) + Some(value) => value + .parse::() .map(Some) .map_err(serde::de::Error::custom), } @@ -306,7 +231,9 @@ pub struct KubernetesComputeConfig { /// the driver's `TokenReview` bootstrap authenticator. pub service_account_name: String, pub default_image: String, - pub image_pull_policy: String, + /// Pull policy for sandbox images. Omit to use Kubernetes's image default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_pull_policy: Option, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, /// Managed-mode SSH ingress isolation. When enabled, the driver creates a @@ -317,9 +244,10 @@ pub struct KubernetesComputeConfig { /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. pub supervisor_image: String, - /// Kubernetes `imagePullPolicy` for the supervisor image. - /// Empty string delegates to the Kubernetes default. - pub supervisor_image_pull_policy: String, + /// Pull policy for the supervisor image. Omit to use Kubernetes's image + /// default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supervisor_image_pull_policy: Option, /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. @@ -439,15 +367,13 @@ impl Default for KubernetesComputeConfig { operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), - // Default empty so the gateway omits `imagePullPolicy` from pod - // specs and Kubernetes applies its own default (Always for `latest`, - // IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing") - // is Podman vocabulary and is not a valid Kubernetes value. - image_pull_policy: String::new(), + // Omit the field so Kubernetes applies its own default (Always for + // `latest`, IfNotPresent otherwise). + image_pull_policy: None, image_pull_secrets: Vec::new(), managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), - supervisor_image_pull_policy: String::new(), + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), @@ -506,17 +432,52 @@ impl KubernetesComputeConfig { self.sidecar.validate_proxy_uid() } + /// Reject pull policies Kubernetes cannot express before creating pods. + pub fn validate_image_pull_policies(&self) -> Result<(), String> { + for (field, policy) in [ + ("image_pull_policy", self.image_pull_policy), + ( + "supervisor_image_pull_policy", + self.supervisor_image_pull_policy, + ), + ] { + if policy == Some(ImagePullPolicy::Newer) { + return Err(format!( + "{field} = \"newer\" is supported only by the Podman compute driver" + )); + } + } + Ok(()) + } + + /// Translate a validated shared policy to Kubernetes's API vocabulary. + #[must_use] + pub fn image_pull_policy_value(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "Always", + ImagePullPolicy::IfNotPresent => "IfNotPresent", + ImagePullPolicy::Never => "Never", + ImagePullPolicy::Newer => unreachable!("newer must be rejected during validation"), + } + } + /// Validate the operator-owned corporate upstream proxy configuration. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - if let Some(url) = &self.https_proxy { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - })?; - } + let proxy_addr = self + .https_proxy + .as_deref() + .map(|url| { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => { + "https_proxy must not be empty when set".to_string() + } + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + }) + }) + .transpose()?; if let Some(list) = self.no_proxy.as_deref() { if list.trim().is_empty() { @@ -575,7 +536,9 @@ impl KubernetesComputeConfig { .to_string(), ); } - if self.proxy_auth_allow_insecure != Some(true) { + if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure) + && self.proxy_auth_allow_insecure != Some(true) + { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } if self.topology == SupervisorTopology::Combined { @@ -928,6 +891,34 @@ mod tests { assert!(cfg.sidecar.process_binary_aware_network_policy); } + #[test] + fn image_pull_policy_uses_shared_canonical_values() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "image_pull_policy": "if_not_present", + "supervisor_image_pull_policy": "never" + })) + .unwrap(); + assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent)); + assert_eq!( + cfg.supervisor_image_pull_policy, + Some(ImagePullPolicy::Never) + ); + assert_eq!( + KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::IfNotPresent), + "IfNotPresent" + ); + } + + #[test] + fn image_pull_policy_rejects_newer() { + let cfg = KubernetesComputeConfig { + image_pull_policy: Some(ImagePullPolicy::Newer), + ..KubernetesComputeConfig::default() + }; + let error = cfg.validate_image_pull_policies().unwrap_err(); + assert!(error.contains("supported only by the Podman")); + } + #[test] fn serde_override_topology_sidecar() { let json = serde_json::json!({ @@ -1387,6 +1378,18 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } + #[test] + fn upstream_proxy_config_accepts_tls_protected_secret_credentials() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + #[test] fn toml_deserializes_sidecar_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index bb1b75e8a9..4ee23b26b8 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -508,6 +508,9 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_image_pull_policies() + .map_err(KubernetesDriverError::Precondition)?; config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; @@ -1505,10 +1508,16 @@ impl KubernetesComputeDriver { let params = SandboxPodParams { default_image: &self.config.default_image, - image_pull_policy: &self.config.image_pull_policy, + image_pull_policy: self + .config + .image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), image_pull_secrets: &self.config.image_pull_secrets, supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, + supervisor_image_pull_policy: self + .config + .supervisor_image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, @@ -2753,13 +2762,13 @@ fn supervisor_volume_mount() -> serde_json::Value { /// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. fn supervisor_image_volume( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let mut image_spec = serde_json::json!({ "reference": supervisor_image, }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + image_spec["pullPolicy"] = serde_json::json!(policy); } serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2777,7 +2786,7 @@ fn supervisor_image_volume( /// emissary executor. fn supervisor_init_container( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ @@ -2795,8 +2804,8 @@ fn supervisor_init_container( "readOnly": false }] }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + spec["imagePullPolicy"] = serde_json::json!(policy); } spec } @@ -2804,7 +2813,7 @@ fn supervisor_init_container( fn apply_supervisor_binary_source( spec: &mut serde_json::Map, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, ) { let volumes = spec @@ -2938,7 +2947,7 @@ fn apply_supervisor_sideload_with_params( fn apply_supervisor_sideload( pod_template: &mut serde_json::Value, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, sandbox_uid: u32, sandbox_gid: u32, @@ -3148,8 +3157,8 @@ fn supervisor_sidecar_container( .into_iter() .map(serde_json::Value::String), ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if params.provider_spiffe_enabled { container["volumeMounts"] @@ -3211,8 +3220,8 @@ fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_jso sidecar_tls_volume_mount(), ] }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if !params.client_tls_secret_name.is_empty() { container["volumeMounts"] @@ -3412,7 +3421,7 @@ fn apply_supervisor_sidecar_topology( fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, - image_pull_policy: &str, + image_pull_policy: Option<&str>, sandbox_gid: u32, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { @@ -3502,8 +3511,8 @@ fn apply_workspace_persistence( "mountPath": WORKSPACE_INIT_MOUNT_PATH }] }); - if !image_pull_policy.is_empty() { - init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + if let Some(policy) = image_pull_policy { + init_spec["imagePullPolicy"] = serde_json::json!(policy); } init_containers.push(init_spec); } @@ -3550,10 +3559,10 @@ fn default_workspace_volume_claim_templates( #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, - image_pull_policy: &'a str, + image_pull_policy: Option<&'a str>, image_pull_secrets: &'a [String], supervisor_image: &'a str, - supervisor_image_pull_policy: &'a str, + supervisor_image_pull_policy: Option<&'a str>, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, proxy_uid: u32, @@ -3591,10 +3600,10 @@ impl Default for SandboxPodParams<'_> { fn default() -> Self { Self { default_image: "", - image_pull_policy: "", + image_pull_policy: None, image_pull_secrets: &[], supervisor_image: "", - supervisor_image_pull_policy: "", + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, @@ -3919,11 +3928,8 @@ fn sandbox_template_to_k8s_with_validated_config( }; if !image.is_empty() { container.insert("image".to_string(), serde_json::json!(image)); - if !params.image_pull_policy.is_empty() { - container.insert( - "imagePullPolicy".to_string(), - serde_json::json!(params.image_pull_policy), - ); + if let Some(policy) = params.image_pull_policy { + container.insert("imagePullPolicy".to_string(), serde_json::json!(policy)); } } @@ -4225,7 +4231,7 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { let mut value = serde_json::json!({ - "type": profile.to_k8s_type() + "type": profile.kubernetes_type() }); if let Some(localhost_profile) = profile.localhost_profile() { value["localhostProfile"] = serde_json::json!(localhost_profile); @@ -6202,7 +6208,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "custom-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, // sandbox_uid 1500, // sandbox_gid @@ -6239,7 +6245,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, 1600, @@ -6286,7 +6292,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6313,7 +6319,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6400,7 +6406,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6443,7 +6449,7 @@ mod tests { } #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { + fn supervisor_image_volume_omits_pull_policy_when_unspecified() { let mut pod_template = serde_json::json!({ "spec": { "containers": [{ @@ -6456,7 +6462,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "", + None, SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6466,7 +6472,7 @@ mod tests { assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); assert!( volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + "pullPolicy should be omitted when unspecified" ); } @@ -6476,7 +6482,7 @@ mod tests { topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", + supervisor_image_pull_policy: Some("IfNotPresent"), grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", client_tls_secret_name: "openshell-client-tls", proxy_uid: 2200, @@ -7354,7 +7360,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "openshell/sandbox:latest", - "IfNotPresent", + Some("IfNotPresent"), 1000, // sandbox_gid ); @@ -7413,7 +7419,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "my-custom-image:v2", - "IfNotPresent", + Some("IfNotPresent"), 1000, ); @@ -7437,7 +7443,7 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence(&mut pod_template, "img:latest", Some("Always"), 1000); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 9b11f5da2a..3047a9b108 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -8,8 +8,9 @@ use std::net::SocketAddr; use std::path::PathBuf; use tracing::info; -use openshell_core::VERSION; +use openshell_core::driver_utils::{GatewayCallbackTopology, gateway_callback_endpoint}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, @@ -72,7 +73,7 @@ struct Args { sandbox_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, + sandbox_image_pull_policy: Option, #[arg( long, @@ -114,7 +115,7 @@ struct Args { supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, + supervisor_image_pull_policy: Option, #[arg( long, @@ -239,6 +240,15 @@ async fn main() -> Result<()> { .collect::>>()?; let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let grpc_endpoint = args.grpc_endpoint.unwrap_or_else(|| { + gateway_callback_endpoint( + GatewayCallbackTopology::Kubernetes { + namespace: &args.sandbox_namespace, + }, + openshell_core::config::DEFAULT_SERVER_PORT, + false, + ) + }); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { workspace_mode: args.workspace_mode, @@ -248,7 +258,7 @@ async fn main() -> Result<()> { operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, image_pull_secrets: args.sandbox_image_pull_secrets, managed_ssh_ingress: ManagedSshIngressConfig { enabled: args.managed_ssh_ingress_enabled, @@ -258,7 +268,7 @@ async fn main() -> Result<()> { supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_image_pull_policy: args.supervisor_image_pull_policy, supervisor_sideload_method: args.supervisor_sideload_method, topology: args.topology, sidecar: KubernetesSidecarConfig { @@ -272,7 +282,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 9f8baa3446..837913ed1f 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -240,7 +240,7 @@ Step "Prepare DemoDir $DemoDir" New-Item -ItemType Directory -Force $DemoDir | Out-Null Ok "DemoDir ready" -$env:OPENSHELL_DRIVERS = "mxc" +$env:OPENSHELL_COMPUTE_DRIVER = "mxc" $env:OPENSHELL_MXC_SHARE_DIR = $DemoDir # ── Start gateway ───────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index dfccb47fae..cdb8f5d78c 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -380,14 +380,14 @@ Podman resources after out-of-band container removal or label drift. |---|---|---|---| | `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | | `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | +| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | | `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | | `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | | `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | unset | Podman cgroup PID limit for sandbox containers. Omit it to inherit Podman's runtime/default PID limit; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | @@ -405,6 +405,15 @@ Through the gateway, the same settings are the `https_proxy`, `no_proxy`, and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see `docs/reference/gateway-config.mdx`. +`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX +Workload API socket, projected through a dedicated read-only mount, or an +explicit container-reachable `tcp:IP:port` endpoint. The driver sets the +supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. +`app_armor_profile` shares the canonical +`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and +Kubernetes. Podman defaults to explicit `Unconfined` for the supervisor mount +setup; confined choices fail early when Podman reports AppArmor unavailable. + This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any `ENV` baked into the sandbox image — cannot override them, and the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..8088a50418 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -275,6 +275,9 @@ pub struct HostInfo { pub struct SecurityInfo { #[serde(default)] pub rootless: bool, + /// Whether the Podman host has `AppArmor` support enabled. + #[serde(default)] + pub apparmor_enabled: bool, } // ── Client ─────────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 04ae3fe5e3..196c1fa571 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::net::IpAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; -use std::str::FromStr; + +use openshell_core::{AppArmorProfile, ImagePullPolicy}; /// Default Podman bridge network name. pub const DEFAULT_NETWORK_NAME: &str = "openshell"; @@ -11,59 +13,14 @@ pub const MACOS_PODMAN_MACHINE_HOST_GATEWAY_IP: &str = "192.168.127.254"; /// Default Podman stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_PODMAN_STOP_TIMEOUT_SECS: u32 = 45; -// Re-export the shared default so existing imports inside this crate keep working. -pub use openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT; - -/// Image pull policy for sandbox and supervisor images. -/// -/// Controls when the Podman driver fetches a newer copy of an OCI image -/// from the registry. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ImagePullPolicy { - /// Always pull, even if a local copy exists. - Always, - /// Pull only when no local copy exists (default). - #[default] - Missing, - /// Never pull; fail if not available locally. - Never, - /// Pull only if the remote image is newer. - Newer, -} - -impl ImagePullPolicy { - /// Return the policy string expected by the Podman libpod API. - #[must_use] - pub fn as_str(&self) -> &'static str { - match self { - Self::Always => "always", - Self::Missing => "missing", - Self::Never => "never", - Self::Newer => "newer", - } - } -} - -impl std::fmt::Display for ImagePullPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ImagePullPolicy { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "always" => Ok(Self::Always), - "missing" => Ok(Self::Missing), - "never" => Ok(Self::Never), - "newer" => Ok(Self::Newer), - other => Err(format!( - "invalid pull policy '{other}'; expected one of: always, missing, never, newer" - )), - } +/// Translate the shared pull-policy vocabulary to the Podman libpod API. +#[must_use] +pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "always", + ImagePullPolicy::IfNotPresent => "missing", + ImagePullPolicy::Never => "never", + ImagePullPolicy::Newer => "newer", } } @@ -90,7 +47,6 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - #[serde(alias = "sandbox_ssh_socket_path")] pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. @@ -121,8 +77,10 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Set to `0` to leave Podman's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to leave Podman's runtime/default PID limit unchanged. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] @@ -130,14 +88,19 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, + /// `AppArmor` confinement requested for sandbox containers. The default + /// explicitly opts out because the supervisor needs mount operations that + /// the runtime default profile denies. + pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but /// increase process churn (each check spawns a conmon subprocess). - /// Set to `0` to disable health checks entirely. - /// Defaults to [`DEFAULT_HEALTH_CHECK_INTERVAL_SECS`] (10 seconds). - pub health_check_interval_secs: u64, + /// Omit the field to disable health checks entirely. Explicit zero is + /// invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub health_check_interval_secs: Option, /// Corporate forward proxy URL passed to the in-container supervisor /// (e.g. `http://proxy.corp.com:8080` or `https://proxy.corp.com:3130`). /// @@ -218,8 +181,6 @@ pub struct PodmanComputeConfig { pub gidmap: Vec, } -pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; - /// Parse a single `"container_id:host_id:size"` mapping entry. /// /// Returns `(container_id, host_id, size)` on success. @@ -299,9 +260,9 @@ impl PodmanComputeConfig { /// Validate runtime resource-limit configuration. pub fn validate_runtime_limits(&self) -> Result<(), crate::client::PodmanApiError> { - if self.sandbox_pids_limit < 0 { + if self.sandbox_pids_limit.is_some_and(|limit| limit.get() < 0) { return Err(crate::client::PodmanApiError::InvalidInput( - "sandbox_pids_limit must be zero or greater".to_string(), + "sandbox_pids_limit must be positive when set".to_string(), )); } Ok(()) @@ -319,91 +280,19 @@ impl PodmanComputeConfig { /// the URL is rejected because it would otherwise be stored in /// `gateway.toml` and exposed in container metadata. pub fn validate_proxy_config(&self) -> Result<(), crate::client::PodmanApiError> { - use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - let proxy_secure = if let Some(url) = &self.https_proxy { - let addr = parse_upstream_proxy_url(url).map_err(|err| { - crate::client::PodmanApiError::InvalidInput(match err { - UpstreamProxyUrlError::Empty => { - "https_proxy must not be empty when set".to_string() - } - UpstreamProxyUrlError::InlineCredentials => { - "https_proxy must not embed credentials in the URL; supply them via \ - proxy_auth_file so they are not stored in config or container metadata" - .to_string() - } - err => format!("https_proxy {err}"), - }) - })?; - addr.secure - } else { - false - }; - - // The supervisor treats a present-but-empty driver-supplied argument - // as a fatal misconfiguration, so never accept (and later pass) one. - if let Some(list) = self.no_proxy.as_deref() { - if list.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy must not be empty when set; omit it instead".to_string(), - )); - } - // A bypass list only makes sense relative to a proxy boundary. An - // operator who set one believed proxying was in effect, so accepting - // it while all egress dials directly would hide a fail-open state. - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy is set but no https_proxy is configured".to_string(), - )); - } - } - - if let Some(path) = self.proxy_auth_file.as_deref() { - if path.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file must not be empty when set".to_string(), - )); - } - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file is set but no https_proxy is configured".to_string(), - )); - } - // Basic auth over the plain-TCP proxy connection is readable by - // anyone on the network path; sending it requires an explicit - // operator acknowledgement rather than being an implicit side - // effect of configuring credentials. For an https:// proxy the - // credential is inside the verified TLS session, so the - // acknowledgement is unnecessary (but tolerated). - if self.proxy_auth_allow_insecure != Some(true) && !proxy_secure { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file sends the credential as cleartext Basic auth over the \ - plain-TCP connection to the http:// proxy; set proxy_auth_allow_insecure \ - = true to accept that exposure, or remove proxy_auth_file" - .to_string(), - )); - } - } else if self.proxy_auth_allow_insecure.is_some() { - // The acknowledgement without credentials means the operator - // believed an auth file was configured; surface the mismatch. - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), - )); - } - - // The CONNECT-target mode only means something relative to a proxy - // boundary the operator believed was in effect. - if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), - )); + // Keep the Podman-only CA-bundle behaviour below, but delegate the + // shared URL, bypass-list, credential-file, and acknowledgement + // contract to openshell-core so Docker and VM cannot drift. + openshell_core::UpstreamProxyConfig { + https_proxy: self.https_proxy.clone(), + no_proxy: self.no_proxy.clone(), + proxy_auth_file: self.proxy_auth_file.as_ref().map(PathBuf::from), + proxy_auth_allow_insecure: self.proxy_auth_allow_insecure, + proxy_connect_by_hostname: self.proxy_connect_by_hostname, } + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; - // A CA bundle only makes sense relative to a proxy boundary (an - // https:// proxy handshake, or a TLS-intercepting proxy's re-sign CA). - // Mirror the proxy_auth_file pairing so a stray setting cannot hide a - // fail-open state. The file's readability and certificate content are - // checked at sandbox-create time (see the driver) and fail closed in - // the supervisor. if let Some(path) = self.proxy_ca_bundle.as_deref() { if path.trim().is_empty() { return Err(crate::client::PodmanApiError::InvalidInput( @@ -502,6 +391,18 @@ impl PodmanComputeConfig { } /// Validate optional host gateway override. + /// Validate `AppArmor` syntax before contacting the runtime. Drivers check + /// runtime availability after querying their backend. + pub fn validate_app_armor_profile(&self) -> Result<(), crate::client::PodmanApiError> { + if matches!(self.app_armor_profile, Some(AppArmorProfile::Localhost(ref name)) if name.is_empty()) + { + return Err(crate::client::PodmanApiError::InvalidInput( + "app_armor_profile Localhost profile must not be empty".to_string(), + )); + } + Ok(()) + } + pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); if trimmed.is_empty() { @@ -545,10 +446,11 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + app_armor_profile: Some(AppArmorProfile::Unconfined), + health_check_interval_secs: None, https_proxy: None, no_proxy: None, proxy_auth_file: None, @@ -567,7 +469,7 @@ impl std::fmt::Debug for PodmanComputeConfig { f.debug_struct("PodmanComputeConfig") .field("socket_path", &self.socket_path) .field("default_image", &self.default_image) - .field("image_pull_policy", &self.image_pull_policy.as_str()) + .field("image_pull_policy", &self.image_pull_policy) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) .field("ssh_socket_path", &self.ssh_socket_path) @@ -584,6 +486,7 @@ impl std::fmt::Debug for PodmanComputeConfig { "provider_spiffe_workload_api_socket", &self.provider_spiffe_workload_api_socket, ) + .field("app_armor_profile", &self.app_armor_profile) .field( "health_check_interval_secs", &self.health_check_interval_secs, @@ -619,30 +522,19 @@ mod tests { } #[test] - fn config_accepts_legacy_sandbox_ssh_socket_path_alias() { - let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ - "sandbox_ssh_socket_path": "/run/test.sock" - })) - .unwrap(); - assert_eq!(config.ssh_socket_path, "/run/test.sock"); - } - - #[test] - fn config_rejects_canonical_and_legacy_ssh_socket_path_names_together() { + fn config_rejects_legacy_sandbox_ssh_socket_path() { let error = serde_json::from_value::(serde_json::json!({ - "ssh_socket_path": "/run/canonical.sock", - "sandbox_ssh_socket_path": "/run/legacy.sock" + "sandbox_ssh_socket_path": "/run/test.sock" })) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("legacy sandbox_ssh_socket_path must be rejected"); + assert!(error.to_string().contains("sandbox_ssh_socket_path")); } #[test] - fn default_config_sets_health_check_interval() { - let cfg = PodmanComputeConfig::default(); + fn default_config_disables_health_checks() { assert_eq!( - cfg.health_check_interval_secs, - DEFAULT_HEALTH_CHECK_INTERVAL_SECS + PodmanComputeConfig::default().health_check_interval_secs, + None ); } @@ -653,11 +545,10 @@ mod tests { } #[test] - fn default_config_sets_driver_owned_pids_limit() { + fn default_config_uses_runtime_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, DEFAULT_SANDBOX_PIDS_LIMIT); + assert_eq!(cfg.sandbox_pids_limit, None); assert!(!cfg.enable_bind_mounts); - assert!(cfg.validate_runtime_limits().is_ok()); } #[test] @@ -687,13 +578,28 @@ mod tests { } #[test] - fn runtime_limit_validation_rejects_negative_pids_limit() { - let cfg = PodmanComputeConfig { - sandbox_pids_limit: -1, - ..PodmanComputeConfig::default() - }; - let err = cfg.validate_runtime_limits().unwrap_err(); - assert!(err.to_string().contains("sandbox_pids_limit")); + fn runtime_limit_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = negative.validate_runtime_limits().unwrap_err(); + assert!(error.to_string().contains("must be positive")); + } + + #[test] + fn health_check_interval_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "health_check_interval_secs": 0 + })) + .expect_err("zero health-check interval must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); } // ── Proxy config validation ─────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1743015e26..1d0fce6f2a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,11 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + security_opt: Vec, image_pull_policy: String, - healthconfig: HealthConfig, + #[serde(skip_serializing_if = "Option::is_none")] + healthconfig: Option, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -652,14 +655,10 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: podman_pids_limit(config.sandbox_pids_limit), + pids_limit: config.sandbox_pids_limit.map(std::num::NonZeroI64::get), } } -fn podman_pids_limit(value: i64) -> Option { - if value > 0 { Some(value) } else { None } -} - pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -1175,8 +1174,14 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), + security_opt: config + .app_armor_profile + .as_ref() + .and_then(openshell_core::AppArmorProfile::oci_security_opt) + .into_iter() + .collect(), image_pull_policy: "never".to_string(), - healthconfig: HealthConfig { + healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ "CMD-SHELL".into(), format!( @@ -1185,11 +1190,11 @@ pub fn build_container_spec_for_image( openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: config.health_check_interval_secs * 1_000_000_000, + interval: interval_secs.get() * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }, + }), resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1561,7 +1566,8 @@ mod tests { }), ..Default::default() }); - let config = test_config(); + let mut config = test_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); let spec = build_container_spec(&sandbox, &config); assert_eq!( @@ -1572,17 +1578,14 @@ mod tests { spec["resource_limits"]["memory"]["limit"].as_u64(), Some(2 * 1024 * 1024 * 1024) ); - assert_eq!( - spec["resource_limits"]["PidsLimit"].as_i64(), - Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) - ); + assert_eq!(spec["resource_limits"]["PidsLimit"].as_i64(), Some(2048)); } #[test] fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = 0; + config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); assert!(spec["resource_limits"].get("PidsLimit").is_none()); @@ -1961,7 +1964,8 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let config = test_config(); + let mut config = test_config(); + config.health_check_interval_secs = std::num::NonZeroU64::new(10); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -1977,11 +1981,18 @@ mod tests { ); } + #[test] + fn container_spec_omits_healthcheck_when_disabled() { + let sandbox = test_sandbox("test-id", "test-name"); + let spec = build_container_spec(&sandbox, &test_config()); + assert!(spec.get("healthconfig").is_none()); + } + #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = 30; + config.health_check_interval_secs = std::num::NonZeroU64::new(30); let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 16c5780780..593748920d 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -4,7 +4,7 @@ //! Podman compute driver. use crate::client::{ContainerListEntry, PodmanApiError, PodmanClient, VolumeInspect}; -use crate::config::PodmanComputeConfig; +use crate::config::{PodmanComputeConfig, podman_image_pull_policy}; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ self, LifecycleEventFences, WatchStream, driver_sandbox_from_inspect, @@ -13,8 +13,9 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + GatewayCallbackTopology, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -373,6 +374,22 @@ impl PodmanComputeDriver { config.validate_runtime_limits()?; config.validate_host_gateway_ip()?; config.validate_proxy_config()?; + config.validate_app_armor_profile()?; + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_deref() { + let raw = socket.to_str().ok_or_else(|| { + PodmanApiError::InvalidInput( + "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), + ) + })?; + // Preserve Podman's established pass-through support for an + // explicitly configured container-reachable Workload API TCP + // endpoint. The Workload API client validates its endpoint grammar + // when it connects. + if !raw.starts_with("tcp:") { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(PodmanApiError::InvalidInput)?; + } + } config.canonicalize_userns()?; config.validate_userns_mappings()?; @@ -411,11 +428,25 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } + if matches!( + config.app_armor_profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ) && !info.host.security.apparmor_enabled + { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, rootless_network_cmd = %info.host.rootless_network_cmd, + apparmor_enabled = info.host.security.apparmor_enabled, "Connected to Podman" ); (info.host.security.rootless, info.host.rootless_network_cmd) @@ -437,14 +468,10 @@ impl PodmanComputeDriver { // Auto-detect the gRPC callback endpoint before deciding whether this // topology needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = format!( - "{scheme}://host.containers.internal:{}", - config.gateway_port + config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Podman, + config.gateway_port, + config.tls_enabled(), ); info!( grpc_endpoint = %config.grpc_endpoint, @@ -791,7 +818,7 @@ impl PodmanComputeDriver { .to_string(), )); } - let pull_policy = self.config.image_pull_policy.as_str(); + let pull_policy = podman_image_pull_policy(self.config.image_pull_policy); info!(image = %image, policy = %pull_policy, "Ensuring sandbox image"); self.client .pull_image(image, pull_policy) diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 8f223ed802..0f962fce5e 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -5,15 +5,13 @@ use clap::Parser; use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use tracing::info; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; +use openshell_core::{AppArmorProfile, ImagePullPolicy, VERSION}; +use openshell_driver_podman::config::{DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS}; use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; #[derive(Parser)] @@ -50,7 +48,7 @@ struct Args { #[arg( long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing + default_value_t = ImagePullPolicy::IfNotPresent )] sandbox_image_pull_policy: ImagePullPolicy, @@ -89,14 +87,19 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit + /// Container cgroup PID limit for sandbox containers. Omit to inherit /// Podman's runtime/default PID limit. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT")] + sandbox_pids_limit: Option, + + /// Health check interval in seconds. Omit it in gateway TOML to disable + /// health checks; the standalone driver keeps its prior 10-second default. #[arg( long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + env = "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + default_value = "10" )] - sandbox_pids_limit: i64, + health_check_interval_secs: Option, /// OCI image containing the openshell-sandbox supervisor binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] @@ -114,6 +117,14 @@ struct Args { #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] podman_tls_key: Option, + /// Host UNIX socket projected into supervisors for provider SPIFFE token exchange. + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. + #[arg(long, env = "OPENSHELL_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, /// in explicit `http://host:port` form (scheme and port required). /// Credentials must not be embedded in the URL; use @@ -202,7 +213,10 @@ async fn main() -> Result<()> { guest_tls_ca: args.podman_tls_ca, guest_tls_cert: args.podman_tls_cert, guest_tls_key: args.podman_tls_key, + provider_spiffe_workload_api_socket: args.provider_spiffe_workload_api_socket, + app_armor_profile: args.app_armor_profile, sandbox_pids_limit: args.sandbox_pids_limit, + health_check_interval_secs: args.health_check_interval_secs, https_proxy: args.sandbox_https_proxy, no_proxy: args.sandbox_no_proxy, proxy_auth_file: args.sandbox_proxy_auth_file, @@ -213,7 +227,6 @@ async fn main() -> Result<()> { uidmap: args.uidmap, gidmap: args.gidmap, enable_bind_mounts: args.enable_bind_mounts, - ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; @@ -309,4 +322,21 @@ mod tests { ); assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); } + + #[test] + fn standalone_defaults_preserve_health_checks_and_reject_zero_limits() { + let defaults = Args::try_parse_from(["openshell-driver-podman"]) + .expect("standalone driver defaults should parse"); + assert_eq!( + defaults.health_check_interval_secs.map(NonZeroU64::get), + Some(10) + ); + + for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { + let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); + assert!(result.is_err(), "zero must be rejected for {flag}"); + let error = result.err().expect("error was asserted above"); + assert!(error.to_string().contains("invalid value"), "flag: {flag}"); + } + } } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 1750b0390e..8646beb95b 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -113,7 +113,7 @@ codesign \ mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm cat > .cache/gateway-vm/gateway.toml <` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | +| `grpc_endpoint` | topology-derived | Optional override for the URL the sandbox guest dials to reach the gateway. The gateway derives `http(s)://host.openshell.internal:` when absent. Use `host.containers.internal`, `host.docker.internal`, or another routable host only for a non-standard topology. Loopback URLs are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | | `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | @@ -151,17 +152,21 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | -| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend (GPU sandboxes) has no such NAT and its nftables rules expose only the gateway port to the guest, so a gateway-host proxy URL is rejected at launch there; use an address routable from the guest's masqueraded egress. | +| `sandbox_uid` / `sandbox_gid` | image account or `1000` / UID | Explicit values override the image account. When omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`; persisted legacy identity is retained when recorded in sandbox state. | +| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend has no such NAT, so a gateway-host proxy URL is rejected before GPU sandbox launch; use an address routable from the guest's masqueraded egress. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | -| `proxy_auth_file` | unset | Gateway-host path to a `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox. | +| `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | | `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | | `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. | -| `proxy_ca_bundle` | unset | Gateway-host path to a PEM CA bundle trusted for an `https://` proxy and for certificates a TLS-intercepting proxy re-signs. | +| `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | -The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. +The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor through a protected per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. + +For gateway-managed VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and +`guest_tls_key` together under `[openshell.gateway]`; the gateway validates and +injects that bundle into only the selected local driver. The standalone +`openshell-driver-vm` CLI retains its `--guest-tls-*` inputs for independent +operation. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -282,8 +287,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical On Debian-family Linux amd64 and arm64 systems, `install.sh` installs the Debian package from the selected `OPENSHELL_VERSION` release tag. That package includes `openshell-gateway` and `openshell-driver-vm`, but leaves -`OPENSHELL_DRIVERS` unset so the gateway uses its normal runtime -auto-detection. Set `OPENSHELL_DRIVERS=vm` to force the VM driver. +`OPENSHELL_COMPUTE_DRIVER` unset so the gateway uses its normal runtime +auto-detection. Set `OPENSHELL_COMPUTE_DRIVER=vm` to force the VM driver. On RPM-family Linux x86_64 and aarch64 systems, `install.sh` installs the `openshell` and `openshell-gateway` RPM packages from the selected release tag. @@ -294,7 +299,7 @@ formula from the selected release in the `nvidia/openshell` Homebrew tap. Homebrew installs `openshell`, `openshell-gateway`, and `openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless +also leaves `OPENSHELL_COMPUTE_DRIVER` unset so driver choice remains automatic unless the user explicitly overrides it. ## TODOs diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 32d6ed1dff..b8001efb2c 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -90,7 +90,9 @@ sandbox_owner_from_passwd() { done < "$passwd_path" fi - printf '10001:10001\n' + # New images use the conventional sandbox UID. Existing images retain the + # sandbox account read above, including the legacy 10001:10001 identity. + printf '1000:1000\n' } source_overlay_env_if_present() { @@ -103,6 +105,8 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-1000}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-$sandbox_uid}" mkdir -p \ "$image_root/srv" \ @@ -119,14 +123,22 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - printf 'sandbox:x:10001:\n' >> "$image_root/etc/group" + # This is a newly prepared target image, so replace a baked-in legacy + # sandbox account with the identity selected by the driver. Persisted + # overlays do not take this path; setup_sandbox_workdir preserves their + # existing 10001:10001 account instead. + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" fi if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" fi - if ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - printf 'sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n' >> "$image_root/etc/passwd" + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" fi if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" @@ -136,7 +148,7 @@ ensure_target_runtime() { owner="$(sandbox_owner_for_root "$image_root")" if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then owner_normalized=1 - elif chown -R 10001:10001 "$image_root/sandbox" 2>/dev/null; then + elif chown -R 1000:1000 "$image_root/sandbox" 2>/dev/null; then owner_normalized=1 fi chmod 0755 "$image_root/sandbox" @@ -275,16 +287,14 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" \ - "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@" fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$supervisor" "$@" fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ - --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@" fi done @@ -617,10 +627,13 @@ setup_sandbox_workdir() { owner="$(sandbox_owner)" mkdir -p "$sandbox_dir" current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)" + if [ "$owner" = "10001:10001" ]; then + ts "preserving legacy sandbox ownership (10001:10001)" + fi if [ "$current_owner" != "$owner" ] \ || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 10001:10001 "$sandbox_dir" + chown -R 1000:1000 "$sandbox_dir" fi fi chmod 0755 "$sandbox_dir" @@ -897,12 +910,13 @@ if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then fi read_supervisor_extra_args +set -- --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then - exec_supervisor_in_newroot + exec_supervisor_in_newroot "$@" fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" +exec /opt/openshell/bin/openshell-sandbox "$@" } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 0f442f78f2..d827f5ec3d 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -30,6 +30,7 @@ use oci_client::manifest::{ }; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; +use openshell_core::UpstreamProxyConfig; use openshell_core::gpu::{ driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; @@ -170,8 +171,6 @@ const GUEST_INIT_DROPIN_MANIFEST: &str = /// Guest path of the root-only corporate proxy credential staged by the driver. const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; -/// Guest path of the corporate proxy CA bundle staged by the driver. -const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROXY_CA_PATH; /// Guest path of the driver-authored supervisor argument list. /// /// The counterpart of [`GUEST_INIT_DROPIN_MANIFEST`] for the supervisor's own @@ -236,8 +235,8 @@ enum GuestImagePayloadSource { } #[derive(Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct VmDriverConfig { - #[serde(alias = "openshell_endpoint")] pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, @@ -251,80 +250,27 @@ pub struct VmDriverConfig { pub guest_tls_ca: Option, pub guest_tls_cert: Option, pub guest_tls_key: Option, + /// Corporate forward proxy settings delivered to the guest init script. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + /// Guest-reachable SPIFFE Workload API TCP endpoint. A VM cannot safely + /// project a host UNIX socket; this must be a deliberately exposed TCP + /// listener and requires `provider_spiffe_allow_guest_tcp`. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for rootfs `/etc/passwd` entry. - /// When empty, defaults to 10001 (the legacy hardcoded value). + /// Resolved sandbox UID for newly prepared rootfs `/etc/passwd` entries. + /// When empty, new sandboxes use 1000. Existing rootfs and overlays retain + /// their recorded sandbox account for legacy 10001 compatibility. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. /// When empty, defaults to the resolved UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, - - /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) - /// passed to the in-guest supervisor. - /// - /// The supervisor chains policy-approved TLS tunnels through this proxy - /// with HTTP CONNECT instead of dialing destinations directly. This is an - /// operator-owned egress boundary: it travels on the supervisor's argv, - /// which sandbox spec/template environment and image `ENV` cannot - /// influence. A proxy on the gateway host's loopback is reachable from the - /// guest only through the gvproxy host alias - /// (`host.openshell.internal`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub https_proxy: Option, - - /// Comma-separated `NO_PROXY` list passed alongside the proxy URL. - /// - /// Matching destinations are dialed directly instead of through the - /// corporate proxy. This bypasses only the corporate proxy, never - /// `OpenShell` policy evaluation. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub no_proxy: Option, - - /// Path (on the gateway host) to a file containing the corporate proxy - /// credential in `user:pass` form. - /// - /// The driver validates it at sandbox-create time and stages it into the - /// per-sandbox overlay at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], root-only. - /// Credentials are never embedded in the proxy URL and never reach the - /// guest environment. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_auth_file: Option, - - /// Explicit acknowledgement that proxy credentials are sent in cleartext. - /// - /// `Proxy-Authorization: Basic` over the plain-TCP connection to an - /// `http://` proxy is recoverable by anyone on the network path, so - /// [`Self::proxy_auth_file`] requires this acknowledgement. An `https://` - /// proxy carries the credential inside the verified TLS session and does - /// not need it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_auth_allow_insecure: Option, - - /// Send the destination hostname in CONNECT requests instead of a - /// validated IP. - /// - /// The default binds the tunnel to an address that passed the sandbox's - /// SSRF and `allowed_ips` validation. Set this only when the proxy's ACLs - /// filter on hostnames and reject IP CONNECT targets: the proxy then - /// resolves the name itself and its own ACLs become the effective egress - /// control for proxied TLS. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_connect_by_hostname: Option, - - /// Path (on the gateway host) to a PEM CA bundle trusted for the - /// corporate proxy. - /// - /// The driver stages it into the per-sandbox overlay at - /// [`GUEST_PROXY_CA_PATH`] and passes that path via - /// `--upstream-proxy-ca-bundle`. It is trusted both for the handshake - /// with an `https://` proxy and for server certificates re-signed by a - /// TLS-intercepting proxy. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_ca_bundle: Option, } /// Redacting `Debug` so a proxy URL or credential path never reaches a log. @@ -352,18 +298,44 @@ impl std::fmt::Debug for VmDriverConfig { .field("gpu_vcpus", &self.gpu_vcpus) .field("sandbox_uid", &self.sandbox_uid) .field("sandbox_gid", &self.sandbox_gid) - .field("https_proxy", &self.https_proxy.is_some()) - .field("no_proxy", &self.no_proxy) - .field("proxy_auth_file", &self.proxy_auth_file.is_some()) - .field("proxy_auth_allow_insecure", &self.proxy_auth_allow_insecure) - .field("proxy_connect_by_hostname", &self.proxy_connect_by_hostname) - .field("proxy_ca_bundle", &self.proxy_ca_bundle) + .field( + "upstream_proxy_configured", + &self.upstream_proxy.https_proxy.is_some(), + ) + .field( + "no_proxy_configured", + &self.upstream_proxy.no_proxy.is_some(), + ) + .field( + "proxy_auth_file_configured", + &self.upstream_proxy.proxy_auth_file.is_some(), + ) + .field( + "proxy_auth_allow_insecure", + &self.upstream_proxy.proxy_auth_allow_insecure, + ) + .field( + "proxy_connect_by_hostname", + &self.upstream_proxy.proxy_connect_by_hostname, + ) + .field( + "provider_spiffe_workload_api_tcp_endpoint_configured", + &self.provider_spiffe_workload_api_tcp_endpoint.is_some(), + ) + .field( + "provider_spiffe_allow_guest_tcp", + &self.provider_spiffe_allow_guest_tcp, + ) .finish() } } -/// Default sandbox UID used by the VM driver when no config value is set. -pub const DEFAULT_SANDBOX_UID: u32 = 10001; +/// Default sandbox UID used when preparing new VM rootfs images. +/// +/// The guest-init script detects an existing `sandbox` account and preserves +/// its UID/GID, so persisted rootfs and overlays prepared with legacy UID 10001 +/// continue to start without an ownership migration. +pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { fn default() -> Self { @@ -381,17 +353,14 @@ impl Default for VmDriverConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, sandbox_uid: None, sandbox_gid: None, - https_proxy: None, - no_proxy: None, - proxy_auth_file: None, - proxy_auth_allow_insecure: None, - proxy_connect_by_hostname: None, - proxy_ca_bundle: None, } } } @@ -407,6 +376,19 @@ impl VmDriverConfig { self.sandbox_gid.unwrap_or(resolved_uid) } + pub fn validate_runtime_security_config(&self) -> Result<(), String> { + self.upstream_proxy.validate()?; + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + )?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err("provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured".to_string()); + } + Ok(()) + } + pub fn validate_sandbox_identity(&self) -> Result<(), String> { let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; if let Some(uid) = self.sandbox_uid @@ -430,29 +412,6 @@ impl VmDriverConfig { Ok(()) } - /// Validate the operator's corporate upstream-proxy settings, fail-closed. - /// - /// Delegates to the validator shared with the Podman and Kubernetes - /// drivers and with the in-guest supervisor, so a value accepted here is - /// never rejected inside the guest — and no misconfiguration can silently - /// degrade to a direct dial. - /// - /// # Errors - /// - /// Returns a message naming the offending key. - pub fn validate_proxy_config(&self) -> Result<(), String> { - openshell_core::driver_utils::validate_upstream_proxy_settings( - &openshell_core::driver_utils::UpstreamProxySettings { - url: self.https_proxy.as_deref(), - no_proxy: self.no_proxy.as_deref(), - auth_file: self.proxy_auth_file.as_deref(), - auth_allow_insecure: self.proxy_auth_allow_insecure, - connect_by_hostname: self.proxy_connect_by_hostname, - ca_bundle: self.proxy_ca_bundle.as_deref(), - }, - ) - } - fn requires_tls_materials(&self) -> bool { self.grpc_endpoint.starts_with("https://") } @@ -593,7 +552,7 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; - config.validate_proxy_config()?; + config.validate_runtime_security_config()?; if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } @@ -1877,7 +1836,7 @@ impl VmDriver { // compare against is this sandbox's own TAP host address. Fail the // create with the reason rather than boot a sandbox whose // policy-approved CONNECTs all time out against an unreachable proxy. - if let Some(url) = self.config.https_proxy.as_deref() + if let Some(url) = self.config.upstream_proxy.https_proxy.as_deref() && proxy_url_targets_gateway_host(url, plan.host_ip.as_deref()) { let tap_host = plan.host_ip.as_deref().unwrap_or("the TAP host address"); @@ -2228,6 +2187,7 @@ impl VmDriver { None => None, }; let sandbox_token = sandbox_token.map(str::to_string); + let proxy_auth = self.read_proxy_auth_credential().await?; let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2257,6 +2217,7 @@ impl VmDriver { &overlay_disk, tls_materials.as_ref(), sandbox_token.as_deref(), + proxy_auth.as_deref(), preparation, overlay_size_bytes, ) @@ -2266,6 +2227,26 @@ impl VmDriver { span_status.finish(result) } + async fn read_proxy_auth_credential(&self) -> Result, String> { + let Some(path) = self.config.upstream_proxy.proxy_auth_file.as_ref() else { + return Ok(None); + }; + let path = path.clone(); + Ok(Some( + tokio::task::spawn_blocking(move || { + let path = path + .to_str() + .ok_or_else(|| "proxy_auth_file must be valid UTF-8".to_string())?; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file(path)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map(str::to_owned) + .map_err(|error| format!("proxy_auth_file is invalid: {error}")) + }) + .await + .map_err(|error| format!("proxy_auth_file read task failed: {error}"))??, + )) + } + fn resolved_sandbox_image(&self, sandbox: &Sandbox) -> Option { requested_sandbox_image(sandbox) .map(ToOwned::to_owned) @@ -2954,6 +2935,7 @@ impl VmDriver { Ok(()) } + #[allow(clippy::similar_names)] async fn run_image_prep_vm( &self, bootstrap_root_disk: &Path, @@ -2982,6 +2964,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); + let resolved_uid = self.config.resolve_sandbox_uid(); + let resolved_gid = self.config.resolve_sandbox_gid(resolved_uid); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={resolved_uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={resolved_gid}")); let mut child = command .spawn() @@ -4902,6 +4892,39 @@ fn build_guest_environment( GUEST_TLS_KEY_PATH.to_string(), ); } + if let Some(url) = config.upstream_proxy.https_proxy.as_ref() { + environment.insert("OPENSHELL_VM_UPSTREAM_PROXY".to_string(), url.clone()); + } + if let Some(no_proxy) = config.upstream_proxy.no_proxy.as_ref() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_NO_PROXY".to_string(), + no_proxy.clone(), + ); + } + if config.upstream_proxy.proxy_auth_file.is_some() { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE".to_string(), + GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string(), + ); + } + if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE".to_string(), + "true".to_string(), + ); + } + if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { + environment.insert( + "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME".to_string(), + "true".to_string(), + ); + } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + endpoint.clone(), + ); + } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), @@ -5363,6 +5386,7 @@ fn create_sandbox_overlay_image_from_template( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, ) -> Result<(), String> { clone_or_copy_sparse_file(template_path, overlay_disk)?; if let Some(tls) = tls_materials { @@ -5371,6 +5395,9 @@ fn create_sandbox_overlay_image_from_template( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } Ok(()) } @@ -5379,6 +5406,7 @@ fn prepare_sandbox_overlay_image( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { @@ -5391,6 +5419,9 @@ fn prepare_sandbox_overlay_image( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5422,6 +5453,7 @@ fn prepare_sandbox_overlay_image( overlay_disk, tls_materials, sandbox_token, + proxy_auth, ) } @@ -5450,6 +5482,12 @@ fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), St set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) } +fn inject_guest_proxy_auth(overlay_disk: &Path, credential: &str) -> Result<(), String> { + let path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH); + write_rootfs_image_file(overlay_disk, &path, format!("{credential}\n").as_bytes())?; + set_rootfs_image_file_mode(overlay_disk, &path, 0o600) +} + #[allow(clippy::result_large_err)] #[tracing::instrument( name = "vm.prepare_guest", @@ -5505,15 +5543,15 @@ fn inject_guest_init_dropins( /// the supervisor reads the credential from that file. fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { let mut args = Vec::new(); - if let Some(url) = &config.https_proxy { + if let Some(url) = &config.upstream_proxy.https_proxy { args.push("--upstream-proxy".to_string()); args.push(url.clone()); } - if let Some(list) = &config.no_proxy { + if let Some(list) = &config.upstream_proxy.no_proxy { args.push("--upstream-no-proxy".to_string()); args.push(list.clone()); } - if config.proxy_auth_file.is_some() { + if config.upstream_proxy.proxy_auth_file.is_some() { args.push("--upstream-proxy-auth-file".to_string()); // The guest path, never the gateway-host path the operator configured. args.push(GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string()); @@ -5521,18 +5559,14 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { // Config validation guarantees the acknowledgement is `true` whenever an // auth file is configured against an http:// proxy; the supervisor // independently refuses credentials without it. - if config.proxy_auth_allow_insecure == Some(true) { + if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { args.push("--upstream-proxy-auth-allow-insecure".to_string()); } // Absent means the default validated-IP CONNECT binding; only the // explicit hostname opt-in is passed through. - if config.proxy_connect_by_hostname == Some(true) { + if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { args.push("--upstream-proxy-connect-by-hostname".to_string()); } - if config.proxy_ca_bundle.is_some() { - args.push("--upstream-proxy-ca-bundle".to_string()); - args.push(GUEST_PROXY_CA_PATH.to_string()); - } args } @@ -5571,53 +5605,31 @@ fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> { /// Uses the validators shared with the supervisor, so a credential accepted /// here is never rejected inside the guest. The error never carries the file /// contents. -async fn read_sandbox_proxy_credential(path: &str) -> Result { - let path_owned = path.to_string(); +async fn read_sandbox_proxy_credential(path: &Path) -> Result { + let path_owned = path.to_path_buf(); + let display_path = path.display().to_string(); let raw = tokio::task::spawn_blocking(move || { openshell_core::driver_utils::read_upstream_proxy_credential_file(&path_owned) }) .await .map_err(|err| Status::internal(format!("proxy_auth_file read task failed: {err}")))? .map_err(Status::invalid_argument)?; - let credential = openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) - .map_err(|err| Status::invalid_argument(format!("proxy_auth_file '{path}': {err}")))?; + let credential = + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw).map_err(|err| { + Status::invalid_argument(format!("proxy_auth_file '{display_path}': {err}")) + })?; Ok(credential.to_string()) } -/// Read and validate the corporate proxy CA bundle from the gateway host. -/// -/// Uses the reader shared with the supervisor, so the bundle is bounded and -/// non-regular files are rejected (an operator path such as `/dev/zero` can -/// otherwise exhaust driver memory), and a bundle accepted here contributes at -/// least one trust anchor rustls accepts rather than merely looking like PEM. -/// Checked here rather than only in the guest so the operator gets an error -/// attributable to `proxy_ca_bundle` instead of an opaque supervisor startup -/// failure inside every sandbox. The error never carries the file contents. -async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { - let path_owned = path.to_string(); - let pem = tokio::task::spawn_blocking(move || { - openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file( - &path_owned, - "proxy_ca_bundle", - ) - }) - .await - .map_err(|err| Status::internal(format!("proxy_ca_bundle read task failed: {err}")))? - .map_err(Status::invalid_argument)?; - Ok(pem.into_bytes()) -} - /// Stage the corporate upstream-proxy configuration into the guest overlay. /// -/// Writes three files into the overlay upperdir the driver owns: +/// Writes two files into the overlay upperdir the driver owns: /// /// * the credential at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], mode `0600`; -/// * the CA bundle at [`GUEST_PROXY_CA_PATH`], mode `0644` (a CA certificate -/// is not secret); /// * the supervisor argument list at [`GUEST_SUPERVISOR_ARGS_PATH`], mode /// `0644`. /// -/// All three are written on every launch, empty when the corresponding +/// Both are written on every launch, empty when the corresponding /// setting is absent. Writing rather than skipping is what makes the channel /// unforgeable: the upperdir copy always shadows the read-only image layer, so /// a sandbox image cannot supply its own arguments or credential by baking a @@ -5638,7 +5650,7 @@ async fn inject_guest_upstream_proxy( // the operator removed a setting clears material a previous launch staged // into a preserved overlay, and shadows anything an image baked at these // paths, so a staged file is only ever the one this launch produced. - let credential = match config.proxy_auth_file.as_deref() { + let credential = match config.upstream_proxy.proxy_auth_file.as_deref() { Some(path) => format!("{}\n", read_sandbox_proxy_credential(path).await?).into_bytes(), None => Vec::new(), }; @@ -5648,16 +5660,6 @@ async fn inject_guest_upstream_proxy( set_rootfs_image_file_mode(overlay_disk, &credential_path, 0o600) .map_err(|err| Status::internal(format!("set VM guest proxy credential mode: {err}")))?; - let ca_bundle = match config.proxy_ca_bundle.as_deref() { - Some(path) => read_sandbox_proxy_ca_bundle(path).await?, - None => Vec::new(), - }; - let ca_path = overlay_upper_path(GUEST_PROXY_CA_PATH); - write_rootfs_image_file(overlay_disk, &ca_path, &ca_bundle) - .map_err(|err| Status::internal(format!("write VM guest proxy CA bundle: {err}")))?; - set_rootfs_image_file_mode(overlay_disk, &ca_path, 0o644) - .map_err(|err| Status::internal(format!("set VM guest proxy CA bundle mode: {err}")))?; - let args = upstream_proxy_cli_args(config); validate_guest_supervisor_args(&args).map_err(Status::failed_precondition)?; let guest_path = overlay_upper_path(GUEST_SUPERVISOR_ARGS_PATH); @@ -6212,7 +6214,7 @@ mod tests { } #[test] - fn vm_config_accepts_legacy_openshell_endpoint_alias() { + fn vm_config_rejects_legacy_openshell_endpoint() { let config = VmDriverConfig::default(); let mut serialized = serde_json::to_value(config).unwrap(); let fields = serialized.as_object_mut().unwrap(); @@ -6222,25 +6224,9 @@ mod tests { serde_json::json!("http://127.0.0.1:8080"), ); - let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); - assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); - } - - #[test] - fn vm_config_rejects_canonical_and_legacy_endpoint_names_together() { - let config = VmDriverConfig { - grpc_endpoint: "http://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let mut serialized = serde_json::to_value(config).unwrap(); - serialized.as_object_mut().unwrap().insert( - "openshell_endpoint".to_string(), - serde_json::json!("http://127.0.0.1:9090"), - ); - let error = serde_json::from_value::(serialized) - .expect_err("canonical and legacy names must not both be accepted"); - assert!(error.to_string().contains("duplicate field")); + .expect_err("legacy openshell_endpoint must be rejected"); + assert!(!error.to_string().is_empty()); } struct TestTracing { @@ -7434,6 +7420,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -7457,6 +7444,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -7709,6 +7697,16 @@ mod tests { ))); } + #[test] + fn new_vm_sandbox_identity_defaults_to_1000() { + let config = VmDriverConfig::default(); + assert_eq!(config.resolve_sandbox_uid(), 1000); + assert_eq!( + config.resolve_sandbox_gid(config.resolve_sandbox_uid()), + 1000 + ); + } + #[test] fn validate_sandbox_identity_accepts_non_root_system_ids() { let config = VmDriverConfig { @@ -8130,6 +8128,63 @@ mod tests { ); } + #[test] + fn vm_proxy_and_spiffe_config_require_explicit_safe_acknowledgements() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: None, + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: false, + ..Default::default() + }; + let error = config.validate_runtime_security_config().unwrap_err(); + assert!(error.contains("provider_spiffe_allow_guest_tcp")); + + let config = VmDriverConfig { + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + assert!(config.validate_runtime_security_config().is_ok()); + } + + #[test] + fn build_guest_environment_projects_operator_proxy_and_spiffe_endpoint() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let sandbox = Sandbox { + id: "vm-spiffe".to_string(), + name: "vm-spiffe".to_string(), + ..Default::default() + }; + let env = build_guest_environment(&sandbox, &config, None); + assert!( + env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY=https://proxy.example:8443".to_string()) + ); + assert!(env.contains(&format!( + "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE={GUEST_UPSTREAM_PROXY_AUTH_PATH}" + ))); + assert!(env.contains( + &"OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=tcp:192.0.2.10:8081".to_string() + )); + assert!(!env.iter().any(|value| value.contains("user:pass"))); + } + #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { @@ -8712,7 +8767,7 @@ mod tests { fn test_driver_with_proxy(https_proxy: &str) -> VmDriver { let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - driver.config.https_proxy = Some(https_proxy.to_string()); + driver.config.upstream_proxy.https_proxy = Some(https_proxy.to_string()); driver } @@ -9047,17 +9102,15 @@ mod tests { } /// A driver config carrying only corporate proxy settings. - fn proxy_config( - https_proxy: Option<&str>, - auth_file: Option<&str>, - ca_bundle: Option<&str>, - ) -> VmDriverConfig { + fn proxy_config(https_proxy: Option<&str>, auth_file: Option<&str>) -> VmDriverConfig { VmDriverConfig { grpc_endpoint: "http://127.0.0.1:8080".to_string(), - https_proxy: https_proxy.map(ToString::to_string), - proxy_auth_file: auth_file.map(ToString::to_string), - proxy_auth_allow_insecure: auth_file.map(|_| true), - proxy_ca_bundle: ca_bundle.map(ToString::to_string), + upstream_proxy: UpstreamProxyConfig { + https_proxy: https_proxy.map(ToString::to_string), + proxy_auth_file: auth_file.map(PathBuf::from), + proxy_auth_allow_insecure: auth_file.map(|_| true), + ..UpstreamProxyConfig::default() + }, ..Default::default() } } @@ -9071,7 +9124,6 @@ mod tests { proxy_config( Some("http://user:secret@proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ) ); assert!( @@ -9083,14 +9135,10 @@ mod tests { "the credential path must be logged as presence only: {rendered}" ); assert!( - rendered.contains("https_proxy: true") && rendered.contains("proxy_auth_file: true"), + rendered.contains("upstream_proxy_configured: true") + && rendered.contains("proxy_auth_file_configured: true"), "presence of each must still be visible for debugging: {rendered}" ); - // A CA path is not sensitive and stays readable. - assert!( - rendered.contains("corp-ca.pem"), - "the CA bundle path is not a secret and should stay legible: {rendered}" - ); } #[test] @@ -9100,11 +9148,7 @@ mod tests { // credential removable with the sandbox (remove_sandbox_state_dir // deletes the whole directory) and unforgeable by the guest image // (the upperdir shadows the read-only image layer). - for guest_path in [ - GUEST_UPSTREAM_PROXY_AUTH_PATH, - GUEST_PROXY_CA_PATH, - GUEST_SUPERVISOR_ARGS_PATH, - ] { + for guest_path in [GUEST_UPSTREAM_PROXY_AUTH_PATH, GUEST_SUPERVISOR_ARGS_PATH] { assert!( guest_path.starts_with("/opt/openshell/"), "{guest_path} must be under the reserved guest control root" @@ -9130,22 +9174,16 @@ mod tests { let config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ); let args = upstream_proxy_cli_args(&config); - // The credential and CA live at fixed guest paths; the gateway-host - // paths the operator configured must never reach the guest argv. + // The credential lives at a fixed guest path; the gateway-host path + // the operator configured must never reach the guest argv. let auth = args .iter() .position(|arg| arg == "--upstream-proxy-auth-file") .map(|i| args[i + 1].as_str()); assert_eq!(auth, Some(GUEST_UPSTREAM_PROXY_AUTH_PATH)); - let ca = args - .iter() - .position(|arg| arg == "--upstream-proxy-ca-bundle") - .map(|i| args[i + 1].as_str()); - assert_eq!(ca, Some(GUEST_PROXY_CA_PATH)); assert!( !args .iter() @@ -9156,8 +9194,8 @@ mod tests { #[test] fn upstream_proxy_args_pass_only_explicit_opt_ins() { - let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None, None); - config.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string()); + let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None); + config.upstream_proxy.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string()); let args = upstream_proxy_cli_args(&config); assert_eq!( args, @@ -9171,13 +9209,13 @@ mod tests { // `Some(false)` must not be passed as the presence flag it is on the // supervisor side. - config.proxy_connect_by_hostname = Some(false); + config.upstream_proxy.proxy_connect_by_hostname = Some(false); assert!( !upstream_proxy_cli_args(&config) .iter() .any(|arg| arg == "--upstream-proxy-connect-by-hostname") ); - config.proxy_connect_by_hostname = Some(true); + config.upstream_proxy.proxy_connect_by_hostname = Some(true); assert!( upstream_proxy_cli_args(&config) .iter() @@ -9217,17 +9255,20 @@ mod tests { #[test] fn proxy_config_validation_rejects_settings_without_a_proxy_url() { let config = VmDriverConfig { - no_proxy: Some("10.0.0.0/8".to_string()), + upstream_proxy: UpstreamProxyConfig { + no_proxy: Some("10.0.0.0/8".to_string()), + ..UpstreamProxyConfig::default() + }, ..Default::default() }; let err = config - .validate_proxy_config() + .validate_runtime_security_config() .expect_err("a bypass list without a proxy would hide a fail-open state"); assert!(err.contains("no_proxy"), "{err}"); - let config = proxy_config(Some("http://proxy.corp.test:3128"), None, None); + let config = proxy_config(Some("http://proxy.corp.test:3128"), None); config - .validate_proxy_config() + .validate_runtime_security_config() .expect("a lone proxy URL is a complete configuration"); } @@ -9236,78 +9277,14 @@ mod tests { let mut config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - None, ); - config.proxy_auth_allow_insecure = None; + config.upstream_proxy.proxy_auth_allow_insecure = None; let err = config - .validate_proxy_config() + .validate_runtime_security_config() .expect_err("Basic auth to an http:// proxy is cleartext on the wire"); assert!(err.contains("proxy_auth_allow_insecure"), "{err}"); } - #[tokio::test] - async fn proxy_ca_bundle_without_a_certificate_fails_the_sandbox() { - let dir = std::env::temp_dir().join(format!("openshell-vm-ca-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("not-a-ca.pem"); - std::fs::write(&path, b"this is not a certificate\n").unwrap(); - - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("a certificate-free bundle must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("no PEM certificate"), "{err}"); - - std::fs::write(&path, b"").unwrap(); - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("an empty bundle must fail closed"); - assert!(err.message().contains("no PEM certificate"), "{err}"); - - // PEM framing that base64-decodes but is not X.509 DER: accepted by - // `rustls_pemfile` alone, contributes zero trust anchors at runtime, - // and so would make every guest supervisor fail after boot. - std::fs::write( - &path, - b"-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n", - ) - .unwrap(); - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("a bundle with invalid DER must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("no usable trust anchors"), "{err}"); - - let err = read_sandbox_proxy_ca_bundle(dir.join("missing.pem").to_str().unwrap()) - .await - .expect_err("an unreadable bundle must fail closed"); - assert!(err.message().contains("could not be read"), "{err}"); - - // A special file must be rejected on its type, not read: an - // unbounded read of /dev/zero would exhaust driver memory. - #[cfg(unix)] - { - let err = read_sandbox_proxy_ca_bundle("/dev/zero") - .await - .expect_err("a non-regular bundle path must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("not a regular file"), "{err}"); - } - - // Oversized regular file: rejected on the stat'd length, again - // without reading it whole. - let oversized = dir.join("oversized.pem"); - let bound = openshell_core::driver_utils::MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES; - std::fs::write(&oversized, vec![b'x'; usize::try_from(bound).unwrap() + 1]).unwrap(); - let err = read_sandbox_proxy_ca_bundle(oversized.to_str().unwrap()) - .await - .expect_err("an oversized bundle must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("exceeds"), "{err}"); - - std::fs::remove_dir_all(&dir).unwrap(); - } - #[test] fn qemu_backend_rejects_a_gateway_host_proxy() { // gvproxy's host-loopback NAT has no QEMU/TAP equivalent, so a proxy @@ -9441,7 +9418,6 @@ mod tests { let config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ); let sandbox = Sandbox { id: "sb-proxy".to_string(), @@ -9487,7 +9463,6 @@ mod tests { "proxy_auth_file", "proxy_auth_allow_insecure", "proxy_connect_by_hostname", - "proxy_ca_bundle", ] { let template = SandboxTemplate { driver_config: Some(Struct { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index c875768bfb..b421780668 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -91,11 +91,7 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg( - long = "grpc-endpoint", - alias = "openshell-endpoint", - env = "OPENSHELL_GRPC_ENDPOINT" - )] + #[arg(long = "grpc-endpoint", env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] @@ -120,6 +116,47 @@ struct Args { #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] guest_tls_key: Option, + /// Corporate forward proxy for supervisor TLS egress. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY")] + upstream_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_NO_PROXY")] + upstream_no_proxy: Option, + + /// Root-owned gateway-host file containing `user:pass` proxy credentials. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE")] + upstream_proxy_auth_file: Option, + + /// Explicitly acknowledge cleartext Basic authentication to an http proxy. + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE", + default_value_t = false + )] + upstream_proxy_auth_allow_insecure: bool, + + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", + default_value_t = false + )] + upstream_proxy_connect_by_hostname: bool, + + /// Guest-reachable SPIFFE Workload API endpoint (`tcp:IP:port`). + #[arg( + long = "provider-spiffe-workload-api-tcp-endpoint", + env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT" + )] + provider_spiffe_workload_api_tcp_endpoint: Option, + + /// Explicit acknowledgement that the configured Workload API listener is exposed to VM guests. + #[arg( + long, + env = "OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP", + default_value_t = false + )] + provider_spiffe_allow_guest_tcp: bool, + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] krun_log_level: u32, @@ -254,6 +291,17 @@ async fn main() -> Result<()> { guest_tls_ca: args.guest_tls_ca.clone(), guest_tls_cert: args.guest_tls_cert.clone(), guest_tls_key: args.guest_tls_key.clone(), + upstream_proxy: openshell_core::UpstreamProxyConfig { + https_proxy: args.upstream_proxy.clone(), + no_proxy: args.upstream_no_proxy.clone(), + proxy_auth_file: args.upstream_proxy_auth_file.clone(), + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname.then_some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: args + .provider_spiffe_workload_api_tcp_endpoint + .clone(), + provider_spiffe_allow_guest_tcp: args.provider_spiffe_allow_guest_tcp, gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, @@ -780,14 +828,14 @@ mod tests { } #[test] - fn accepts_legacy_openshell_endpoint_flag_alias() { - let args = Args::try_parse_from([ + fn rejects_legacy_openshell_endpoint_flag() { + let error = Args::try_parse_from([ "openshell-driver-vm", "--openshell-endpoint", "http://127.0.0.1:8080", ]) - .unwrap(); - assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + .expect_err("legacy --openshell-endpoint must be rejected"); + assert!(error.to_string().contains("--openshell-endpoint")); } #[test] diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..81f68f6c44 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -822,23 +822,32 @@ fn ensure_line_in_file( line: &str, exists: impl Fn(&str) -> bool, ) -> Result<(), String> { - let mut contents = if path.exists() { + let contents = if path.exists() { fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? } else { String::new() }; - if contents.lines().any(exists) { - return Ok(()); + let mut replaced = false; + let mut updated = String::new(); + for existing in contents.lines() { + if exists(existing) { + if !replaced { + updated.push_str(line); + updated.push('\n'); + replaced = true; + } + } else { + updated.push_str(existing); + updated.push('\n'); + } } - - if !contents.is_empty() && !contents.ends_with('\n') { - contents.push('\n'); + if !replaced { + updated.push_str(line); + updated.push('\n'); } - contents.push_str(line); - contents.push('\n'); - fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) + fs::write(path, updated).map_err(|e| format!("write {}: {e}", path.display())) } fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { @@ -959,10 +968,10 @@ mod tests { write_fake_runtime_binaries(&rootfs); fs::write( rootfs.join("etc/passwd"), - "root:x:0:0:root:/root:/bin/bash\n", + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n", ) .expect("write passwd"); - fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:10001:\n").expect("write group"); fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); fs::create_dir_all(rootfs.join("bin")).expect("create bin"); fs::create_dir_all(rootfs.join("sbin")).expect("create sbin"); @@ -1002,6 +1011,12 @@ mod tests { .expect("read group") .contains(&format!("sandbox:x:{uid}:")) ); + assert!( + !fs::read_to_string(rootfs.join("etc/passwd")) + .expect("read passwd") + .contains("sandbox:x:10001:"), + "newly prepared rootfs must replace the legacy sandbox account" + ); assert_eq!( fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), "127.0.0.1 localhost\n" diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 0065b62e87..f091960a8e 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -117,16 +117,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("kubernetes")) .without_mtls_user_auth() .with_in_process_tracing(openshell_driver_kubernetes::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ]) }), ComputeDriverRegistration::new( "podman", @@ -139,14 +129,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("podman")) .with_local_singleplayer() .with_in_process_tracing(openshell_driver_podman::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new( "docker", @@ -159,26 +141,11 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("docker")) .with_local_singleplayer() .with_in_process_tracing(openshell_driver_docker::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "sandbox_label", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { registration .with_telemetry_category(TelemetryComputeDriver::anonymous_category("vm")) .with_local_singleplayer() - .with_inherited_config_keys(&[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ] { registry diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index e7ff24808b..8ca8a4b5f1 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -31,7 +31,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{Error, Result}; +use openshell_core::{Error, Result, UpstreamProxyConfig}; #[cfg(unix)] use openshell_otel::TraceContextInterceptor; use openshell_server::AcquiredRemoteDriverEndpoint; @@ -100,33 +100,16 @@ pub struct VmComputeConfig { /// Host-side private key for the guest's mTLS client bundle. pub guest_tls_key: Option, - /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) - /// for policy-approved TLS egress from VM sandboxes. - /// - /// Deployment-level configuration, not a per-sandbox setting: it is passed - /// to the driver, which puts it on the guest supervisor's argv. A proxy on - /// this host's loopback is reachable from a guest only through the gvproxy - /// host alias `host.openshell.internal`. - pub https_proxy: Option, - - /// Comma-separated `NO_PROXY` list. Bypasses only the corporate proxy, - /// never `OpenShell` policy evaluation. - pub no_proxy: Option, - - /// Path on this host to a `user:pass` corporate proxy credential file. - pub proxy_auth_file: Option, - - /// Acknowledgement that Basic auth to an `http://` proxy is cleartext. - /// Required alongside `proxy_auth_file` unless the proxy is `https://`. - pub proxy_auth_allow_insecure: Option, - - /// Send hostnames rather than validated IPs in CONNECT requests. Last - /// resort for proxies whose ACLs reject IP CONNECT targets. - pub proxy_connect_by_hostname: Option, - - /// Path on this host to a PEM CA bundle trusted for the corporate proxy - /// and for server certificates a TLS-intercepting proxy re-signs. - pub proxy_ca_bundle: Option, + /// Corporate forward-proxy settings passed to the VM driver. Flattening + /// preserves the shared local-driver TOML field names. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Explicit guest-reachable SPIFFE Workload API TCP listener. VM guests + /// cannot receive a host UNIX socket, so this requires acknowledgement. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, } impl VmComputeConfig { @@ -163,29 +146,6 @@ impl VmComputeConfig { 4096 } - /// Validate the corporate upstream-proxy settings, fail-closed. - /// - /// Runs in the gateway as well as in the driver so an invalid - /// `[openshell.drivers.vm]` table reports the offending key instead of - /// surfacing as an opaque driver-startup timeout. - /// - /// # Errors - /// - /// Returns a [`Error::config`] naming the offending key. - pub fn validate_proxy_config(&self) -> Result<()> { - openshell_core::driver_utils::validate_upstream_proxy_settings( - &openshell_core::driver_utils::UpstreamProxySettings { - url: self.https_proxy.as_deref(), - no_proxy: self.no_proxy.as_deref(), - auth_file: self.proxy_auth_file.as_deref(), - auth_allow_insecure: self.proxy_auth_allow_insecure, - connect_by_hostname: self.proxy_connect_by_hostname, - ca_bundle: self.proxy_ca_bundle.as_deref(), - }, - ) - .map_err(Error::config) - } - #[must_use] fn default_driver_search_dirs(home: Option) -> Vec { let mut dirs = Vec::new(); @@ -214,12 +174,9 @@ impl Default for VmComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - https_proxy: None, - no_proxy: None, - proxy_auth_file: None, - proxy_auth_allow_insecure: None, - proxy_connect_by_hostname: None, - proxy_ca_bundle: None, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, } } } @@ -517,8 +474,21 @@ pub async fn spawn( )); } - vm_config.validate_proxy_config()?; - + vm_config.upstream_proxy.validate().map_err(Error::config)?; + if let Some(endpoint) = vm_config + .provider_spiffe_workload_api_tcp_endpoint + .as_deref() + { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + vm_config.provider_spiffe_allow_guest_tcp, + ) + .map_err(Error::config)?; + } else if vm_config.provider_spiffe_allow_guest_tcp { + return Err(Error::config( + "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", + )); + } let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; @@ -535,9 +505,7 @@ pub async fn spawn( .arg(std::process::id().to_string()); command.arg("--log-level").arg(gateway_log_level); append_otlp_args(&mut command, otlp_config, gateway_name); - command - .arg("--grpc-endpoint") - .arg(&vm_config.grpc_endpoint); + command.arg("--grpc-endpoint").arg(&vm_config.grpc_endpoint); command.arg("--state-dir").arg(&vm_config.state_dir); if !vm_config.default_image.trim().is_empty() { command.arg("--default-image").arg(&vm_config.default_image); @@ -560,7 +528,7 @@ pub async fn spawn( command.arg("--guest-tls-cert").arg(tls.cert); command.arg("--guest-tls-key").arg(tls.key); } - append_upstream_proxy_args(&mut command, vm_config); + append_vm_proxy_and_spiffe_args(&mut command, vm_config); let mut child = command.spawn().map_err(|e| { Error::execution(format!( @@ -575,39 +543,32 @@ pub async fn spawn( )) } -/// Forward the operator's corporate proxy settings to the driver subprocess. -/// -/// Only keys the operator actually set are passed, so the driver keeps the -/// same "omitted means no proxy" contract the supervisor enforces. The -/// booleans travel as explicit values rather than presence flags so an -/// explicit `false` still trips the driver's pairing checks. #[cfg(unix)] -fn append_upstream_proxy_args(command: &mut Command, vm_config: &VmComputeConfig) { - if let Some(url) = &vm_config.https_proxy { - command.arg("--https-proxy").arg(url); +fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { + let proxy = &config.upstream_proxy; + if let Some(url) = proxy.https_proxy.as_ref() { + command.arg("--upstream-proxy").arg(url); } - if let Some(list) = &vm_config.no_proxy { - command.arg("--no-proxy").arg(list); + if let Some(no_proxy) = proxy.no_proxy.as_ref() { + command.arg("--upstream-no-proxy").arg(no_proxy); } - if let Some(path) = &vm_config.proxy_auth_file { - command.arg("--proxy-auth-file").arg(path); + if let Some(auth_file) = proxy.proxy_auth_file.as_ref() { + command.arg("--upstream-proxy-auth-file").arg(auth_file); } - if let Some(allow) = vm_config.proxy_auth_allow_insecure { - command - .arg("--proxy-auth-allow-insecure") - .arg(allow.to_string()); + if proxy.proxy_auth_allow_insecure == Some(true) { + command.arg("--upstream-proxy-auth-allow-insecure"); } - if let Some(by_hostname) = vm_config.proxy_connect_by_hostname { - command - .arg("--proxy-connect-by-hostname") - .arg(by_hostname.to_string()); + if proxy.proxy_connect_by_hostname == Some(true) { + command.arg("--upstream-proxy-connect-by-hostname"); } - if let Some(path) = &vm_config.proxy_ca_bundle { - command.arg("--proxy-ca-bundle").arg(path); + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + command + .arg("--provider-spiffe-workload-api-tcp-endpoint") + .arg(endpoint); + command.arg("--provider-spiffe-allow-guest-tcp"); } } -#[cfg(unix)] fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { command.arg("--otlp-endpoint").arg(&config.endpoint); @@ -698,7 +659,7 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, append_upstream_proxy_args, + VmComputeConfig, append_otlp_args, append_vm_proxy_and_spiffe_args, compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, @@ -740,15 +701,18 @@ mod tests { #[test] fn vm_driver_command_forwards_corporate_proxy_settings() { let mut command = tokio::process::Command::new("openshell-driver-vm"); - append_upstream_proxy_args( + append_vm_proxy_and_spiffe_args( &mut command, &VmComputeConfig { - https_proxy: Some("http://proxy.corp.com:8080".to_string()), - no_proxy: Some("10.0.0.0/8".to_string()), - proxy_auth_file: Some("/etc/openshell/secrets/proxy-auth".to_string()), - proxy_auth_allow_insecure: Some(true), - proxy_connect_by_hostname: Some(false), - proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), + upstream_proxy: openshell_core::UpstreamProxyConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + no_proxy: Some("10.0.0.0/8".to_string()), + proxy_auth_file: Some(PathBuf::from("/etc/openshell/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, ..VmComputeConfig::default() }, ); @@ -761,20 +725,17 @@ mod tests { assert_eq!( args, [ - "--https-proxy", + "--upstream-proxy", "http://proxy.corp.com:8080", - "--no-proxy", + "--upstream-no-proxy", "10.0.0.0/8", - "--proxy-auth-file", + "--upstream-proxy-auth-file", "/etc/openshell/secrets/proxy-auth", - "--proxy-auth-allow-insecure", - "true", - // Passed as an explicit value, not a presence flag, so the - // driver still sees the operator's `false`. - "--proxy-connect-by-hostname", - "false", - "--proxy-ca-bundle", - "/etc/openshell/tls/proxy-ca.pem", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--provider-spiffe-workload-api-tcp-endpoint", + "tcp:192.0.2.10:8081", + "--provider-spiffe-allow-guest-tcp", ] ); } @@ -782,7 +743,7 @@ mod tests { #[test] fn vm_driver_command_omits_unset_corporate_proxy_settings() { let mut command = tokio::process::Command::new("openshell-driver-vm"); - append_upstream_proxy_args(&mut command, &VmComputeConfig::default()); + append_vm_proxy_and_spiffe_args(&mut command, &VmComputeConfig::default()); assert_eq!(command.as_std().get_args().count(), 0); } @@ -790,27 +751,19 @@ mod tests { fn invalid_corporate_proxy_config_is_rejected_before_the_driver_starts() { // Without this the operator would see an opaque driver-readiness // timeout instead of an error naming the offending key. - let err = VmComputeConfig { + let err = openshell_core::UpstreamProxyConfig { https_proxy: Some("socks5://proxy.corp.com:1080".to_string()), - ..VmComputeConfig::default() + ..Default::default() } - .validate_proxy_config() + .validate() .expect_err("only http:// and https:// proxies are supported"); - assert!(err.to_string().contains("https_proxy"), "{err}"); + assert!(err.contains("https_proxy"), "{err}"); - let err = VmComputeConfig { - proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), - ..VmComputeConfig::default() - } - .validate_proxy_config() - .expect_err("a CA bundle without a proxy URL would hide a fail-open state"); - assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); - - VmComputeConfig { + openshell_core::UpstreamProxyConfig { https_proxy: Some("http://proxy.corp.com:8080".to_string()), - ..VmComputeConfig::default() + ..Default::default() } - .validate_proxy_config() + .validate() .expect("a lone proxy URL is a complete configuration"); } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 0f31a3951c..257c213d24 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -101,29 +101,25 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, - /// Compute drivers configured for this gateway. + /// Compute driver configured for this gateway. /// - /// Accepts a comma-delimited list of registered driver names. The - /// configuration format is future-proofed for multiple drivers, but the - /// gateway currently requires exactly one. When unset, the gateway runs + /// Accepts one registered driver name. When unset, the gateway runs /// detection probes supplied by the drivers compiled into the binary. #[arg( - long, - alias = "driver", - env = "OPENSHELL_DRIVERS", - value_delimiter = ',', + long = "compute-driver", + env = "OPENSHELL_COMPUTE_DRIVER", value_parser = parse_compute_driver )] - drivers: Vec, + compute_driver: Option, /// Path to a Unix domain socket served by a remote compute driver /// implementing `compute_driver.proto`. /// - /// When set, the socket is associated with the single driver name supplied - /// by `--drivers` or `OPENSHELL_DRIVERS` and replaces normal construction - /// for that selected name, including a compiled registration with the same - /// name. The gateway connects to this operator-provided endpoint; it does - /// not provision the remote driver. + /// When set, the socket is associated with the driver name supplied by + /// `--compute-driver` or `OPENSHELL_COMPUTE_DRIVER` and replaces normal + /// construction for that selected name, including a compiled registration + /// with the same name. The gateway connects to this operator-provided + /// endpoint; it does not provision the remote driver. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] compute_driver_socket: Option, @@ -267,12 +263,17 @@ fn prepare_server_config_with_drivers( } normalize_compute_driver_socket_args(args, matches)?; let compute_driver = compute_drivers - .select(&args.drivers) + .select(args.compute_driver.as_deref()) .map_err(|error| miette::miette!("{error}"))?; let selected_registration = compute_drivers.get(compute_driver.name()); let local_tls = apply_runtime_defaults(args)?; - let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); + let guest_tls = GuestTlsPaths::resolve( + file.as_ref().map(|file| &file.openshell.gateway), + local_tls.as_ref(), + args.disable_tls, + ) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; let local_jwt = defaults::complete_local_jwt_config()?; let bind = SocketAddr::new(args.bind_address, args.port); @@ -409,9 +410,11 @@ fn prepare_server_config_with_drivers( config = config.with_metrics_bind_address(addr); } + config = config.with_database_url(db_url); + if let Some(driver) = &args.compute_driver { + config = config.with_compute_driver(driver); + } config = config - .with_database_url(db_url) - .with_compute_drivers(args.drivers.clone()) .with_grpc_rate_limit( args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, @@ -444,8 +447,8 @@ fn prepare_server_config_with_drivers( )?; if let Some(socket) = args.compute_driver_socket.clone() { let driver = args - .drivers - .first() + .compute_driver + .as_ref() .expect("normalize_compute_driver_socket_args sets a driver for socket endpoints"); config = config.with_compute_driver_endpoint(driver.clone(), socket); } @@ -697,10 +700,10 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.log_level.clone_from(level); } - if let Some(drivers) = &file.compute_drivers - && arg_defaulted(matches, "drivers") + if let Some(driver) = &file.compute_driver + && arg_defaulted(matches, "compute_driver") { - args.drivers.clone_from(drivers); + args.compute_driver = Some(driver.clone()); } if let Some(sans) = &file.server_sans && args.server_sans.is_empty() @@ -801,24 +804,21 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches "--compute-driver-socket must not be an empty path" )); } - if arg_defaulted(matches, "drivers") { + if arg_defaulted(matches, "compute_driver") { return Err(miette::miette!( - "--compute-driver-socket requires --drivers or OPENSHELL_DRIVERS= to select a compute driver name" + "--compute-driver-socket requires --compute-driver or OPENSHELL_COMPUTE_DRIVER=" )); } - match args.drivers.as_slice() { - [driver] => { - let driver = openshell_core::config::normalize_compute_driver_name(driver) - .map_err(|err| miette::miette!("{err}"))?; - args.drivers[0] = driver; - Ok(()) - } - drivers => Err(miette::miette!( - "--compute-driver-socket requires exactly one compute driver name, got: {}", - drivers.join(",") - )), - } + let driver = args + .compute_driver + .as_deref() + .expect("explicit compute driver is required for socket endpoints"); + args.compute_driver = Some( + openshell_core::config::normalize_compute_driver_name(driver) + .map_err(|err| miette::miette!("{err}"))?, + ); + Ok(()) } fn is_singleplayer_driver(registration: Option<&crate::ComputeDriverRegistration>) -> bool { @@ -1249,6 +1249,14 @@ mod tests { toml::from_str(toml).expect("valid TOML in test fixture") } + #[test] + fn rejects_legacy_drivers_flag() { + let error = command() + .try_get_matches_from(["openshell-gateway", "--drivers", "docker"]) + .expect_err("legacy --drivers flag must be rejected"); + assert!(error.to_string().contains("--drivers")); + } + #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK @@ -1263,7 +1271,7 @@ mod tests { let config = tmp.path().join("openshell").join("gateway.toml"); std::fs::create_dir_all(config.parent().unwrap()).unwrap(); - std::fs::write(&config, "[openshell]\nversion = 1\n").unwrap(); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); assert_eq!(super::resolve_config_path(&args).unwrap(), Some(config)); } @@ -1357,7 +1365,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1385,7 +1393,7 @@ mod tests { let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); - let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _drivers = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); let (mut args, matches) = parse_with_args(&[ @@ -1405,7 +1413,7 @@ mod tests { super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); assert_eq!(prepared.compute_driver.name(), "local"); - assert!(prepared.config.compute_drivers.is_empty()); + assert!(prepared.config.compute_driver.is_none()); assert!(prepared.config.mtls_auth.enabled); assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } @@ -1421,7 +1429,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "shared", "--tls-cert", "/tmp/server.crt", @@ -1450,7 +1458,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1724,13 +1732,13 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "Kyma", "--compute-driver-socket", "/run/openshell/kyma.sock", @@ -1740,7 +1748,7 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1749,7 +1757,7 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", @@ -1761,7 +1769,7 @@ ssh_session_ttl_secs = 1234 let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); assert!( - err.to_string().contains("requires --drivers "), + err.to_string().contains("requires --compute-driver "), "unexpected error: {err}" ); } @@ -1772,19 +1780,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--compute-driver-socket", "/run/openshell/extension.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["docker"]); + assert_eq!(args.compute_driver.as_deref(), Some("docker")); } #[test] @@ -1793,19 +1801,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); let (mut args, matches) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "vm", "--compute-driver-socket", "/run/openshell/vm.sock", ]); super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["vm"]); + assert_eq!(args.compute_driver.as_deref(), Some("vm")); } #[test] @@ -1817,7 +1825,7 @@ ssh_session_ttl_secs = 1234 "OPENSHELL_COMPUTE_DRIVER_SOCKET", "/var/run/openshell/kyma.sock", ); - let _g2 = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _g2 = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "kyma"); let (mut args, matches) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); @@ -1826,7 +1834,7 @@ ssh_session_ttl_secs = 1234 args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/var/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1882,19 +1890,14 @@ enable_loopback_service_http = false } #[test] - fn canonical_and_legacy_file_driver_selectors_merge_equivalently() { - for input in [ - "[openshell.gateway]\ncompute_driver = \"podman\"\n", - "[openshell.gateway]\ncompute_drivers = [\"podman\"]\n", - ] { - let (mut args, matches) = - parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); - let file = config_file_from_toml(input); + fn canonical_file_driver_selector_populates_cli_args() { + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml("[openshell.gateway]\ncompute_driver = \"podman\"\n"); - merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); - assert_eq!(args.drivers, vec!["podman".to_string()]); - } + assert_eq!(args.compute_driver.as_deref(), Some("podman")); } #[test] @@ -1913,6 +1916,9 @@ enable_loopback_service_http = false std::fs::write( &config_path, r#" +[openshell] +version = 2 + [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" @@ -1931,7 +1937,7 @@ mem_mib = "not-a-number" config_path.to_str().unwrap(), "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "podman", "--disable-tls", ]); @@ -1939,7 +1945,7 @@ mem_mib = "not-a-number" let prepared = super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); - assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); + assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index d06e6fbc8f..c4e4d91ecd 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -27,13 +27,68 @@ impl GuestTlsPaths { } } -impl From<&LocalTlsPaths> for GuestTlsPaths { - fn from(paths: &LocalTlsPaths) -> Self { - Self { +impl GuestTlsPaths { + /// Resolve gateway-owned guest TLS inputs. Explicit TOML values take + /// precedence over the package-managed local bundle; partial bundles are + /// rejected before any driver is deserialized or constructed. + pub(crate) fn resolve( + gateway: Option<&config_file::GatewayFileSection>, + local: Option<&LocalTlsPaths>, + tls_disabled: bool, + ) -> std::result::Result, String> { + let configured = gateway.map(|gateway| { + ( + gateway.guest_tls_ca.as_ref(), + gateway.guest_tls_cert.as_ref(), + gateway.guest_tls_key.as_ref(), + ) + }); + let provided = configured + .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); + + if tls_disabled { + if provided { + return Err( + "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" + .to_string(), + ); + } + return Ok(None); + } + + if let Some((ca, cert, key)) = configured + && (ca.is_some() || cert.is_some() || key.is_some()) + { + let (Some(ca), Some(cert), Some(key)) = (ca, cert, key) else { + return Err( + "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" + .to_string(), + ); + }; + for (field, path) in [ + ("guest_tls_ca", ca), + ("guest_tls_cert", cert), + ("guest_tls_key", key), + ] { + if !path.is_file() { + return Err(format!( + "{field} '{}' does not exist or is not a file", + path.display() + )); + } + } + return Ok(Some(Self { + ca: ca.clone(), + cert: cert.clone(), + key: key.clone(), + })); + } + + Ok(local.map(|paths| Self { ca: paths.ca.clone(), cert: paths.client_cert.clone(), key: paths.client_key.clone(), - } + })) } } @@ -57,6 +112,7 @@ pub fn remote_driver_config_from_context( &file.openshell.gateway, file.openshell.drivers.get(name), ); + reject_driver_owned_guest_tls_fields(&merged)?; if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { cfg.socket_path = PathBuf::from(socket_path); } @@ -100,6 +156,7 @@ where file.openshell.drivers.get(driver_name), inherited_config_keys, ); + reject_driver_owned_guest_tls_fields(&merged)?; merged.try_into().map_err(|e| { Error::config(format!( "invalid [openshell.drivers.{driver_name}] table: {e}" @@ -107,6 +164,23 @@ where }) } +/// Reject TLS paths in gateway driver tables. These credentials are gateway +/// inputs and are injected only into the selected local driver after the +/// gateway has validated the complete bundle. +fn reject_driver_owned_guest_tls_fields(table: &toml::Value) -> Result<()> { + let Some(table) = table.as_table() else { + return Ok(()); + }; + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + if table.contains_key(field) { + return Err(Error::config(format!( + "{field} belongs in [openshell.gateway], not a [openshell.drivers.*] table" + ))); + } + } + Ok(()) +} + fn apply_remote_driver_overrides( cfg: &mut RemoteDriverConfig, context: DriverStartupContext<'_>, @@ -150,6 +224,30 @@ mod tests { } } + #[test] + fn gateway_guest_tls_requires_complete_bundle() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + + #[test] + fn gateway_guest_tls_rejects_plaintext_gateway() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: Some(PathBuf::from("/tmp/cert.pem")), + guest_tls_key: Some(PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, true) + .expect_err("guest TLS and plaintext gateway conflict"); + assert!(error.contains("require gateway TLS")); + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( @@ -167,12 +265,9 @@ socket_path = "/run/openshell/kyma.sock" } #[test] - fn remote_driver_config_ignores_in_process_driver_fields() { + fn remote_driver_config_reads_only_socket_path() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" - [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" workspace_mode = "shared" diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 18aa6aa49d..1e717c17c4 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -6,9 +6,8 @@ //! See `rfc/0003-gateway-configuration/README.md` for the file format. This //! module parses the file into [`ConfigFile`], rejects fields that must be //! supplied via env/CLI (database URL), and provides -//! [`driver_table`] which overlays shared `[openshell.gateway]` defaults onto -//! a `[openshell.drivers.]` table so each driver crate's -//! `Deserialize` impl sees a fully-populated table. +//! [`driver_table`] which returns a driver-owned +//! `[openshell.drivers.]` table without gateway-level inheritance. //! //! The merge precedence for gateway process settings is: //! ```text @@ -31,11 +30,10 @@ use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, TlsConfig, }; -use serde::de::{SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; -/// Latest schema version this build understands. -pub const SCHEMA_VERSION: u32 = 1; +/// Gateway configuration schema version supported by this build. +pub const SCHEMA_VERSION: u32 = 2; /// Root of the gateway TOML config file. /// @@ -53,8 +51,8 @@ pub struct ConfigFile { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OpenShellRoot { - /// Reserved for future schema migrations. Versions greater than - /// [`SCHEMA_VERSION`] are rejected at load time. + /// Gateway configuration schema version. Loaded files must set this to + /// [`SCHEMA_VERSION`]. #[serde(default)] pub version: Option, @@ -65,8 +63,8 @@ pub struct OpenShellRoot { pub supervisor: SupervisorFileSection, /// `[openshell.drivers.]` tables — passed verbatim to each driver - /// crate's `Deserialize` impl after the gateway-side inheritance merge. - /// Stored as raw [`toml::Value`] so each driver can evolve its schema + /// crate's `Deserialize` impl. Stored as raw [`toml::Value`] so each + /// driver can evolve its schema /// independently of this crate. #[serde(default)] pub drivers: BTreeMap, @@ -81,9 +79,8 @@ pub struct OpenShellRoot { /// /// All fields are `Option` so the loader can tell whether a key was set /// in the file (`Some`) or not (`None` — value is taken from CLI/env/default). -/// -/// The fields under "Shared driver defaults" are inherited into -/// `[openshell.drivers.]` tables per [`inheritable_keys`]. +/// Driver-specific settings belong exclusively in +/// `[openshell.drivers.]` tables. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayFileSection { @@ -105,19 +102,9 @@ pub struct GatewayFileSection { pub log_level: Option, // ── Drivers ────────────────────────────────────────────────────────── - /// Canonical TOML uses the singular `compute_driver = "..."`. The legacy - /// `compute_drivers = ["..."]` form remains accepted and is normalized to - /// this existing vector representation so Rust callers and runtime - /// validation retain their current behavior. - #[serde( - default, - rename = "compute_driver", - alias = "compute_drivers", - deserialize_with = "deserialize_compute_drivers", - serialize_with = "serialize_compute_drivers", - skip_serializing_if = "Option::is_none" - )] - pub compute_drivers: Option>, + /// Explicit compute driver selection. `None` enables auto-detection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compute_driver: Option, #[serde(default)] pub credential_drivers: Option>, #[serde(default)] @@ -126,11 +113,6 @@ pub struct GatewayFileSection { pub credential_storage: Option, // ── Sandbox / SSH ──────────────────────────────────────────────────── - /// Compatibility input for Kubernetes `namespace` and Docker - /// `sandbox_label`. Canonical configurations set those driver-owned - /// fields in their respective `[openshell.drivers.]` tables. - #[serde(default)] - pub sandbox_namespace: Option, #[serde(default)] pub ssh_session_ttl_secs: Option, #[serde(default)] @@ -150,26 +132,7 @@ pub struct GatewayFileSection { #[serde(default)] pub enable_loopback_service_http: Option, - // ── Shared driver defaults (inherited into [openshell.drivers.]) ─ - #[serde(default)] - pub default_image: Option, - #[serde(default)] - pub supervisor_image: Option, - #[serde(default)] - pub client_tls_secret_name: Option, - /// Compatibility input for Kubernetes `service_account_name`. - #[serde(default)] - pub service_account_name: Option, - #[serde(default)] - pub host_gateway_ip: Option, - /// Compatibility input for Kubernetes `enable_user_namespaces`. - #[serde(default)] - pub enable_user_namespaces: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet - /// writes for the `IssueSandboxToken` bootstrap exchange. Driver - /// clamps to `[600, 86400]`. - #[serde(default)] - pub sa_token_ttl_secs: Option, + // ── Sandbox client TLS ─────────────────────────────────────────────── #[serde(default)] pub guest_tls_ca: Option, #[serde(default)] @@ -210,62 +173,6 @@ pub struct GatewayFileSection { pub database_url: Option, } -fn deserialize_compute_drivers<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - struct ComputeDriversVisitor; - - impl<'de> Visitor<'de> for ComputeDriversVisitor { - type Value = Option>; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("a compute driver name or an array of compute driver names") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - Ok(Some(vec![value.to_string()])) - } - - fn visit_string(self, value: String) -> Result - where - E: serde::de::Error, - { - Ok(Some(vec![value])) - } - - fn visit_seq(self, mut sequence: A) -> Result - where - A: SeqAccess<'de>, - { - let mut drivers = Vec::new(); - while let Some(driver) = sequence.next_element::()? { - drivers.push(driver); - } - Ok(Some(drivers)) - } - } - - deserializer.deserialize_any(ComputeDriversVisitor) -} - -fn serialize_compute_drivers( - drivers: &Option>, - serializer: S, -) -> Result -where - S: Serializer, -{ - match drivers { - Some(drivers) if drivers.len() == 1 => serializer.serialize_str(&drivers[0]), - Some(drivers) => drivers.serialize(serializer), - None => serializer.serialize_none(), - } -} - /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -408,7 +315,11 @@ pub enum ConfigFileError { source: toml::de::Error, }, #[error( - "unsupported gateway config version {version}; this build only supports version {SCHEMA_VERSION}" + "gateway config schema version is required; add `[openshell]` and `version = {SCHEMA_VERSION}`" + )] + MissingVersion, + #[error( + "unsupported gateway config version {version}; this build requires version {SCHEMA_VERSION}; migrate legacy fields to the version {SCHEMA_VERSION} schema" )] UnsupportedVersion { version: u32 }, #[error( @@ -424,6 +335,8 @@ pub enum ConfigFileError { field: &'static str, message: &'static str, }, + #[error("invalid gateway config field `openshell.drivers.{name}`: expected a TOML table")] + InvalidDriverTable { name: String }, #[error( "failed to read TLS CA certificate for supervisor middleware '{name}' from '{}': {source}", path.display() @@ -447,8 +360,8 @@ pub enum ConfigFileError { /// Load and validate a TOML config file. /// -/// Returns `Ok(ConfigFile::default())` for an empty file (the gateway then -/// falls back entirely to CLI/env/built-in defaults). +/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running +/// without a config file still uses CLI, environment, and built-in defaults. #[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] pub fn load(path: &Path) -> Result { let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { @@ -456,17 +369,17 @@ pub fn load(path: &Path) -> Result { source, })?; if contents.trim().is_empty() { - return Ok(ConfigFile::default()); + return Err(ConfigFileError::MissingVersion); } let file: ConfigFile = toml::from_str(&contents).map_err(|source| ConfigFileError::Parse { path: path.to_path_buf(), source, })?; - if let Some(version) = file.openshell.version - && version > SCHEMA_VERSION - { - return Err(ConfigFileError::UnsupportedVersion { version }); + match file.openshell.version { + Some(SCHEMA_VERSION) => {} + Some(version) => return Err(ConfigFileError::UnsupportedVersion { version }), + None => return Err(ConfigFileError::MissingVersion), } if file.openshell.gateway.database_url.is_some() { @@ -488,79 +401,39 @@ pub fn load(path: &Path) -> Result { message: "omit the field to use default encrypted gateway credential storage, or specify exactly one external credential driver", }); } + if let Some((name, _)) = file + .openshell + .drivers + .iter() + .find(|(_, value)| !value.is_table()) + { + return Err(ConfigFileError::InvalidDriverTable { name: name.clone() }); + } Ok(file) } -/// Build the merged TOML table for `driver` by overlaying inheritable -/// `[openshell.gateway]` defaults onto `[openshell.drivers.]`. -/// -/// The returned [`toml::Value`] is a Table ready to feed into the driver's -/// `Deserialize` impl — keys present in `raw` win over the gateway defaults. -/// Keys outside [`inheritable_keys`] for this driver are never copied from -/// the gateway section, which keeps each driver's `deny_unknown_fields` -/// invariant intact. +/// Return a driver's table without gateway-level inheritance. +/// Driver-specific configuration belongs exclusively to +/// `[openshell.drivers.]` in schema version 2. pub fn driver_table( - driver_name: &str, - gateway: &GatewayFileSection, + _driver_name: &str, + _gateway: &GatewayFileSection, raw: Option<&toml::Value>, ) -> toml::Value { - driver_table_with_inherited_keys(driver_name, gateway, raw, &[]) + match raw { + Some(toml::Value::Table(table)) => toml::Value::Table(table.clone()), + _ => toml::Value::Table(toml::Table::new()), + } } pub(crate) fn driver_table_with_inherited_keys( - _driver_name: &str, + driver_name: &str, gateway: &GatewayFileSection, raw: Option<&toml::Value>, - inheritable_keys: &[&str], + _inherited_config_keys: &[&str], ) -> toml::Value { - let mut merged = match raw { - Some(toml::Value::Table(table)) => table.clone(), - _ => toml::Table::new(), - }; - - for key in inheritable_keys { - if driver_field_is_present(&merged, key) { - continue; - } - if let Some(value) = gateway_inherited_value(gateway, key) { - merged.insert((*key).to_string(), value); - } - } - - toml::Value::Table(merged) -} - -fn driver_field_is_present(table: &toml::Table, key: &str) -> bool { - table.contains_key(key) - || (key == "sandbox_label" && table.contains_key("sandbox_namespace")) -} - -fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { - match key { - "namespace" | "sandbox_namespace" | "sandbox_label" => { - g.sandbox_namespace.as_deref().map(string_value) - } - "default_image" => g.default_image.as_deref().map(string_value), - "supervisor_image" => g.supervisor_image.as_deref().map(string_value), - "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), - "service_account_name" => g.service_account_name.as_deref().map(string_value), - "host_gateway_ip" => g.host_gateway_ip.as_deref().map(string_value), - "enable_user_namespaces" => g.enable_user_namespaces.map(toml::Value::Boolean), - "sa_token_ttl_secs" => g.sa_token_ttl_secs.map(toml::Value::Integer), - "guest_tls_ca" => g.guest_tls_ca.as_deref().map(path_value), - "guest_tls_cert" => g.guest_tls_cert.as_deref().map(path_value), - "guest_tls_key" => g.guest_tls_key.as_deref().map(path_value), - _ => None, - } -} - -fn string_value(s: &str) -> toml::Value { - toml::Value::String(s.to_owned()) -} - -fn path_value(p: &Path) -> toml::Value { - toml::Value::String(p.display().to_string()) + driver_table(driver_name, gateway, raw) } #[cfg(test)] @@ -568,7 +441,7 @@ mod tests { use super::*; use std::io::Write; - fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + fn write_raw_tmp(contents: &str) -> tempfile::NamedTempFile { let mut tmp = tempfile::Builder::new() .suffix(".toml") .tempfile() @@ -577,17 +450,38 @@ mod tests { tmp } + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + if contents.contains("[openshell]") { + write_raw_tmp(contents) + } else { + write_raw_tmp(&format!("[openshell]\nversion = 2\n\n{contents}")) + } + } + #[test] - fn empty_file_yields_default_config() { - let tmp = write_tmp(""); - let file = load(tmp.path()).expect("empty file parses"); - assert!(file.openshell.version.is_none()); - assert!(file.openshell.gateway.bind_address.is_none()); - assert!(file.openshell.drivers.is_empty()); + fn empty_file_requires_schema_version() { + let tmp = write_raw_tmp(""); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); } #[test] - fn canonical_compute_driver_scalar_normalizes_to_existing_vector() { + fn compute_driver_entries_must_be_tables() { + for value in ["\"not-a-table\"", "[\"also\", \"not-a-table\"]", "42"] { + let tmp = write_raw_tmp(&format!( + "[openshell]\nversion = 2\n\n[openshell.drivers]\ndocker = {value}\n" + )); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::InvalidDriverTable { ref name }) if name == "docker" + )); + } + } + + #[test] + fn canonical_compute_driver_is_singular() { let file: ConfigFile = toml::from_str( r#" [openshell.gateway] @@ -597,64 +491,37 @@ compute_driver = "docker" .expect("canonical compute driver parses"); assert_eq!( - file.openshell.gateway.compute_drivers, - Some(vec!["docker".to_string()]) + file.openshell.gateway.compute_driver.as_deref(), + Some("docker") ); } #[test] - fn legacy_compute_drivers_list_remains_accepted() { - for (input, expected) in [ - ("compute_drivers = []", Vec::::new()), - ("compute_drivers = [\"docker\"]", vec!["docker".to_string()]), - ( - "compute_drivers = [\"docker\", \"podman\"]", - vec!["docker".to_string(), "podman".to_string()], - ), - ] { - let file: ConfigFile = toml::from_str(&format!("[openshell.gateway]\n{input}\n")) - .expect("legacy compute drivers parse"); - assert_eq!(file.openshell.gateway.compute_drivers, Some(expected)); - } + fn legacy_compute_drivers_list_is_rejected() { + let error = + toml::from_str::("[openshell.gateway]\ncompute_drivers = [\"docker\"]\n") + .expect_err("legacy compute_drivers must be rejected"); + assert!(error.to_string().contains("compute_drivers")); } #[test] - fn compute_driver_rejects_non_string_values_with_a_clear_error() { + fn compute_driver_rejects_non_string_values() { let error = toml::from_str::( r" [openshell.gateway] compute_driver = 42 ", ) - .expect_err("compute driver must be a string or string array"); - - assert!( - error - .to_string() - .contains("a compute driver name or an array of compute driver names") - ); - } - - #[test] - fn canonical_and_legacy_compute_driver_names_are_rejected_together() { - let error = toml::from_str::( - r#" -[openshell.gateway] -compute_driver = "docker" -compute_drivers = ["docker"] -"#, - ) - .expect_err("canonical and legacy names must not both be accepted"); - - assert!(error.to_string().contains("duplicate field")); + .expect_err("compute driver must be a string"); + assert!(error.to_string().contains("invalid type")); } #[test] - fn compute_driver_serialization_uses_canonical_scalar_name() { + fn compute_driver_serialization_uses_scalar_name() { let file = ConfigFile { openshell: OpenShellRoot { gateway: GatewayFileSection { - compute_drivers: Some(vec!["docker".to_string()]), + compute_driver: Some("docker".to_string()), ..Default::default() }, ..Default::default() @@ -663,14 +530,13 @@ compute_drivers = ["docker"] let serialized = toml::to_string(&file).expect("config serializes"); assert!(serialized.contains("compute_driver = \"docker\"")); - assert!(!serialized.contains("compute_drivers")); } #[test] fn parses_full_example() { let toml = r#" [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" @@ -678,7 +544,6 @@ health_bind_address = "0.0.0.0:8081" log_level = "info" compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 policy_validation_failure_mode = "retain_last_valid" @@ -698,6 +563,10 @@ audience = "openshell-cli" [openshell.drivers.kubernetes] namespace = "agents" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +client_tls_secret_name = "openshell-sandbox-tls" +service_account_name = "openshell-sandbox" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" [openshell.credential_drivers.kubernetes-secrets] @@ -1055,6 +924,27 @@ nonsense = true assert!(matches!(err, ConfigFileError::Parse { .. })); } + #[test] + fn rejects_removed_driver_fields_at_gateway_scope() { + for field in [ + "sandbox_namespace = \"agents\"", + "default_image = \"sandbox:latest\"", + "supervisor_image = \"supervisor:latest\"", + "client_tls_secret_name = \"sandbox-tls\"", + "service_account_name = \"sandbox-sa\"", + "host_gateway_ip = \"10.0.0.1\"", + "enable_user_namespaces = true", + "sa_token_ttl_secs = 3600", + ] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field}\n")); + let err = load(tmp.path()).expect_err("gateway-scoped driver field must be rejected"); + assert!( + matches!(err, ConfigFileError::Parse { .. }), + "field: {field}" + ); + } + } + #[test] fn rejects_unknown_field_in_nested_gateway_jwt_table() { // Regression guard for the class of silent-misconfig bug fixed in @@ -1087,218 +977,41 @@ ssh_gateway_port = 8080 } #[test] - fn rejects_unsupported_version() { - let toml = r" -[openshell] -version = 2 -"; - let tmp = write_tmp(toml); - let err = load(tmp.path()).expect_err("version > 1 must be rejected"); + fn rejects_legacy_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 1\n"); + let err = load(tmp.path()).expect_err("version 1 must be rejected"); assert!(matches!( err, - ConfigFileError::UnsupportedVersion { version: 2 } + ConfigFileError::UnsupportedVersion { version: 1 } )); } #[test] - fn driver_table_inherits_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), - ), - supervisor_image: Some("ghcr.io/nvidia/openshell/supervisor:0.9".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - namespace = "agents" - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - Some(&toml::Value::Table(raw)), - &["default_image", "supervisor_image"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("supervisor_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/supervisor:0.9") - ); - } - - #[test] - fn registered_driver_table_inherits_selected_gateway_defaults() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("agents".to_string()), - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), - ), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - None, - &["sandbox_label", "default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_label").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("10.0.0.1") - ); - } - - #[test] - fn canonical_driver_label_overrides_legacy_gateway_default() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("gateway-default".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - sandbox_label = "driver-specific" - }; - let merged = driver_table_with_inherited_keys( - "docker", - &gateway, - Some(&toml::Value::Table(raw)), - &["sandbox_label"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_label").and_then(toml::Value::as_str), - Some("driver-specific") - ); - assert!(!table.contains_key("sandbox_namespace")); - } - - #[test] - fn legacy_driver_label_suppresses_canonical_gateway_inheritance() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("gateway-default".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - sandbox_namespace = "driver-specific" - }; - let merged = driver_table_with_inherited_keys( - "docker", - &gateway, - Some(&toml::Value::Table(raw)), - &["sandbox_label"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table - .get("sandbox_namespace") - .and_then(toml::Value::as_str), - Some("driver-specific") - ); - assert!(!table.contains_key("sandbox_label")); - } - - #[test] - fn registered_driver_table_can_select_network_defaults() { - let gateway = GatewayFileSection { - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), - ), - host_gateway_ip: Some("192.168.127.254".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "beta", - &gateway, - None, - &["default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("192.168.127.254") - ); - } - - #[test] - fn driver_table_specific_value_overrides_gateway_default() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default".to_string()), - ..Default::default() - }; + fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { default_image = "driver-specific" + socket_path = "/run/openshell/driver.sock" }; - let merged = driver_table_with_inherited_keys( + let table = driver_table( "alpha", - &gateway, + &GatewayFileSection::default(), Some(&toml::Value::Table(raw)), - &["default_image"], ); + let table = table.as_table().expect("driver table"); assert_eq!( - merged - .as_table() - .unwrap() - .get("default_image") - .and_then(|v| v.as_str()), + table.get("default_image").and_then(toml::Value::as_str), Some("driver-specific") ); - } - - #[test] - fn driver_table_does_not_leak_keys_outside_allowlist() { - // Fields not selected by the registration must remain gateway-only. - let gateway = GatewayFileSection { - client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys("alpha", &gateway, None, &["default_image"]); - assert!( - !merged - .as_table() - .unwrap() - .contains_key("client_tls_secret_name") + assert_eq!( + table.get("socket_path").and_then(toml::Value::as_str), + Some("/run/openshell/driver.sock") ); } #[test] - fn remote_driver_table_does_not_inherit_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default:1.0".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - socket_path = "/run/openshell/kyma.sock" - }; - - let merged = driver_table("kyma", &gateway, Some(&toml::Value::Table(raw))); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("socket_path").and_then(|v| v.as_str()), - Some("/run/openshell/kyma.sock") - ); - assert!(!table.contains_key("default_image")); - assert!(!table.contains_key("host_gateway_ip")); + fn driver_table_does_not_inject_gateway_values() { + let table = driver_table("alpha", &GatewayFileSection::default(), None); + assert!(table.as_table().expect("driver table").is_empty()); } #[test] @@ -1332,15 +1045,24 @@ version = 2 ); } - let drivers = gw - .compute_drivers - .as_ref() - .expect("compute_driver must be explicitly set in the RPM default config"); assert_eq!( - drivers, - &["podman".to_string()], + gw.compute_driver.as_deref(), + Some("podman"), "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); + + let podman = driver_table( + "podman", + &config.openshell.gateway, + config.openshell.drivers.get("podman"), + ); + assert_eq!( + podman + .get("health_check_interval_secs") + .and_then(toml::Value::as_integer), + Some(10), + "RPM defaults must retain Podman's readiness health check" + ); } } diff --git a/crates/openshell-server/src/defaults.rs b/crates/openshell-server/src/defaults.rs index b5a5a5e924..21e66b02bd 100644 --- a/crates/openshell-server/src/defaults.rs +++ b/crates/openshell-server/src/defaults.rs @@ -104,7 +104,7 @@ pub fn complete_local_jwt_config() -> Result> { public_key_path: paths.public_key, kid_path: paths.kid, gateway_id: "openshell".to_string(), - ttl_secs: 0, + ttl_secs: None, })), _ => Err(miette::miette!( "partial local sandbox JWT state in {}: expected jwt/signing.pem, jwt/public.pem, and jwt/kid", @@ -237,6 +237,6 @@ mod tests { assert_eq!(config.public_key_path, tmp.path().join("jwt/public.pem")); assert_eq!(config.kid_path, tmp.path().join("jwt/kid")); assert_eq!(config.gateway_id, "openshell"); - assert_eq!(config.ttl_secs, 0); + assert_eq!(config.ttl_secs, None); } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 5058a1d8a7..363f00e561 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -506,7 +506,7 @@ pub(crate) async fn run_server( ); info!( gateway_id = %jwt.gateway_id, - ttl_secs = jwt.ttl_secs, + ttl_secs = jwt.ttl_secs.map(std::num::NonZeroU64::get), "gateway-minted sandbox JWT enabled" ); (Some(issuer), Some(authenticator)) @@ -1262,27 +1262,23 @@ impl ComputeDriverRegistry { ComputeDriverDetection { available } } - pub(crate) fn select(&self, configured_drivers: &[String]) -> Result { - match configured_drivers { - [] => { + pub(crate) fn select(&self, configured_driver: Option<&str>) -> Result { + match configured_driver { + None => { let detection = self.detect(); if detection.selected().is_none() { return Err(Error::config( "no compute driver configured and auto-detection found no suitable installed \ - driver; set --drivers or OPENSHELL_DRIVERS=", + driver; set --compute-driver or OPENSHELL_COMPUTE_DRIVER=", )); } Ok(ComputeDriverSelection::AutoDetected(detection)) } - [driver] => { + Some(driver) => { let name = openshell_core::config::normalize_compute_driver_name(driver) .map_err(Error::config)?; Ok(ComputeDriverSelection::Configured { name }) } - drivers => Err(Error::config(format!( - "multiple compute drivers are not supported yet; configured drivers: {}", - drivers.join(",") - ))), } } } @@ -1500,7 +1496,7 @@ fn configured_compute_driver( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { - let selection = registry.select(&config.compute_drivers)?; + let selection = registry.select(config.compute_driver.as_deref())?; resolve_configured_compute_driver(registry, selection.name(), driver_startup) } @@ -2097,7 +2093,7 @@ mod tests { .unwrap(), ) .unwrap(); - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let config = Config::new(None); let result = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) .unwrap(); @@ -2166,25 +2162,9 @@ mod tests { ); } - #[test] - fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None).with_compute_drivers(["alpha", "beta"]); - let err = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap_err(); - assert!( - err.to_string() - .contains("multiple compute drivers are not supported yet") - ); - assert!(err.to_string().contains("alpha,beta")); - } - #[test] fn configured_compute_driver_accepts_registered_name() { - let config = Config::new(None).with_compute_drivers(["beta"]); + let config = Config::new(None).with_compute_driver("beta"); let registry = test_compute_drivers(); let driver = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) @@ -2201,7 +2181,7 @@ mod tests { #[test] fn configured_compute_driver_resolves_named_remote() { - let config = Config::new(None).with_compute_drivers(["kyma"]); + let config = Config::new(None).with_compute_driver("kyma"); let registry = test_compute_drivers(); let driver = @@ -2228,7 +2208,7 @@ mod tests { #[test] fn configured_compute_driver_uses_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["alpha"]) + .with_compute_driver("alpha") .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); let registry = test_compute_drivers(); @@ -2248,7 +2228,7 @@ mod tests { #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["beta"]) + .with_compute_driver("beta") .with_compute_driver_endpoint("beta", "/run/openshell/beta.sock"); let driver = configured_compute_driver( diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index da8ef72873..9fbd574035 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -22,7 +22,7 @@ # - "host.openshell.internal:host-gateway" [openshell] -version = 1 +version = 2 [openshell.gateway] # Bind to loopback only. The Docker driver adds an extra listener on the @@ -40,7 +40,7 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # first start. The binary is cached to XDG_DATA_HOME and reused on restart. supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. @@ -49,3 +49,6 @@ sandbox_label = "openshell" # The gateway must be published on port 8080 on the Docker host so that # host.openshell.internal:8080 resolves to the gateway container. grpc_endpoint = "http://host.openshell.internal:8080" +# Explicit supervisor-compatible Docker default. Set RuntimeDefault or +# Localhost/ only when the daemon host has AppArmor available. +app_armor_profile = "Unconfined" diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index fb3161a604..20e781dd09 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -276,7 +276,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. | +| server.sandboxImagePullPolicy | string | `nil` | Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Use always, if_not_present, or never; newer is supported only by Podman. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -297,7 +297,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | +| supervisor.image.pullPolicy | string | `nil` | Canonical sandbox supervisor pull policy. Leave unset to use the Kubernetes image default; use always, if_not_present, or never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 15ff87554e..706df3eca5 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -3,7 +3,7 @@ # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). server: - sandboxImagePullPolicy: IfNotPresent + sandboxImagePullPolicy: if_not_present otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). @@ -13,4 +13,4 @@ server: supervisor: image: - pullPolicy: IfNotPresent + pullPolicy: if_not_present diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 515417a740..f31b23274e 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -32,7 +32,7 @@ metadata: data: gateway.toml: | [openshell] - version = 1 + version = 2 [openshell.gateway] name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} @@ -44,6 +44,7 @@ data: metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" {{- end }} log_level = {{ .Values.server.logLevel | quote }} + compute_driver = "kubernetes" {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} @@ -52,17 +53,8 @@ data: {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} {{- end }} policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true - {{- else }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} {{- end }} enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} {{- $sans := list -}} @@ -145,6 +137,16 @@ data: [openshell.drivers.kubernetes] namespace = {{ include "openshell.sandboxNamespace" . | quote }} + default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.supervisorImageOverrideEnabled" . }} + supervisor_image = {{ include "openshell.supervisorImage" . | quote }} + {{- end }} + {{- if .Values.server.hostGatewayIP }} + host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} + {{- end }} + {{- if not .Values.server.disableTls }} + client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} + {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 156d8db19d..13ca8b1769 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -27,6 +27,16 @@ tests: path: data["gateway.toml"] pattern: '(?m)^name\s*=\s*"production-us-west"$' + - it: renders schema version 2 and the Kubernetes compute driver selector + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\]\s*version\s*=\s*2' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?compute_driver\s*=\s*"kubernetes"' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum @@ -138,12 +148,36 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?grpc_endpoint' - - it: renders the sandbox service account name under [openshell.drivers.kubernetes] + - it: omits pull policies by default so Kubernetes applies its own defaults + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^\s*(image_pull_policy|supervisor_image_pull_policy)\s*=' + + - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: never asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + + - it: renders driver-owned Kubernetes settings only in its driver table + template: templates/gateway-config.yaml + set: + server.hostGatewayIP: 10.0.0.1 + server.enableUserNamespaces: true + supervisor.image.tag: test + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' - it: renders user namespace enablement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 453654f790..6c34963246 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,9 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Supervisor image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" + # -- Canonical sandbox supervisor pull policy. Leave unset to use the + # Kubernetes image default; use always, if_not_present, or never. + pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" # -- How the supervisor binary is delivered into sandbox pods. @@ -224,10 +225,10 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default - # (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev - # clusters so new images are picked up without manual eviction. - sandboxImagePullPolicy: "" + # -- Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes + # image default (Always for :latest, IfNotPresent otherwise). Use always, + # if_not_present, or never; newer is supported only by Podman. + sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. sandboxImagePullSecrets: [] diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 2d584c4ba1..68439ea596 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -58,12 +58,11 @@ TLS. stores SQLite state under *~/.local/state/openshell/gateway/*. Environment: **OPENSHELL_DB_URL**. -**--drivers** *DRIVER*\[,*DRIVER*\] -: Compute driver. Accepts a comma-delimited list. The gateway - currently requires exactly one driver. Options: **podman**, +**--compute-driver** *DRIVER* +: Compute driver. Selects exactly one driver. Options: **podman**, **docker**, **kubernetes**, **vm**. When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. VM is opt-in. - Environment: **OPENSHELL_DRIVERS**. + Environment: **OPENSHELL_COMPUTE_DRIVER**. **--tls-cert** *PATH* : Path to server TLS certificate file. Defaults to the local generated diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index aaa97d08d0..45a813d0e8 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -17,7 +17,7 @@ The defaults are tuned for rootless Podman use: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" @@ -215,9 +215,9 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. The legacy `compute_drivers` list remains accepted. | -| `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | +| `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | | `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -232,14 +232,15 @@ settings: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell" stop_timeout_secs = 10 ``` @@ -247,7 +248,7 @@ stop_timeout_secs = 10 ### Image management The gateway pulls container images automatically on first sandbox -creation. The default pull policy is `missing`, which means images are +creation. The default pull policy is `if_not_present`, which means images are pulled once and then cached by Podman. To update cached images: @@ -260,9 +261,10 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest Or set `image_pull_policy = "always"` in `[openshell.drivers.podman]` to pull on every sandbox creation. -To pin specific image versions instead of `:latest`: +To pin specific image versions instead of `:latest`, set these values in +`[openshell.drivers.podman]`: -```shell +```toml supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index f67b69149b..a8460a473e 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -182,7 +182,7 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ### Images not updating -The default image pull policy is `missing` -- images are pulled once +The default image pull policy is `if_not_present` -- images are pulled once and cached. To update: ```shell @@ -255,7 +255,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | +| `OPENSHELL_COMPUTE_DRIVER=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index ba76f873b2..a0a6e296f0 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -15,7 +15,7 @@ # systemctl --user edit openshell-gateway [openshell] -version = 1 +version = 2 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The @@ -26,3 +26,8 @@ version = 1 # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. compute_driver = "podman" + +[openshell.drivers.podman] +# Keep the packaged local-gateway readiness behavior after health checks became +# opt-in in the Podman driver. Omit this setting only to disable health checks. +health_check_interval_secs = 10 diff --git a/docs/about/container-gateway.mdx b/docs/about/container-gateway.mdx index 42fe2c7858..57be7d0142 100644 --- a/docs/about/container-gateway.mdx +++ b/docs/about/container-gateway.mdx @@ -59,7 +59,7 @@ docker run -d \ -v openshell-state:/var/openshell \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=http://host.openshell.internal:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ @@ -128,7 +128,7 @@ docker run -d \ -v "$HOME/.local/state/openshell:/home/openshell/.local/state/openshell" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=https://127.0.0.1:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/home/openshell/.local/state/openshell/openshell.db \ @@ -188,7 +188,7 @@ podman run -d \ -p 127.0.0.1:8080:8080 \ -v openshell-state:/var/openshell \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/podman.sock" \ - -e OPENSHELL_DRIVERS=podman \ + -e OPENSHELL_COMPUTE_DRIVER=podman \ -e OPENSHELL_PODMAN_SOCKET=/var/run/podman.sock \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ -e OPENSHELL_DISABLE_TLS=true \ diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index f8bd7361a4..d71fa055bb 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -37,11 +37,11 @@ The Homebrew formula creates its prefix config without setting `bind_address`, s ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] # ... gateway-wide settings ... @@ -59,7 +59,39 @@ version = 1 # ... credential-driver-specific settings ... ``` -The canonical gateway selector is `compute_driver = ""`. The legacy `compute_drivers = [""]` list remains accepted for compatibility. An omitted selector or an empty legacy list retains auto-detection; a legacy list with multiple entries retains the existing startup error because only one compute driver can be active. +The gateway selector is `compute_driver = ""`. It accepts one scalar driver name. Omit it to retain auto-detection. + +## Migrate to schema version 2 + +Schema version 2 is an intentional breaking cutover. The gateway rejects files +that omit `[openshell] version`, declare version 1, or declare an unsupported +future version. To migrate an existing file: + +1. Set `[openshell] version = 2`. +2. Replace `compute_drivers = [""]` with the scalar + `compute_driver = ""`. Replace `--drivers` and `OPENSHELL_DRIVERS` + with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. +3. Move every compute-driver option into `[openshell.drivers.]`. Schema + version 2 does not inherit driver defaults from `[openshell.gateway]`. + Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway + scope; set all three or omit all three when TLS is disabled. +4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman + `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM + `openshell_endpoint` to `grpc_endpoint`. +5. Use canonical image pull policies: `always`, `if_not_present`, `never`, or + Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` + spelling are rejected. +6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, + omit Docker or Podman `sandbox_pids_limit` for the runtime default, and omit + Podman `health_check_interval_secs` to disable health checks. Explicit zero + values are invalid. +7. Remove `grpc_endpoint` when the topology-derived callback is correct, or + retain it as an explicit override. New VM root filesystems use UID/GID 1000; + existing persisted VM state using 10001 remains compatible. + +Unknown fields and non-table `[openshell.drivers.]` values fail startup. +This strict validation prevents misspelled or misplaced security-sensitive +settings from being silently ignored. ## Full Example @@ -70,7 +102,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] name = "production-us-west" @@ -104,14 +136,8 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Shared driver defaults. These inherit into [openshell.drivers.] tables -# when the driver-specific table does not override them. -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -host_gateway_ip = "10.0.0.1" -sa_token_ttl_secs = 3600 +# Guest TLS paths remain gateway settings. Set all three for TLS, or omit all +# three only when TLS is disabled. Driver tables must not repeat these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -156,7 +182,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. +# Omit only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -201,8 +227,14 @@ phases = ["validate"] [openshell.drivers.kubernetes] namespace = "openshell" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" enable_user_namespaces = false +sa_token_ttl_secs = 3600 [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" @@ -215,7 +247,7 @@ Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -359,7 +391,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. +`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. ## Credential Drivers @@ -448,9 +480,9 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. -Canonical Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical gateway-level locations remain accepted as compatibility inputs and retain the same lower precedence. Gateway-level `sandbox_namespace` also remains a compatibility default for Docker `sandbox_label`. +Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. ### Kubernetes @@ -458,7 +490,7 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" @@ -488,11 +520,11 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. @@ -527,6 +559,8 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true +# Optional override. When omitted, the gateway derives +# https://openshell-gateway..svc:. grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" @@ -597,34 +631,36 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" compute_driver = "docker" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -# Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. -image_pull_policy = "IfNotPresent" +# Canonical values: always | if_not_present | never. `newer` is Podman-only. +image_pull_policy = "if_not_present" # Value assigned to the openshell.sandbox_namespace label on sandbox containers. sandbox_label = "docker-dev" -# Empty auto-detects https://host.openshell.internal: when guest TLS is set. +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" ssh_socket_path = "/run/openshell/ssh.sock" @@ -632,26 +668,42 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. +# Omit to leave Docker's runtime default unchanged. Explicit 0 is invalid. sandbox_pids_limit = 2048 +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" +# Corporate TLS egress proxy. These are supervisor argv settings, not workload +# environment variables. Do not embed credentials in the URL. +https_proxy = "https://proxy.corp.example:8443" +no_proxy = ".svc.cluster.local,10.0.0.0/8" +# Optional root-owned host file containing user:pass. An http:// proxy also +# requires proxy_auth_allow_insecure = true as an explicit acknowledgement. +proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# Project a host Unix Workload API socket into the supervisor for provider +# token exchange. The socket parent must be a dedicated absolute directory. +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` -Use `sandbox_label` for new Docker configurations. The legacy -`sandbox_namespace` key remains accepted as a compatibility alias. Do not set -both keys in the same driver table. +Use `sandbox_label` for Docker configurations. The legacy +`sandbox_namespace` key is rejected. ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" compute_driver = "podman" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -660,7 +712,8 @@ compute_driver = "podman" # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer +# Optional override. When omitted, the gateway derives this endpoint. grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -672,18 +725,15 @@ ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Podman's runtime default unchanged. +# Omit to leave Podman's runtime default unchanged. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Health check interval in seconds. Lower values detect readiness faster -# but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Health check interval in seconds. Omit to disable health checks; explicit 0 +# is invalid. Lower values detect readiness faster but increase process churn +# (each check spawns a conmon subprocess). health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -772,11 +822,17 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# Project a host Workload API Unix socket into the supervisor, or use an +# explicit container-reachable TCP endpoint, for provider token exchange. +# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" +# Explicit supervisor-compatible default. RuntimeDefault and Localhost/ +# require Podman to report AppArmor support. +app_armor_profile = "Unconfined" ``` -Use `ssh_socket_path` for new Podman configurations. The legacy -`sandbox_ssh_socket_path` key remains accepted as a compatibility alias. Do not -set both keys in the same driver table. +Use `ssh_socket_path` for Podman configurations. The legacy +`sandbox_ssh_socket_path` key is rejected. ### MicroVM @@ -784,31 +840,35 @@ Each sandbox runs inside its own libkrun microVM managed by the standalone `open ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. compute_driver = "vm" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -grpc_endpoint = "https://host.containers.internal:17670" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for the VM topology. +grpc_endpoint = "https://host.openshell.internal:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" -# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. -# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. +# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. +# Defaults to 1000 when unset; matching GID is used if sandbox_gid is empty. +# Existing persisted VM rootfs/overlays with UID 10001 retain that identity. # Any non-root Linux UID/GID is valid. # sandbox_uid = 20001 # Corporate forward proxy for sandbox egress. The keys, their semantics, and @@ -855,6 +915,11 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# VM guests cannot mount a host Workload API Unix socket. Configure only a +# separately operated guest-reachable TCP listener and explicitly acknowledge +# the exposure; host-only sockets are never exposed automatically. +# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" +# provider_spiffe_allow_guest_tcp = true ``` ### Extension Driver @@ -867,7 +932,7 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index ef7c93a017..a8bda78b2f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -54,9 +54,9 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. -The legacy `compute_drivers = [""]` list remains accepted for compatibility. Empty legacy lists retain auto-detection, and lists with more than one entry retain the existing startup error because a gateway supports exactly one active compute driver. +`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. Common gateway options: @@ -83,8 +83,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -342,7 +342,7 @@ Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. @@ -400,13 +400,13 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | | `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | +| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 878aee677c..63e587ef62 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" @@ -18,10 +18,10 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index 2064a081f5..6eff3b5615 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" @@ -18,12 +18,13 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.podman] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 5d650bae89..4bda320440 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -11,16 +11,16 @@ ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes -ENV OPENSHELL_DRIVERS=kubernetes \ +ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ OPENSHELL_GATEWAY_ID=openshell \ OPENSHELL_SANDBOX_NAMESPACE=openshell \ OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ - OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ OPENSHELL_K8S_TOPOLOGY=combined diff --git a/e2e/run.sh b/e2e/run.sh index 9764fde979..8d9cee8dac 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -144,9 +144,6 @@ gateway_driver="$(python3 -c ' import sys, tomllib gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] driver = gateway.get("compute_driver") -if driver is None: - drivers = gateway.get("compute_drivers", []) - driver = drivers[0] if drivers else None if not driver: raise SystemExit("gateway config must explicitly select a compute driver") print(driver) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 3081adf988..4bc2cfe1fa 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -256,11 +256,14 @@ e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" cat >"${GATEWAY_CONFIG}" <&2 exit 2 @@ -501,8 +501,11 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" { - printf '[openshell]\nversion = 1\n\n' - printf '[openshell.gateway]\nlog_level = "info"\n\n' + printf '[openshell]\nversion = 2\n\n' + printf '[openshell.gateway]\nlog_level = "info"\n' + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-docker-${HOST_PORT}" if [ "${OIDC_MODE}" != "1" ]; then e2e_write_gateway_mtls_auth_config @@ -519,9 +522,6 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then @@ -560,7 +560,7 @@ GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" - --drivers docker + --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 089b9923fd..efc829cb62 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -459,15 +459,22 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" -{ - e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" - if [ "${OIDC_MODE}" != "1" ]; then - e2e_write_gateway_mtls_auth_config - if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" - fi +# The TLS listener credentials are supplied by CLI below. Schema v2 keeps the +# supervisor client bundle gateway-owned, so add it to [openshell.gateway] +# before the RPM template opens the Podman driver table. +GATEWAY_CONFIG_WITH_TLS="${GATEWAY_CONFIG}.tls" +while IFS= read -r line; do + if [ "${line}" = "[openshell.drivers.podman]" ]; then + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" fi - printf '\n[openshell.drivers.podman]\n' + printf '%s\n' "${line}" +done <"${GATEWAY_CONFIG}" >"${GATEWAY_CONFIG_WITH_TLS}" +mv "${GATEWAY_CONFIG_WITH_TLS}" "${GATEWAY_CONFIG}" +{ + # The RPM template ends in [openshell.drivers.podman]. Append driver-owned + # overrides before opening any nested gateway tables below. if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else @@ -475,14 +482,12 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" printf 'gateway_port = %s\n' "${HOST_PORT}" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = "missing"\n' + printf 'image_pull_policy = "if_not_present"\n' + # The RPM template already opts into the 10-second Podman health check. # Keep CI teardown bounded while the production Podman driver default stays # conservative for real user workloads. printf 'stop_timeout_secs = %s\n' "${PODMAN_STOP_TIMEOUT_SECS}" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' if [ -n "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" ]; then printf 'provider_spiffe_workload_api_socket = %s\n' "$(toml_string "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" @@ -496,13 +501,22 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" fi fi + + e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" + if [ "${OIDC_MODE}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then + e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" + fi + fi } >> "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="if_not_present" \ + OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -549,6 +563,7 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_LOG}" \ "${GATEWAY_PID_FILE}" +OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" >"${GATEWAY_LOG}" 2>&1 & diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index f6f0b10204..94b73c73cc 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -87,13 +87,11 @@ if your gateway cache directory differs): ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] compute_driver = "podman" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" disable_tls = true -supervisor_image = "localhost/openshell/supervisor:dev" [openshell.gateway.auth] allow_unauthenticated_users = true @@ -106,7 +104,10 @@ gateway_id = "podman-dev" ttl_secs = 3600 [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +supervisor_image = "localhost/openshell/supervisor:dev" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 ``` If the JWT key files do not exist yet, run `mise run gateway` once to generate @@ -118,7 +119,7 @@ Start the gateway: eval "$(aws configure export-credentials --format env)" ./target/debug/openshell-gateway \ --config .cache/gateway-podman/gateway.toml \ - --port 18080 --log-level info --drivers podman --disable-tls \ + --port 18080 --log-level info --compute-driver podman --disable-tls \ --db-url "sqlite:.cache/gateway-podman/gateway.db?mode=rwc" ``` diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index e25ee0d3cb..8e777f921f 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -325,7 +325,7 @@ generate_gateway_jwt_bundle() { write_gateway_config() { cat >"$GATEWAY_CONFIG" <"$config_path" <"$config_path" <"$GATEWAY_CONFIG" < None: ) assert "EnvironmentFile=-%%E/openshell/gateway.env" in spec assert "%%S/openshell/tls" not in spec - assert "Environment=OPENSHELL_DRIVERS" not in spec + assert "Environment=OPENSHELL_COMPUTE_DRIVER" not in spec assert "Environment=OPENSHELL_BIND_ADDRESS" not in spec assert "Environment=OPENSHELL_PODMAN_TLS_CA" not in spec assert "ExecStart=/usr/bin/openshell-gateway" in spec diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index 41aac552c2..e544e844da 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. +The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 1 # optional; reserved for future schema migrations +version = 2 # required schema version # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -93,6 +93,11 @@ enable_loopback_service_http = true # ignores the [openshell.gateway.tls] table below. disable_tls = false +# Gateway-owned TLS bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" + [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -118,9 +123,9 @@ scopes_claim = "" # empty disables scope enforcement [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -128,26 +133,20 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -156,9 +155,6 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -166,7 +162,7 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). +2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. @@ -208,17 +204,17 @@ The following cross-field validations are applied after merging file + env + CLI - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. - When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list remains accepted: an empty list auto-detects, a singleton selects that driver, and multiple entries retain the existing startup error. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list is rejected. -### Backwards compatibility +### Schema compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). Legacy `compute_drivers = [""]` TOML remains accepted, while canonical configurations use the singular `compute_driver = ""`. +Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index aea67ba528..c981ec9d93 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -84,12 +84,20 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_driver|compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n '^version|compute_driver|socket_path|guest_tls_' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 ``` +Gateway configuration requires `[openshell] version = 2`, a singular +`compute_driver` selector, and driver-owned settings under +`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, +`--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating +them. Guest TLS CA, certificate, and key paths are the exception: configure the +complete bundle under `[openshell.gateway]`, and the gateway injects it only +into the selected local driver. + Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. For configured gateway interceptors, inspect `[[openshell.gateway.interceptors]]`, their Unix or network endpoints, and gateway startup logs: diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 2695566476..d6c6e1efe6 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -29,7 +29,7 @@ GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" @@ -52,6 +52,18 @@ linux_target_triple() { esac } +# Escape a value for a TOML basic string before copying an operator-provided +# proxy path or URL into the generated local configuration. +toml_escape() { + local s=$1 + s=${s//\\/\\\\} + s=${s//\"/\\\"} + s=${s//$'\n'/\\n} + s=${s//$'\r'/\\r} + s=${s//$'\t'/\\t} + printf '%s' "${s}" +} + port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -211,7 +223,7 @@ mkdir -p "${STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" < only on a Docker host with AppArmor enabled. +app_armor_profile = "Unconfined" EOF +# Keep the local task's proxy inputs aligned with [openshell.drivers.docker]. +# Credentials stay in the referenced root-owned file; do not echo their value. +if [[ -n "${OPENSHELL_SANDBOX_HTTPS_PROXY+x}" ]]; then + printf 'https_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_HTTPS_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then + printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -256,6 +292,6 @@ exec "${GATEWAY_BIN}" \ --config "${CONFIG_PATH}" \ --port "${PORT}" \ --log-level "${LOG_LEVEL}" \ - --drivers docker \ + --compute-driver docker \ --disable-tls \ --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index 2b9d9bc349..d1d86ad4a2 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -26,7 +26,7 @@ GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -90,19 +90,6 @@ ensure_podman_supervisor_image() { fi } -podman_pull_policy() { - case "$1" in - Always|always) echo "always" ;; - IfNotPresent|ifnotpresent|missing|"") echo "missing" ;; - Never|never) echo "never" ;; - Newer|newer) echo "newer" ;; - *) - echo "ERROR: unsupported Podman image pull policy '$1'" >&2 - exit 2 - ;; - esac -} - # Escape a value for embedding in a double-quoted TOML basic string, so # quotes, backslashes, or control characters in an environment value cannot # corrupt gateway.toml or inject extra configuration keys. @@ -215,7 +202,7 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" </dev/null 2>&1; then @@ -336,7 +347,7 @@ chmod 700 "${VM_DRIVER_STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT+x}" ]]; then + printf 'provider_spiffe_workload_api_tcp_endpoint = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then + printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -386,7 +422,7 @@ GATEWAY_ARGS=( --config "${CONFIG_PATH}" --port "${PORT}" --log-level "${LOG_LEVEL}" - --drivers vm + --compute-driver vm --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index cffad5ae2b..da7f91fb68 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -10,7 +10,7 @@ # # VM/MicroVM is intentionally explicit-only because it requires runtime setup. # Use either: -# OPENSHELL_DRIVERS=vm mise run gateway +# OPENSHELL_COMPUTE_DRIVER=vm mise run gateway # mise run gateway:vm set -euo pipefail @@ -33,7 +33,7 @@ Options: -h, --help Show this help. Environment: - OPENSHELL_DRIVERS Driver override used by openshell-gateway. + OPENSHELL_COMPUTE_DRIVER Driver override used by openshell-gateway. OPENSHELL_GATEWAY_NAME Gateway name for delegated or Kubernetes runs. OPENSHELL_BIND_ADDRESS Gateway listener address. Defaults to 127.0.0.1, or ::1 for Podman Machine on macOS. @@ -104,7 +104,7 @@ detect_driver() { fi echo "ERROR: no compute driver detected." >&2 - echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_DRIVERS." >&2 + echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_COMPUTE_DRIVER." >&2 exit 2 } @@ -171,17 +171,17 @@ while [[ "$#" -gt 0 ]]; do esac done -if [[ -n "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - echo "ERROR: use either --driver or OPENSHELL_DRIVERS, not both" >&2 +if [[ -n "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + echo "ERROR: use either --driver or OPENSHELL_COMPUTE_DRIVER, not both" >&2 exit 2 fi -if [[ -z "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - if [[ "${OPENSHELL_DRIVERS}" == *,* ]]; then - echo "ERROR: mise run gateway supports one driver; got OPENSHELL_DRIVERS=${OPENSHELL_DRIVERS}" >&2 +if [[ -z "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + if [[ "${OPENSHELL_COMPUTE_DRIVER}" == *,* ]]; then + echo "ERROR: mise run gateway supports one driver; got OPENSHELL_COMPUTE_DRIVER=${OPENSHELL_COMPUTE_DRIVER}" >&2 exit 2 fi - explicit_driver="$(normalize_driver "${OPENSHELL_DRIVERS}")" + explicit_driver="$(normalize_driver "${OPENSHELL_COMPUTE_DRIVER}")" fi DRIVER="${explicit_driver:-$(detect_driver)}" @@ -206,7 +206,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -244,12 +244,11 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" <"$config" < Date: Tue, 1 Sep 2026 18:13:51 -0400 Subject: [PATCH 04/42] fix(config): preserve compute driver runtime guarantees Signed-off-by: Jesse Jaggars --- architecture/gateway.md | 5 +- crates/openshell-core/src/config.rs | 19 +- crates/openshell-core/src/driver_utils.rs | 19 +- crates/openshell-driver-docker/README.md | 2 +- crates/openshell-driver-docker/src/lib.rs | 11 +- crates/openshell-driver-docker/src/tests.rs | 10 + crates/openshell-driver-kubernetes/README.md | 7 +- .../openshell-driver-kubernetes/src/main.rs | 25 +- crates/openshell-driver-podman/README.md | 2 +- crates/openshell-driver-podman/src/config.rs | 28 +- .../openshell-driver-podman/src/container.rs | 56 +++- crates/openshell-driver-podman/src/main.rs | 10 +- crates/openshell-driver-podman/src/watcher.rs | 26 +- crates/openshell-driver-vm/README.md | 4 +- .../scripts/openshell-vm-sandbox-init.sh | 97 ++++--- crates/openshell-driver-vm/src/driver.rs | 239 ++++++++++++++---- crates/openshell-driver-vm/src/rootfs.rs | 88 ++++++- .../openshell/tests/grpc_endpoint_test.yaml | 26 ++ deploy/rpm/CONFIGURATION.md | 2 +- docs/reference/gateway-config.mdx | 22 +- docs/reference/sandbox-compute-drivers.mdx | 12 +- 21 files changed, 539 insertions(+), 171 deletions(-) create mode 100644 deploy/helm/openshell/tests/grpc_endpoint_test.yaml diff --git a/architecture/gateway.md b/architecture/gateway.md index 59a4d42b8e..809de088ea 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -794,7 +794,10 @@ system entry instead of pretending to delete package-manager owned state. - Gateway TLS and client certificate distribution are deployment concerns owned by the operator or packaging layer. - Compute runtimes own the mechanics of starting workloads and injecting - callback configuration. + callback configuration. Local Docker, Podman, and VM callback endpoints can + be derived from their fixed host aliases. Kubernetes requires an explicit + endpoint from deployment topology; Helm renders it from the gateway Service + name and namespace rather than inferring it from sandbox placement. - Docker-backed local gateways use Docker's `host-gateway` callback alias on macOS and Docker Desktop-style runtimes. They request IPv4 loopback callback reachability and add a listener only when the primary does not cover it. diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3d5c2d0709..b5429d6d19 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -8,7 +8,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::fmt; use std::net::SocketAddr; -use std::num::NonZeroU64; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -31,6 +31,18 @@ pub const DEFAULT_GATEWAY_NAME: &str = "openshell"; /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; +/// Default cgroup PID limit for local container sandboxes. +pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; + +/// Typed default cgroup PID limit for local container sandboxes. +#[must_use] +pub fn default_sandbox_pids_limit() -> Option { + NonZeroI64::new(DEFAULT_SANDBOX_PIDS_LIMIT) +} + +/// Default Docker bridge network name for local sandboxes. +pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; + /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; @@ -109,11 +121,6 @@ pub fn resolve_supervisor_image_tag(candidates: &[&str]) -> String { /// CDI device identifier for requesting all NVIDIA GPUs. pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; -/// Default maximum number of processes (PIDs) allowed inside a sandbox container. -/// -/// Compute drivers may override this through backend configuration. -pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; - /// Normalize a configured compute driver name. /// /// Built-in driver names and custom remote driver names share the same diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 03f3adf45a..7182f811d5 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -10,9 +10,7 @@ use crate::proto::compute::v1::DriverSandbox; /// Built-in sandbox network topologies used to derive a callback endpoint /// when an operator does not configure a per-driver `grpc_endpoint` override. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GatewayCallbackTopology<'a> { - /// A sandbox pod reaches the gateway through its Kubernetes service. - Kubernetes { namespace: &'a str }, +pub enum GatewayCallbackTopology { /// A Docker container reaches the host through Docker's gateway alias. Docker, /// A Podman container reaches the host through Podman's gateway alias. @@ -28,15 +26,12 @@ pub enum GatewayCallbackTopology<'a> { /// operator override for remote or non-standard deployments. #[must_use] pub fn gateway_callback_endpoint( - topology: GatewayCallbackTopology<'_>, + topology: GatewayCallbackTopology, gateway_port: u16, gateway_tls_enabled: bool, ) -> String { let scheme = if gateway_tls_enabled { "https" } else { "http" }; let host = match topology { - GatewayCallbackTopology::Kubernetes { namespace } => { - return format!("{scheme}://openshell-gateway.{namespace}.svc:{gateway_port}"); - } GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", GatewayCallbackTopology::Podman => "host.containers.internal", }; @@ -61,16 +56,6 @@ mod callback_endpoint_tests { gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), "https://host.openshell.internal:17670" ); - assert_eq!( - gateway_callback_endpoint( - GatewayCallbackTopology::Kubernetes { - namespace: "agents" - }, - 8080, - true, - ), - "https://openshell-gateway.agents.svc:8080" - ); } } diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 31c59bc6f3..10e085ecf0 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Omit `[openshell.drivers.docker].sandbox_pids_limit` to inherit the Docker/runtime default; explicit `0` is invalid. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. `[openshell.drivers.docker].sandbox_pids_limit` defaults to `2048`; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 617177f516..d8756bc135 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -162,9 +162,12 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Omit the field to leave Docker's runtime/default PID limit unchanged. - /// Explicit zero is invalid. - #[serde(default, skip_serializing_if = "Option::is_none")] + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through @@ -203,7 +206,7 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: None, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, upstream_proxy: UpstreamProxyConfig::default(), provider_spiffe_workload_api_socket: None, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index e04b79f87b..8c3ac54e31 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -150,6 +150,16 @@ fn docker_config_rejects_legacy_sandbox_namespace() { assert!(error.to_string().contains("sandbox_namespace")); } +#[test] +fn docker_config_defaults_to_driver_owned_pids_limit() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!( + config.sandbox_pids_limit.map(std::num::NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); +} + #[test] fn docker_config_rejects_invalid_pids_limits() { let zero = serde_json::from_value::(serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 5d6154bd17..b64bf0c6e4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -96,8 +96,11 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. Driver-owned -values must override image-provided environment variables. +material, and the supervisor SSH socket path into the workload. The callback +endpoint is required because the sandbox namespace does not identify the +Gateway Service; Helm renders it from the release topology, while standalone +and raw TOML configurations must set it explicitly. Driver-owned values must +override image-provided environment variables. Sandbox pods run as `service_account_name` and keep `automountServiceAccountToken: false`. The only Kubernetes token exposed to the diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 3047a9b108..d949d6c72c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -8,7 +8,6 @@ use std::net::SocketAddr; use std::path::PathBuf; use tracing::info; -use openshell_core::driver_utils::{GatewayCallbackTopology, gateway_callback_endpoint}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::{ @@ -95,8 +94,10 @@ struct Args { )] managed_ssh_gateway_pod_selector: Vec, + /// Gateway callback endpoint reachable from sandbox pods. Kubernetes + /// service topology cannot be inferred from the sandbox namespace. #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, + grpc_endpoint: String, #[arg( long, @@ -240,15 +241,6 @@ async fn main() -> Result<()> { .collect::>>()?; let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); - let grpc_endpoint = args.grpc_endpoint.unwrap_or_else(|| { - gateway_callback_endpoint( - GatewayCallbackTopology::Kubernetes { - namespace: &args.sandbox_namespace, - }, - openshell_core::config::DEFAULT_SERVER_PORT, - false, - ) - }); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { workspace_mode: args.workspace_mode, @@ -282,7 +274,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint, + grpc_endpoint: args.grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), @@ -345,6 +337,13 @@ async fn main() -> Result<()> { mod tests { use super::*; + #[test] + fn requires_explicit_gateway_callback_endpoint() { + let error = Args::try_parse_from(["openshell-driver-kubernetes"]) + .expect_err("Kubernetes service topology must be explicit"); + assert!(error.to_string().contains("--grpc-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ @@ -353,6 +352,8 @@ mod tests { "http://collector.example:4317", "--gateway-name", "kubernetes-dev", + "--grpc-endpoint", + "http://openshell.example:8080", ]) .expect("OTLP endpoint should parse"); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index cdb8f5d78c..c2fb5b44c1 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -387,7 +387,7 @@ Podman resources after out-of-band container removal or label drift. | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | unset | Podman cgroup PID limit for sandbox containers. Omit it to inherit Podman's runtime/default PID limit; explicit `0` is invalid. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 196c1fa571..c71df3a727 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -77,9 +77,12 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Omit the field to leave Podman's runtime/default PID limit unchanged. - /// Explicit zero is invalid. - #[serde(default, skip_serializing_if = "Option::is_none")] + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. @@ -446,7 +449,7 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: None, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, app_armor_profile: Some(AppArmorProfile::Unconfined), @@ -545,12 +548,25 @@ mod tests { } #[test] - fn default_config_uses_runtime_pids_limit() { + fn default_config_sets_driver_owned_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, None); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); assert!(!cfg.enable_bind_mounts); } + #[test] + fn omitted_pids_limit_uses_driver_owned_default() { + let cfg: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Podman config should deserialize"); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); + } + #[test] #[cfg(target_os = "macos")] fn default_config_uses_gvproxy_host_gateway_ip_on_macos() { diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1d0fce6f2a..4918c9210a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,11 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, - #[serde(skip_serializing_if = "Vec::is_empty")] - security_opt: Vec, + /// Podman's container create API accepts `AppArmor` through the dedicated + /// `apparmor_profile` `SpecGenerator` field. This is not Docker's + /// `security_opt` representation. + #[serde(skip_serializing_if = "Option::is_none")] + apparmor_profile: Option, image_pull_policy: String, #[serde(skip_serializing_if = "Option::is_none")] healthconfig: Option, @@ -939,6 +942,14 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } +fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { + match profile { + None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, + Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), + Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), + } +} + /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1174,12 +1185,7 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), - security_opt: config - .app_armor_profile - .as_ref() - .and_then(openshell_core::AppArmorProfile::oci_security_opt) - .into_iter() - .collect(), + apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), image_pull_policy: "never".to_string(), healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ @@ -1591,6 +1597,40 @@ mod tests { assert!(spec["resource_limits"].get("PidsLimit").is_none()); } + #[test] + fn container_spec_uses_podman_apparmor_profile_field() { + let sandbox = test_sandbox("test-id", "test-name"); + + for (profile, expected) in [ + (openshell_core::AppArmorProfile::Unconfined, "unconfined"), + ( + openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "openshell-supervisor", + ), + ] { + let mut config = test_config(); + config.app_armor_profile = Some(profile); + let spec = build_container_spec(&sandbox, &config); + + assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); + assert!(spec.get("security_opt").is_none()); + } + } + + #[test] + fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { + let sandbox = test_sandbox("test-id", "test-name"); + + for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { + let mut config = test_config(); + config.app_armor_profile = profile; + let spec = build_container_spec(&sandbox, &config); + + assert!(spec.get("apparmor_profile").is_none()); + assert!(spec.get("security_opt").is_none()); + } + } + #[test] fn container_name_is_workspace_qualified() { assert_eq!( diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 0f962fce5e..e4554602f4 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -87,9 +87,9 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Omit to inherit - /// Podman's runtime/default PID limit. - #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT")] + /// Container cgroup PID limit for sandbox containers. Omit to use + /// `OpenShell`'s 2048-process default. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT", default_value = "2048")] sandbox_pids_limit: Option, /// Health check interval in seconds. Omit it in gateway TOML to disable @@ -331,6 +331,10 @@ mod tests { defaults.health_check_interval_secs.map(NonZeroU64::get), Some(10) ); + assert_eq!( + defaults.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index c3903a0067..257d649a76 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -439,7 +439,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { Some(HealthState { status }) if status == "starting" => { ("False", "HealthCheckStarting", String::new()) } - _ => ("False", CONDITION_STARTING, String::new()), + None => ( + "True", + CONDITION_RUNNING, + "Container is running".to_string(), + ), + Some(_) => ("False", CONDITION_STARTING, String::new()), }, "created" => ("False", "ContainerCreated", String::new()), "exited" | "stopped" => { @@ -578,6 +583,25 @@ mod tests { assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); } + #[test] + fn condition_running_without_healthcheck_is_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: None, + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let cond = condition_from_state(&state); + assert_eq!(cond.r#type, "Ready"); + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, CONDITION_RUNNING); + assert_eq!(cond.message, "Container is running"); + assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); + } + #[test] fn condition_oom_killed() { let state = ContainerState { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 8646beb95b..ec57edfbbc 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,12 +152,12 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | image account or `1000` / UID | Explicit values override the image account. When omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`; persisted legacy identity is retained when recorded in sandbox state. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Existing overlay state without the per-sandbox identity marker is restored as legacy `10001:10001`. | | `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend has no such NAT, so a gateway-host proxy URL is rejected before GPU sandbox launch; use an address routable from the guest's masqueraded egress. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | | `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | | `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | -| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. | +| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP CONNECT targets. | | `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor through a protected per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index b8001efb2c..de2d993913 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -29,7 +29,6 @@ BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) GVPROXY_GATEWAY_IP="192.168.127.1" GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" GATEWAY_IP="$GVPROXY_GATEWAY_IP" -SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized" GPU_ENABLED="${GPU_ENABLED:-false}" VM_NET_IP="${VM_NET_IP:-}" @@ -105,8 +104,21 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" - local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-1000}" - local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-$sandbox_uid}" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local replace_account=0 + + # An omitted identity means the image owns its sandbox account contract. + # Fall back to 1000 only when the image has no sandbox account at all. + if [ -n "$sandbox_uid" ] || [ -n "$sandbox_gid" ]; then + sandbox_uid="${sandbox_uid:-1000}" + sandbox_gid="${sandbox_gid:-$sandbox_uid}" + replace_account=1 + elif ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sandbox_uid=1000 + sandbox_gid=1000 + replace_account=1 + fi mkdir -p \ "$image_root/srv" \ @@ -123,39 +135,28 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - # This is a newly prepared target image, so replace a baked-in legacy - # sandbox account with the identity selected by the driver. Persisted - # overlays do not take this path; setup_sandbox_workdir preserves their - # existing 10001:10001 account instead. - if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" - else - printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" - fi - if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then - printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" - fi - if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" - else - printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" - fi - if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then - printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + if [ "$replace_account" -eq 1 ]; then + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" + fi + if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then + printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" + fi + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" + fi + if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then + printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + fi fi local owner - local owner_normalized=0 owner="$(sandbox_owner_for_root "$image_root")" - if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - elif chown -R 1000:1000 "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - fi + chown -R "$owner" "$image_root/sandbox" 2>/dev/null || chown -R 1000:1000 "$image_root/sandbox" || true chmod 0755 "$image_root/sandbox" - if [ "$owner_normalized" -eq 1 ]; then - mkdir -p "$image_root/opt/openshell" - printf '1\n' > "$image_root${SANDBOX_OWNER_NORMALIZED_MARKER}" - fi } prepare_guest_image_rootfs() { @@ -619,6 +620,34 @@ setup_gpu() { fi } +reconcile_sandbox_account() { + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local etc + + [ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0 + [[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || { + ts "FATAL: invalid requested sandbox identity" + exit 1 + } + etc="$(root_path /etc)" + mkdir -p "$etc" + touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" + if grep -q '^sandbox:' "$etc/group"; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" + fi + if grep -q '^sandbox:' "$etc/passwd"; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" + fi + grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow" + grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow" + ts "reconciled sandbox account (${sandbox_uid}:${sandbox_gid})" +} + setup_sandbox_workdir() { local sandbox_dir local owner @@ -630,8 +659,7 @@ setup_sandbox_workdir() { if [ "$owner" = "10001:10001" ]; then ts "preserving legacy sandbox ownership (10001:10001)" fi - if [ "$current_owner" != "$owner" ] \ - || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then + if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then chown -R 1000:1000 "$sandbox_dir" fi @@ -760,6 +788,7 @@ run_post_overlay_setup() { echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true fi + reconcile_sandbox_account setup_sandbox_workdir configure_hostname diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d827f5ec3d..d9a6ff49fc 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -184,6 +184,8 @@ const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4"; const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; +const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; +const SANDBOX_OWNER_STATE_VERSION: &str = "sandbox-owner-v1"; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -262,13 +264,12 @@ pub struct VmDriverConfig { pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for newly prepared rootfs `/etc/passwd` entries. - /// When empty, new sandboxes use 1000. Existing rootfs and overlays retain - /// their recorded sandbox account for legacy 10001 compatibility. + /// Optional UID override for the sandbox account in newly prepared rootfs images. + /// When both identity fields are empty, an image-provided sandbox account is preserved. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, - /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. - /// When empty, defaults to the resolved UID. + /// Optional GID override for rootfs `/etc/passwd` and `/etc/group` entries. + /// When one override is supplied, its missing counterpart defaults to the UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, } @@ -330,11 +331,8 @@ impl std::fmt::Debug for VmDriverConfig { } } -/// Default sandbox UID used when preparing new VM rootfs images. -/// -/// The guest-init script detects an existing `sandbox` account and preserves -/// its UID/GID, so persisted rootfs and overlays prepared with legacy UID 10001 -/// continue to start without an ownership migration. +/// Fallback sandbox UID for images without a `sandbox` account and partial +/// operator identity overrides. pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { @@ -366,12 +364,12 @@ impl Default for VmDriverConfig { } impl VmDriverConfig { - /// Resolve the sandbox UID, falling back to `DEFAULT_SANDBOX_UID`. + /// Resolve a fallback sandbox UID for an image that has no sandbox account. pub fn resolve_sandbox_uid(&self) -> u32 { self.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID) } - /// Resolve the sandbox GID, falling back to the resolved UID. + /// Resolve a fallback sandbox GID from the selected UID. pub fn resolve_sandbox_gid(&self, resolved_uid: u32) -> u32 { self.sandbox_gid.unwrap_or(resolved_uid) } @@ -506,6 +504,27 @@ enum OverlayPreparation { PreserveExisting, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SandboxOwnerState { + Current, + Legacy, +} + +impl SandboxOwnerState { + fn guest_environment(self) -> Option<[String; 2]> { + match self { + Self::Current => None, + // A state directory with an overlay but no state marker predates + // the 1000 migration. Its upperdir may contain 10001-owned files + // anywhere in the rootfs, not just /sandbox. + Self::Legacy => Some([ + "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), + "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), + ]), + } + } +} + fn provisioning_span( parent: &opentelemetry::Context, sandbox_id: &str, @@ -863,8 +882,9 @@ impl VmDriver { "Preparing writable VM overlay disk".to_string(), ), ); - if let Err(err) = self + let sandbox_owner_state = self .prepare_runtime_overlay( + &state_dir, &overlay_disk, tls_paths.as_ref(), sandbox @@ -875,11 +895,7 @@ impl VmDriver { overlay_preparation, ) .await - { - return Err(Status::internal(format!( - "prepare guest overlay disk failed: {err}" - ))); - } + .map_err(|err| Status::internal(format!("prepare guest overlay disk failed: {err}")))?; self.ensure_provisioning_active(&sandbox.id).await?; if let Err(err) = @@ -1090,6 +1106,11 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } + if let Some(identity_env) = sandbox_owner_state.guest_environment() { + for env in identity_env { + command.arg("--vm-env").arg(env); + } + } info!( sandbox_id = %sandbox.id, @@ -2176,12 +2197,15 @@ impl VmDriver { )] async fn prepare_runtime_overlay( &self, + state_dir: &Path, overlay_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result<(), String> { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); + let (owner_state, write_owner_state) = + sandbox_owner_state_for_launch(state_dir, overlay_disk, preparation).await?; let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, @@ -2224,7 +2248,11 @@ impl VmDriver { }) .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; - span_status.finish(result) + result?; + if write_owner_state { + write_sandbox_owner_state(state_dir).await?; + } + span_status.finish(Ok(owner_state)) } async fn read_proxy_auth_credential(&self) -> Result, String> { @@ -2621,7 +2649,7 @@ impl VmDriver { image_identity: &str, bootstrap_root_disk: &Path, ) -> Result { - let cache_identity = prepared_image_cache_identity(image_identity); + let cache_identity = prepared_image_cache_identity(image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2732,7 +2760,7 @@ impl VmDriver { "failed to resolve vm sandbox image '{image_ref}': {err}" )) })?; - let cache_identity = prepared_image_cache_identity(&source_image_identity); + let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2964,14 +2992,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); - let resolved_uid = self.config.resolve_sandbox_uid(); - let resolved_gid = self.config.resolve_sandbox_gid(resolved_uid); - command - .arg("--vm-env") - .arg(format!("OPENSHELL_VM_SANDBOX_UID={resolved_uid}")); - command - .arg("--vm-env") - .arg(format!("OPENSHELL_VM_SANDBOX_GID={resolved_gid}")); + if let Some((uid, gid)) = configured_sandbox_identity(&self.config) { + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={gid}")); + } let mut child = command .spawn() @@ -3089,19 +3117,20 @@ impl VmDriver { let image_identity_owned = image_identity.to_string(); let exported_rootfs_for_build = exported_rootfs.clone(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!( - "Preparing VM rootfs for local image \"{image_ref}\" (sandbox uid={sandbox_uid})" - ), + format!("Preparing VM rootfs for local image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "local_docker".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -3230,17 +3259,20 @@ impl VmDriver { let image_ref_owned = image_ref.to_string(); let image_identity_owned = image_identity.to_string(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!("Preparing VM rootfs for image \"{image_ref}\" (sandbox uid={sandbox_uid})"), + format!("Preparing VM rootfs for image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "registry".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -5008,6 +5040,55 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } +/// Select the identity the guest must use for this overlay and whether a +/// successful preparation creates the state-version marker. A missing marker +/// is legacy only when an overlay already exists; an interrupted create with +/// no overlay is safe to initialize as current. +async fn sandbox_owner_state_for_launch( + state_dir: &Path, + overlay_disk: &Path, + preparation: OverlayPreparation, +) -> Result<(SandboxOwnerState, bool), String> { + if preparation == OverlayPreparation::Fresh { + return Ok((SandboxOwnerState::Current, true)); + } + + match tokio::fs::read_to_string(state_dir.join(SANDBOX_OWNER_STATE_FILE)).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_VERSION => { + Ok((SandboxOwnerState::Current, false)) + } + Ok(_) => Err(format!( + "sandbox owner state {} has an unsupported version", + state_dir.join(SANDBOX_OWNER_STATE_FILE).display() + )), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + match tokio::fs::metadata(overlay_disk).await { + Ok(_) => Ok((SandboxOwnerState::Legacy, false)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok((SandboxOwnerState::Current, true)) + } + Err(err) => Err(format!( + "stat overlay disk {}: {err}", + overlay_disk.display() + )), + } + } + Err(err) => Err(format!( + "read sandbox owner state {}: {err}", + state_dir.join(SANDBOX_OWNER_STATE_FILE).display() + )), + } +} + +async fn write_sandbox_owner_state(state_dir: &Path) -> Result<(), String> { + write_private_file( + &state_dir.join(SANDBOX_OWNER_STATE_FILE), + format!("{SANDBOX_OWNER_STATE_VERSION}\n").into_bytes(), + ) + .await + .map_err(|err| format!("write sandbox owner state: {err}")) +} + #[allow(clippy::result_large_err)] fn validate_sandbox_state_dir(root: &Path, state_dir: &Path) -> Result<(), Status> { let sandboxes_root = sandboxes_root_dir(root); @@ -5165,9 +5246,20 @@ fn bootstrap_image_cache_identity(image_identity: &str) -> String { ) } -fn prepared_image_cache_identity(image_identity: &str) -> String { +fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> { + (config.sandbox_uid.is_some() || config.sandbox_gid.is_some()).then(|| { + let uid = config.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + (uid, config.sandbox_gid.unwrap_or(uid)) + }) +} + +fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String { + let identity = configured_sandbox_identity(config).map_or_else( + || "image-account".to_string(), + |(uid, gid)| format!("configured-{uid}-{gid}"), + ); format!( - "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", + "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{identity}:{image_identity}", openshell_core::VERSION ) } @@ -6686,7 +6778,13 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay( + Path::new("/unused"), + Path::new("/unused"), + None, + None, + OverlayPreparation::Fresh, + ) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -7272,6 +7370,52 @@ mod tests { } } + #[tokio::test] + async fn legacy_overlay_state_uses_legacy_guest_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + + let (state, write_marker) = + sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) + .await + .unwrap(); + + assert_eq!(state, SandboxOwnerState::Legacy); + assert!(!write_marker); + assert_eq!( + state.guest_environment(), + Some([ + "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), + "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), + ]) + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn current_overlay_state_is_not_inferred_from_the_lower_rootfs() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"current overlay").unwrap(); + write_sandbox_owner_state(&dir).await.unwrap(); + + let (state, write_marker) = + sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) + .await + .unwrap(); + + assert_eq!(state, SandboxOwnerState::Current); + assert!(!write_marker); + assert_eq!( + std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), + "sandbox-owner-v1\n" + ); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") @@ -8467,9 +8611,9 @@ mod tests { #[test] fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { assert_eq!( - prepared_image_cache_identity("sha256:local-image"), + prepared_image_cache_identity("sha256:local-image", &VmDriverConfig::default()), format!( - "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:sha256:local-image", + "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:sha256:local-image", openshell_core::VERSION ) ); @@ -8505,7 +8649,10 @@ mod tests { &staging_dir, &GuestImagePayload { image_ref: "ghcr.io/example/app:latest".to_string(), - image_identity: prepared_image_cache_identity("sha256:abc"), + image_identity: prepared_image_cache_identity( + "sha256:abc", + &VmDriverConfig::default(), + ), source: GuestImagePayloadSource::RegistryOciLayout { layout_dir }, }, ) diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 81f68f6c44..2c84e547aa 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -16,8 +16,7 @@ const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; -const SANDBOX_OWNER_NORMALIZED_MARKER: &str = - openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const DEFAULT_SANDBOX_UID: u32 = 1000; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -31,8 +30,8 @@ pub const fn sandbox_guest_init_path() -> &'static str { pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, image_identity: &str, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { prepare_sandbox_rootfs(rootfs, sandbox_uid, sandbox_gid)?; validate_sandbox_rootfs(rootfs)?; @@ -353,7 +352,11 @@ fn append_symlink_to_archive( } #[allow(clippy::similar_names)] -fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Result<(), String> { +fn prepare_sandbox_rootfs( + rootfs: &Path, + sandbox_uid: Option, + sandbox_gid: Option, +) -> Result<(), String> { for relative in ["opt/openshell/.initialized", "opt/openshell/.rootfs-type"] { remove_rootfs_path(rootfs, relative)?; } @@ -567,8 +570,7 @@ fn normalize_sandbox_owner_in_rootfs_image(source: &Path, image_path: &Path) -> return Ok(()); } - run_debugfs_batch(image_path, &commands)?; - write_rootfs_image_file(image_path, SANDBOX_OWNER_NORMALIZED_MARKER, b"1\n") + run_debugfs_batch(image_path, &commands) } fn collect_sandbox_owner_commands( @@ -789,9 +791,18 @@ fn temporary_injection_path(image_path: &Path) -> PathBuf { #[allow(clippy::similar_names)] fn ensure_sandbox_guest_user( rootfs: &Path, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { + // An image's sandbox account is part of its filesystem contract. Leave it + // intact unless an operator explicitly configured either side of the + // identity. A missing account still gets the OpenShell default. + if sandbox_uid.is_none() && sandbox_gid.is_none() && sandbox_guest_user_ids(rootfs)?.is_some() { + return Ok(()); + } + + let sandbox_uid = sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + let sandbox_gid = sandbox_gid.unwrap_or(sandbox_uid); let etc_dir = rootfs.join("etc"); fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; @@ -983,7 +994,7 @@ mod tests { // Use a non-standard UID so the test doesn't collide with the default. let uid = 20001; - prepare_sandbox_rootfs(&rootfs, uid, uid).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(uid), Some(uid)).expect("prepare sandbox rootfs"); validate_sandbox_rootfs(&rootfs).expect("validate sandbox rootfs"); assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); @@ -1035,7 +1046,7 @@ mod tests { fs::create_dir_all(rootfs.join("sandbox")).expect("create sandbox workdir"); fs::write(rootfs.join("sandbox/app.py"), "print('hello')\n").expect("write app"); - prepare_sandbox_rootfs(&rootfs, 10001, 10001).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(10001), Some(10001)).expect("prepare sandbox rootfs"); assert!(rootfs.join("sandbox").is_dir()); assert_eq!( @@ -1115,6 +1126,61 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn sandbox_user_preserves_image_account_when_identity_is_omitted() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((4242, 4343))); + assert!( + fs::read_to_string(rootfs.join("etc/passwd")) + .unwrap() + .contains("Image:/image-home:/bin/false") + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_defaults_to_1000_when_image_has_no_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((1000, 1000))); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_explicit_identity_overrides_image_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, Some(2000), Some(3000)).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((2000, 3000))); + assert!( + fs::read_to_string(rootfs.join("etc/group")) + .unwrap() + .contains("sandbox:x:3000:") + ); + let _ = fs::remove_dir_all(dir); + } + #[test] fn sandbox_guest_user_ids_reads_existing_sandbox_user() { let dir = unique_temp_dir(); diff --git a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml new file mode 100644 index 0000000000..f84442f605 --- /dev/null +++ b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: Kubernetes gateway callback endpoint + +templates: + - templates/gateway-config.yaml + +release: + name: team-a + namespace: gateway-system + +tests: + - it: derives callback from the gateway Service when sandbox namespace differs + set: + server.sandboxNamespace: agent-sandboxes + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^namespace\s*=\s*"agent-sandboxes"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=\s*"https://team-a-openshell\.gateway-system\.svc\.cluster\.local:8080"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=.*agent-sandboxes' diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 45a813d0e8..ce20d28e25 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -218,7 +218,7 @@ overrides that persist across package upgrades. | `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | | `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | -| `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | +| `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d71fa055bb..0d2e50ff6b 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -82,12 +82,15 @@ future version. To migrate an existing file: Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` spelling are rejected. 6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, - omit Docker or Podman `sandbox_pids_limit` for the runtime default, and omit - Podman `health_check_interval_secs` to disable health checks. Explicit zero - values are invalid. -7. Remove `grpc_endpoint` when the topology-derived callback is correct, or - retain it as an explicit override. New VM root filesystems use UID/GID 1000; - existing persisted VM state using 10001 remains compatible. + omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit + of 2048, and omit Podman `health_check_interval_secs` to disable health + checks. Explicit zero values are invalid. +7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived + callback is correct, or retain it as an explicit override. Kubernetes raw + TOML requires an explicit endpoint; Helm derives one from the release's + gateway Service. New VM root filesystems use an image-provided `sandbox` + account when present and otherwise use UID/GID 1000. Existing persisted VM + state using 10001 remains compatible. Unknown fields and non-table `[openshell.drivers.]` values fail startup. This strict validation prevents misspelled or misplaced security-sensitive @@ -559,9 +562,10 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -# Optional override. When omitted, the gateway derives -# https://openshell-gateway..svc:. -grpc_endpoint = "https://openshell-gateway.agents.svc:8080" +# Required in raw gateway TOML because `namespace` identifies sandbox +# placement, not the gateway Service. Helm renders this from the release's +# gateway Service name and namespace. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index a8bda78b2f..5cac57340f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -64,7 +64,7 @@ Common gateway options: |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. For gateway-managed Docker, Podman, and VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` together in `[openshell.gateway]`; driver tables reject those gateway-owned fields. See the [Gateway Configuration File](./gateway-config) reference for the full schema. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -163,7 +163,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -241,7 +241,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -344,7 +344,7 @@ compute_driver = "vm" For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. @@ -389,7 +389,7 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Their historical `[openshell.gateway]` locations remain accepted as lower-precedence compatibility inputs. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). @@ -403,7 +403,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is empty. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | From 912951d36c8578dedc59273dcdd1bd7278db63bd Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 19:14:51 -0400 Subject: [PATCH 05/42] fix(config): address schema v2 review regressions Signed-off-by: Jesse Jaggars --- architecture/compute-runtimes.md | 11 +- crates/openshell-driver-docker/src/lib.rs | 2 +- crates/openshell-driver-docker/src/tests.rs | 9 +- crates/openshell-driver-podman/README.md | 8 +- crates/openshell-driver-podman/src/client.rs | 7 +- crates/openshell-driver-podman/src/driver.rs | 55 ++- crates/openshell-driver-vm/README.md | 2 +- .../scripts/openshell-vm-sandbox-init.sh | 20 +- crates/openshell-driver-vm/src/driver.rs | 401 ++++++++++++++---- crates/openshell-driver-vm/src/rootfs.rs | 62 ++- crates/openshell-server/src/cli.rs | 27 +- deploy/helm/openshell/README.md | 4 +- deploy/helm/openshell/templates/_helpers.tpl | 20 + .../openshell/templates/gateway-config.yaml | 4 +- .../openshell/tests/gateway_config_test.yaml | 26 ++ deploy/helm/openshell/values.yaml | 12 +- docs/reference/gateway-config.mdx | 6 +- docs/reference/sandbox-compute-drivers.mdx | 2 +- 18 files changed, 551 insertions(+), 127 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 8c7094530e..ac09d9758d 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -320,10 +320,13 @@ can request a specific number of GPUs or the driver-specific default behaviour. For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs -matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a -private `run/` directory plus Unix peer UID/PID checks. Standalone -unauthenticated TCP mode is disabled unless explicitly enabled for local -development. +matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective +sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file +ownership. Unmarked pre-migration overlays recover the account from their +persisted prepared rootfs before falling back to explicit configuration or the +legacy `10001:10001` default. The gateway-owned VM driver socket uses a private +`run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d8756bc135..9d247eac7a 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2716,7 +2716,7 @@ fn build_binds( Status::failed_precondition("provider SPIFFE socket has no parent directory") })?; binds.push(format!( - "{}:{}:ro,rbind", + "{}:{}:ro", parent.display(), PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR )); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 8c3ac54e31..991eb9ae46 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2257,11 +2257,10 @@ fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { .iter() .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) ); - assert!( - binds - .iter() - .any(|bind| bind.contains(PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR)) - ); + assert!(binds.contains(&format!( + "/run/spire:{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}:ro" + ))); + assert!(binds.iter().all(|bind| !bind.contains("rbind"))); let env = body.env.unwrap(); assert!(env.iter().any(|entry| entry == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index c2fb5b44c1..6dad86e0c8 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -356,8 +356,12 @@ signals succeeds: - `test -S` on the configured supervisor Unix socket path. - The prior TCP check for a listener on the in-container SSH port. -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. +The Unix socket check allows relay-only backend readiness when the supervisor +exposes the socket without the old marker or published-port signal. Omitting +`health_check_interval_secs` disables these Podman/conmon probes, but it does +not bypass public readiness gating: the gateway keeps a backend-ready sandbox +in `Provisioning` with `SupervisorNotConnected` until its supervisor control +session is connected. ### Deletion Flow diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 8088a50418..5e0dadfb7f 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -272,6 +272,7 @@ pub struct HostInfo { /// Podman returns `host.security.rootless: true` when the daemon is /// running without root privileges (rootless mode). #[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] pub struct SecurityInfo { #[serde(default)] pub rootless: bool, @@ -962,13 +963,17 @@ mod tests { "cgroupVersion": "v2", "networkBackend": "netavark", "rootlessNetworkCmd": "pasta", - "security": {"rootless": true} + "security": { + "rootless": true, + "apparmorEnabled": true + } } }"#, ) .unwrap(); assert!(info.host.security.rootless); + assert!(info.host.security.apparmor_enabled); assert_eq!(info.host.rootless_network_cmd, "pasta"); } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 593748920d..1b0f48e90f 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -428,19 +428,10 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } - if matches!( - config.app_armor_profile, - Some( - openshell_core::AppArmorProfile::RuntimeDefault - | openshell_core::AppArmorProfile::Localhost(_) - ) - ) && !info.host.security.apparmor_enabled - { - return Err(PodmanApiError::InvalidInput( - "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" - .to_string(), - )); - } + validate_apparmor_support( + config.app_armor_profile.as_ref(), + info.host.security.apparmor_enabled, + )?; info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, @@ -1414,6 +1405,26 @@ impl PodmanComputeDriver { } } +fn validate_apparmor_support( + profile: Option<&openshell_core::AppArmorProfile>, + apparmor_enabled: bool, +) -> Result<(), PodmanApiError> { + let requires_apparmor = matches!( + profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ); + if requires_apparmor && !apparmor_enabled { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } + Ok(()) +} + fn supervisor_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" @@ -2230,6 +2241,24 @@ mod tests { ); } + #[test] + fn confined_apparmor_profiles_follow_podman_capability() { + use openshell_core::AppArmorProfile; + + for profile in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + validate_apparmor_support(Some(&profile), true) + .expect("confined profile should be accepted when Podman reports AppArmor"); + let error = validate_apparmor_support(Some(&profile), false) + .expect_err("confined profile must fail when AppArmor is unavailable"); + assert!(error.to_string().contains("AppArmor is unavailable")); + } + validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) + .expect("Unconfined does not require AppArmor support"); + } + #[test] #[cfg(target_os = "linux")] fn rootless_pasta_requests_default_route_interface() { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index ec57edfbbc..a66b2e88e0 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,7 +152,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Existing overlay state without the per-sandbox identity marker is restored as legacy `10001:10001`. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers that identity from its persisted prepared rootfs, then falls back to explicit configuration or the legacy `10001:10001` default. | | `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend has no such NAT, so a gateway-host proxy URL is rejected before GPU sandbox launch; use an address routable from the guest's masqueraded egress. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | | `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index de2d993913..4e30be40ca 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -634,12 +634,28 @@ reconcile_sandbox_account() { mkdir -p "$etc" touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" if grep -q '^sandbox:' "$etc/group"; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$etc/group" + if ! awk -F: -v OFS=: -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = gid } { print }' \ + "$etc/group" >"$etc/group.openshell"; then + rm -f "$etc/group.openshell" + ts "FATAL: failed to reconcile sandbox group" + exit 1 + fi + mv "$etc/group.openshell" "$etc/group" else printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" fi if grep -q '^sandbox:' "$etc/passwd"; then - sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$etc/passwd" + # Preserve image-owned account metadata (home, shell, and description) + # while restoring the UID/GID contract recorded for this overlay. + if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \ + "$etc/passwd" >"$etc/passwd.openshell"; then + rm -f "$etc/passwd.openshell" + ts "FATAL: failed to reconcile sandbox account" + exit 1 + fi + mv "$etc/passwd.openshell" "$etc/passwd" else printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" fi diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d9a6ff49fc..39eee60ba2 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,7 +11,7 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_user_ids_from_image, set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; use bollard::Docker; @@ -185,7 +185,9 @@ const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; -const SANDBOX_OWNER_STATE_VERSION: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; +const LEGACY_SANDBOX_UID: u32 = 10001; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -505,23 +507,21 @@ enum OverlayPreparation { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SandboxOwnerState { - Current, - Legacy, -} - -impl SandboxOwnerState { - fn guest_environment(self) -> Option<[String; 2]> { - match self { - Self::Current => None, - // A state directory with an overlay but no state marker predates - // the 1000 migration. Its upperdir may contain 10001-owned files - // anywhere in the rootfs, not just /sandbox. - Self::Legacy => Some([ - "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), - "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ]), - } +struct SandboxOwnerIdentity { + uid: u32, + gid: u32, +} + +impl SandboxOwnerIdentity { + fn guest_environment(self) -> [String; 2] { + [ + format!("OPENSHELL_VM_SANDBOX_UID={}", self.uid), + format!("OPENSHELL_VM_SANDBOX_GID={}", self.gid), + ] + } + + fn marker_contents(self) -> String { + format!("{SANDBOX_OWNER_STATE_V2}:{}:{}\n", self.uid, self.gid) } } @@ -871,6 +871,7 @@ impl VmDriver { let disk_paths = sandbox_runtime_disk_paths(&state_dir); let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; + let owner_source_disk = image_disk.as_ref().unwrap_or(&root_disk).clone(); let overlay_disk = disk_paths.overlay_disk; self.publish_platform_event( @@ -886,6 +887,7 @@ impl VmDriver { .prepare_runtime_overlay( &state_dir, &overlay_disk, + &owner_source_disk, tls_paths.as_ref(), sandbox .spec @@ -1106,10 +1108,8 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } - if let Some(identity_env) = sandbox_owner_state.guest_environment() { - for env in identity_env { - command.arg("--vm-env").arg(env); - } + for env in sandbox_owner_state.guest_environment() { + command.arg("--vm-env").arg(env); } info!( @@ -2199,13 +2199,12 @@ impl VmDriver { &self, state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); - let (owner_state, write_owner_state) = - sandbox_owner_state_for_launch(state_dir, overlay_disk, preparation).await?; let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, @@ -2223,6 +2222,14 @@ impl VmDriver { self.config.overlay_disk_mib ) })?; + let (owner_state, write_owner_state) = sandbox_owner_state_for_launch( + state_dir, + &overlay_disk, + owner_source_disk, + &self.config, + preparation, + ) + .await?; let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2250,7 +2257,7 @@ impl VmDriver { .map_err(|err| format!("overlay image preparation panicked: {err}"))?; result?; if write_owner_state { - write_sandbox_owner_state(state_dir).await?; + write_sandbox_owner_state(state_dir, owner_state).await?; } span_status.finish(Ok(owner_state)) } @@ -5040,50 +5047,155 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } -/// Select the identity the guest must use for this overlay and whether a -/// successful preparation creates the state-version marker. A missing marker -/// is legacy only when an overlay already exists; an interrupted create with -/// no overlay is safe to initialize as current. +/// Select the exact identity the guest must use for this overlay and whether a +/// successful preparation must create or upgrade its state marker. +/// +/// For an unmarked persisted overlay, inspect the prepared rootfs recorded by +/// the previous driver before falling back to the old 10001 default. This +/// preserves images and explicit configurations that used another UID/GID. async fn sandbox_owner_state_for_launch( state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, + config: &VmDriverConfig, preparation: OverlayPreparation, -) -> Result<(SandboxOwnerState, bool), String> { - if preparation == OverlayPreparation::Fresh { - return Ok((SandboxOwnerState::Current, true)); +) -> Result<(SandboxOwnerIdentity, bool), String> { + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + match tokio::fs::read_to_string(&marker_path).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { + let identity = match persisted_sandbox_owner_identity(state_dir, config).await? { + Some(identity) => identity, + None => sandbox_owner_identity_from_image(owner_source_disk).await?, + }; + return Ok((identity, true)); + } + Ok(contents) => { + let identity = parse_sandbox_owner_state(&contents).map_err(|error| { + format!( + "invalid sandbox owner state {}: {error}", + marker_path.display() + ) + })?; + return Ok((identity, false)); + } + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!( + "read sandbox owner state {}: {error}", + marker_path.display() + )); + } + Err(_) => {} } - match tokio::fs::read_to_string(state_dir.join(SANDBOX_OWNER_STATE_FILE)).await { - Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_VERSION => { - Ok((SandboxOwnerState::Current, false)) + let overlay_exists = match tokio::fs::metadata(overlay_disk).await { + Ok(metadata) => metadata.is_file(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "stat overlay disk {}: {error}", + overlay_disk.display() + )); + } + }; + + if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { + return Ok((identity, true)); } - Ok(_) => Err(format!( - "sandbox owner state {} has an unsupported version", - state_dir.join(SANDBOX_OWNER_STATE_FILE).display() - )), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - match tokio::fs::metadata(overlay_disk).await { - Ok(_) => Ok((SandboxOwnerState::Legacy, false)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - Ok((SandboxOwnerState::Current, true)) - } - Err(err) => Err(format!( - "stat overlay disk {}: {err}", - overlay_disk.display() - )), + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + return Ok(( + SandboxOwnerIdentity { + uid: LEGACY_SANDBOX_UID, + gid: LEGACY_SANDBOX_UID, + }, + true, + )); + } + + let identity = sandbox_owner_identity_from_image(owner_source_disk) + .await + .or_else(|error| { + configured_sandbox_identity(config) + .map(|(uid, gid)| SandboxOwnerIdentity { uid, gid }) + .ok_or(error) + })?; + Ok((identity, true)) +} + +async fn persisted_sandbox_owner_identity( + state_dir: &Path, + config: &VmDriverConfig, +) -> Result, String> { + let prior_identity = match tokio::fs::read_to_string(state_dir.join(IMAGE_IDENTITY_FILE)).await + { + Ok(identity) if !identity.trim().is_empty() => identity, + Ok(_) => String::new(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(format!("read persisted VM image identity: {error}")), + }; + + if !prior_identity.is_empty() { + let prior_disk = image_cache_rootfs_image(&config.state_dir, prior_identity.trim()); + if tokio::fs::metadata(&prior_disk).await.is_ok() { + match sandbox_owner_identity_from_image(&prior_disk).await { + Ok(identity) => return Ok(Some(identity)), + Err(error) => warn!( + image_path = %prior_disk.display(), + error = %error, + "could not read sandbox identity from persisted VM rootfs; using compatibility fallback" + ), } } - Err(err) => Err(format!( - "read sandbox owner state {}: {err}", - state_dir.join(SANDBOX_OWNER_STATE_FILE).display() - )), } + + Ok(None) +} + +async fn sandbox_owner_identity_from_image( + image_path: &Path, +) -> Result { + let image_path = image_path.to_path_buf(); + let display = image_path.display().to_string(); + let identity = + tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) + .await + .map_err(|error| format!("read sandbox identity task failed: {error}"))??; + let (uid, gid) = identity.ok_or_else(|| { + format!("prepared VM rootfs {display} does not contain a sandbox account") + })?; + Ok(SandboxOwnerIdentity { uid, gid }) +} + +fn parse_sandbox_owner_state(contents: &str) -> Result { + let mut fields = contents.trim().split(':'); + if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { + return Err("unsupported version".to_string()); + } + let uid = fields + .next() + .ok_or_else(|| "missing uid".to_string())? + .parse::() + .map_err(|error| format!("invalid uid: {error}"))?; + let gid = fields + .next() + .ok_or_else(|| "missing gid".to_string())? + .parse::() + .map_err(|error| format!("invalid gid: {error}"))?; + if fields.next().is_some() { + return Err("unexpected fields".to_string()); + } + Ok(SandboxOwnerIdentity { uid, gid }) } -async fn write_sandbox_owner_state(state_dir: &Path) -> Result<(), String> { +async fn write_sandbox_owner_state( + state_dir: &Path, + identity: SandboxOwnerIdentity, +) -> Result<(), String> { write_private_file( &state_dir.join(SANDBOX_OWNER_STATE_FILE), - format!("{SANDBOX_OWNER_STATE_VERSION}\n").into_bytes(), + identity.marker_contents().into_bytes(), ) .await .map_err(|err| format!("write sandbox owner state: {err}")) @@ -6307,18 +6419,19 @@ mod tests { #[test] fn vm_config_rejects_legacy_openshell_endpoint() { - let config = VmDriverConfig::default(); + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; let mut serialized = serde_json::to_value(config).unwrap(); - let fields = serialized.as_object_mut().unwrap(); - fields.remove("grpc_endpoint"); - fields.insert( + serialized.as_object_mut().unwrap().insert( "openshell_endpoint".to_string(), serde_json::json!("http://127.0.0.1:8080"), ); let error = serde_json::from_value::(serialized) - .expect_err("legacy openshell_endpoint must be rejected"); - assert!(!error.to_string().is_empty()); + .expect_err("legacy openshell_endpoint must be rejected as unknown"); + assert!(error.to_string().contains("openshell_endpoint")); } struct TestTracing { @@ -6779,6 +6892,7 @@ mod tests { let result = driver .prepare_runtime_overlay( + Path::new("/unused"), Path::new("/unused"), Path::new("/unused"), None, @@ -7371,51 +7485,176 @@ mod tests { } #[tokio::test] - async fn legacy_overlay_state_uses_legacy_guest_identity() { + async fn unmarked_legacy_overlay_falls_back_to_legacy_default_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; - let (state, write_marker) = - sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) - .await - .unwrap(); + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); - assert_eq!(state, SandboxOwnerState::Legacy); - assert!(!write_marker); assert_eq!( - state.guest_environment(), - Some([ + identity, + SandboxOwnerIdentity { + uid: LEGACY_SANDBOX_UID, + gid: LEGACY_SANDBOX_UID, + } + ); + assert!(write_marker); + assert_eq!( + identity.guest_environment(), + [ "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ]) + ] + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_legacy_overlay_uses_explicit_config_when_old_rootfs_is_missing() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000, + } ); + assert!(write_marker); let _ = std::fs::remove_dir_all(dir); } + #[test] + fn sandbox_owner_marker_rejects_malformed_or_unknown_state() { + for marker in [ + "sandbox-owner-v3:1000:1000", + "sandbox-owner-v2", + "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:1000:1000:extra", + ] { + assert!( + parse_sandbox_owner_state(marker).is_err(), + "marker should be rejected: {marker}" + ); + } + } + #[tokio::test] - async fn current_overlay_state_is_not_inferred_from_the_lower_rootfs() { + async fn persisted_owner_marker_preserves_exact_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); std::fs::write(&overlay, b"current overlay").unwrap(); - write_sandbox_owner_state(&dir).await.unwrap(); + let expected = SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + }; + write_sandbox_owner_state(&dir, expected).await.unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; - let (state, write_marker) = - sandbox_owner_state_for_launch(&dir, &overlay, OverlayPreparation::PreserveExisting) - .await - .unwrap(); + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); - assert_eq!(state, SandboxOwnerState::Current); + assert_eq!(identity, expected); assert!(!write_marker); assert_eq!( std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), - "sandbox-owner-v1\n" + "sandbox-owner-v2:4242:4343\n" ); let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { + let root = unique_temp_dir(); + let state_dir = root.join("sandboxes/sandbox-1"); + std::fs::create_dir_all(&state_dir).unwrap(); + let overlay = state_dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let prior_identity = "legacy-cache:sha256:abc"; + std::fs::write( + state_dir.join(IMAGE_IDENTITY_FILE), + format!("{prior_identity}\n"), + ) + .unwrap(); + let source = root.join("legacy-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let prior_disk = image_cache_rootfs_image(&root, prior_identity); + create_ext4_image_from_dir_with_size(&source, &prior_disk, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: root.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &state_dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 2c84e547aa..5e38c28919 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -657,6 +657,56 @@ fn debugfs_quote_argument(argument: &str) -> Option { Some(quoted) } +/// Read the sandbox account identity directly from an ext4 rootfs image. +/// +/// Persisted VM overlays may outlive the prepared-image cache layout that +/// created them. Reading the matching old lower disk lets the driver preserve +/// that overlay's real ownership contract during an upgrade. +pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { + let quoted_path = debugfs_quote_absolute_path("/etc/passwd") + .expect("the static passwd path is a valid debugfs path"); + let command = format!("cat {quoted_path}"); + let mut last_error = None; + + for candidate in e2fs_tool_candidates("debugfs") { + let label = candidate.display().to_string(); + match Command::new(&candidate) + .arg("-R") + .arg(&command) + .arg(image_path) + .output() + { + Ok(output) if output.status.success() => { + let passwd = String::from_utf8(output.stdout).map_err(|error| { + format!( + "read /etc/passwd from {} as UTF-8: {error}", + image_path.display() + ) + })?; + return parse_sandbox_guest_user_ids(&passwd, &image_path.display().to_string()); + } + Ok(output) => { + last_error = Some(format!( + "{label} failed with status {}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + last_error = Some(format!("{label} not found")); + } + Err(error) => last_error = Some(format!("run {label}: {error}")), + } + } + + Err(format!( + "debugfs command '{command}' failed for {}: {}. Install e2fsprogs (debugfs) and retry", + image_path.display(), + last_error.unwrap_or_else(|| "debugfs not found".to_string()) + )) +} + fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd_path = rootfs.join("etc/passwd"); if !passwd_path.exists() { @@ -665,6 +715,10 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd = fs::read_to_string(&passwd_path) .map_err(|e| format!("read {}: {e}", passwd_path.display()))?; + parse_sandbox_guest_user_ids(&passwd, &passwd_path.display().to_string()) +} + +fn parse_sandbox_guest_user_ids(passwd: &str, source: &str) -> Result, String> { for line in passwd.lines() { let mut parts = line.split(':'); if parts.next() != Some("sandbox") { @@ -673,14 +727,14 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let _password = parts.next(); let uid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing uid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing uid"))? .parse::() - .map_err(|e| format!("sandbox uid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox uid in {source} is invalid: {e}"))?; let gid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing gid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing gid"))? .parse::() - .map_err(|e| format!("sandbox gid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox gid in {source} is invalid: {e}"))?; return Ok(Some((uid, gid))); } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 257c213d24..f357b8e3a1 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -239,6 +239,15 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry } } +fn reject_legacy_driver_selector_env() -> Result<()> { + if std::env::var_os("OPENSHELL_DRIVERS").is_some() { + return Err(miette::miette!( + "OPENSHELL_DRIVERS is no longer supported; use OPENSHELL_COMPUTE_DRIVER with exactly one driver name" + )); + } + Ok(()) +} + #[cfg(test)] fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result { prepare_server_config_with_drivers(args, matches, &ComputeDriverRegistry::new()) @@ -249,6 +258,8 @@ fn prepare_server_config_with_drivers( matches: &ArgMatches, compute_drivers: &ComputeDriverRegistry, ) -> Result { + reject_legacy_driver_selector_env()?; + // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -847,7 +858,7 @@ fn resolve_mtls_auth_enabled( #[cfg(test)] mod tests { - use super::{Cli, command}; + use super::{Cli, command, reject_legacy_driver_selector_env}; use crate::TEST_ENV_LOCK as ENV_LOCK; use clap::Parser; use std::net::{IpAddr, Ipv4Addr}; @@ -1257,6 +1268,20 @@ mod tests { assert!(error.to_string().contains("--drivers")); } + #[test] + fn rejects_legacy_drivers_environment_variable() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for value in ["docker", ""] { + let _guard = EnvVarGuard::set("OPENSHELL_DRIVERS", value); + let error = reject_legacy_driver_selector_env() + .expect_err("legacy OPENSHELL_DRIVERS must be rejected when present"); + assert!(error.to_string().contains("OPENSHELL_DRIVERS")); + assert!(error.to_string().contains("OPENSHELL_COMPUTE_DRIVER")); + } + } + #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 20e781dd09..92bd2a9db8 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -276,7 +276,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `nil` | Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Use always, if_not_present, or never; newer is supported only by Podman. | +| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -297,7 +297,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `nil` | Canonical sandbox supervisor pull policy. Leave unset to use the Kubernetes image default; use always, if_not_present, or never. | +| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3d9f2f3e0b..98243627dd 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -252,6 +252,26 @@ database requires persistent per-pod storage. {{- default "statefulset" (get $workload "kind") | lower -}} {{- end }} +{{/* +Translate chart image pull policy values to the canonical gateway vocabulary. +The Kubernetes spellings remain accepted so existing values files continue to +work across the schema-v2 chart upgrade. +*/}} +{{- define "openshell.canonicalImagePullPolicy" -}} +{{- $policy := printf "%v" . -}} +{{- if eq $policy "Always" -}} +always +{{- else if eq $policy "IfNotPresent" -}} +if_not_present +{{- else if eq $policy "Never" -}} +never +{{- else if has $policy (list "always" "if_not_present" "never") -}} +{{- $policy -}} +{{- else -}} +{{- fail (printf "image pull policy %q must be one of: always, if_not_present, never, Always, IfNotPresent, Never" $policy) -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index f31b23274e..2c2b8216ee 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -185,7 +185,7 @@ data: provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ .Values.server.sandboxImagePullPolicy | quote }} + image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} {{- end }} {{- $sandboxImagePullSecretNames := list -}} {{- range .Values.server.sandboxImagePullSecrets }} @@ -209,7 +209,7 @@ data: app_armor_profile = {{ .Values.server.appArmorProfile | quote }} {{- end }} {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} + supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} {{- end }} [openshell.drivers.kubernetes.managed_ssh_ingress] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 13ca8b1769..ebe57ae1bf 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -165,6 +165,32 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + - it: translates legacy Kubernetes pull policy values to canonical gateway values + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"always".*?supervisor_image_pull_policy\s*=\s*"if_not_present"' + + - it: rejects unsupported sandbox image pull policies + template: templates/statefulset.yaml + set: + server.sandboxImagePullPolicy: Sometimes + asserts: + - failedTemplate: + errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: rejects unsupported supervisor image pull policies + template: templates/statefulset.yaml + set: + supervisor.image.pullPolicy: newer + asserts: + - failedTemplate: + errorMessage: 'image pull policy "newer" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + - it: renders driver-owned Kubernetes settings only in its driver table template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 6c34963246..93144497ca 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,9 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Canonical sandbox supervisor pull policy. Leave unset to use the - # Kubernetes image default; use always, if_not_present, or never. + # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes + # image default. Prefer always, if_not_present, or never; the chart also + # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" @@ -225,9 +226,10 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Canonical pull policy for sandbox pods. Leave unset to use the Kubernetes - # image default (Always for :latest, IfNotPresent otherwise). Use always, - # if_not_present, or never; newer is supported only by Podman. + # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image + # default (Always for :latest, IfNotPresent otherwise). Prefer always, + # if_not_present, or never; the chart also accepts legacy Kubernetes spellings + # Always, IfNotPresent, and Never. sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 0d2e50ff6b..de8702117d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -230,6 +230,8 @@ phases = ["validate"] [openshell.drivers.kubernetes] namespace = "openshell" +# Required in raw TOML; Helm derives this from the gateway Service. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" @@ -672,7 +674,7 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to leave Docker's runtime default unchanged. Explicit 0 is invalid. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 # Explicit supervisor-compatible default. RuntimeDefault requires Docker to # report AppArmor support; Localhost/ requires an operator-loaded profile. @@ -733,7 +735,7 @@ stop_timeout_secs = 45 # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Omit to leave Podman's runtime default unchanged. Explicit 0 is invalid. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 # Health check interval in seconds. Omit to disable health checks; explicit 0 # is invalid. Lower values detect readiness faster but increase process churn diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 5cac57340f..db3eac1a60 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -597,7 +597,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created so a driver upgrade does not rewrite their ownership contract. ### Custom Images From c2f287543a65ba935a72fa638329ce0cdaaa388b Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Tue, 1 Sep 2026 22:24:56 -0400 Subject: [PATCH 06/42] fix(config): complete schema v2 migration safeguards Signed-off-by: Jesse Jaggars --- .agents/skills/test-release-canary/SKILL.md | 4 + Cargo.lock | 1 + architecture/compute-runtimes.md | 19 +- architecture/gateway.md | 14 + crates/openshell-driver-docker/src/lib.rs | 1 + crates/openshell-driver-docker/src/tests.rs | 9 + crates/openshell-driver-podman/README.md | 6 +- crates/openshell-driver-podman/src/config.rs | 34 ++- crates/openshell-driver-podman/src/driver.rs | 2 + crates/openshell-driver-vm/README.md | 4 +- .../scripts/openshell-vm-sandbox-init.sh | 8 +- crates/openshell-driver-vm/src/driver.rs | 268 +++++++++++++++--- crates/openshell-driver-vm/src/rootfs.rs | 30 +- crates/openshell-gateway/Cargo.toml | 2 + crates/openshell-gateway/src/lib.rs | 70 +++++ crates/openshell-gateway/src/vm.rs | 80 +++++- .../openshell-server/src/auth/sandbox_jwt.rs | 60 ++-- .../src/compute/driver_config.rs | 93 +++++- crates/openshell-server/src/config_file.rs | 24 ++ crates/openshell-server/src/grpc/auth_rpc.rs | 12 +- crates/openshell-server/src/lib.rs | 28 +- .../openshell/tests/gateway_config_test.yaml | 3 + deploy/rpm/CONFIGURATION.md | 8 +- deploy/rpm/TROUBLESHOOTING.md | 22 +- deploy/rpm/gateway.toml.default.v1 | 28 ++ deploy/rpm/migrate-gateway-config.sh | 44 +++ docs/about/installation.mdx | 4 +- docs/reference/gateway-config.mdx | 65 +++-- docs/reference/sandbox-compute-drivers.mdx | 10 +- openshell.spec | 19 +- python/openshell/release_formula_test.py | 26 +- .../rpm_gateway_config_migration_test.py | 69 +++++ rfc/0003-gateway-configuration/README.md | 55 ++-- skills/debug-openshell-cluster/SKILL.md | 17 +- tasks/scripts/gateway-docker.sh | 4 +- tasks/scripts/gateway-podman.sh | 7 +- tasks/scripts/gateway-pull-policy.sh | 28 ++ tasks/scripts/gateway.sh | 4 +- tasks/scripts/release.py | 16 +- tasks/scripts/test-gateway-pull-policy.sh | 47 +++ tasks/test.toml | 7 + 41 files changed, 1051 insertions(+), 201 deletions(-) create mode 100644 deploy/rpm/gateway.toml.default.v1 create mode 100755 deploy/rpm/migrate-gateway-config.sh create mode 100644 python/openshell/rpm_gateway_config_migration_test.py create mode 100755 tasks/scripts/gateway-pull-policy.sh create mode 100755 tasks/scripts/test-gateway-pull-policy.sh diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 7788469c3c..c710f289f4 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -26,6 +26,10 @@ does not contribute to product usage metrics. `install.sh` defaults to the *latest tagged* release — the canary is therefore checking that the most recent public release still installs, not the just-published `dev` build. The `kubernetes` job is the exception: it pins to `0.0.0-dev` chart + `:dev` images. +The host-package jobs exercise fresh installs, not upgrades from a persisted +schema-v1 gateway config. Validate Homebrew and RPM exact-default migration with +the release-tooling and package lifecycle tests before relying on the canary. + The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK validation lives in the `TypeScript SDK` branch check, including a publish dry-run. The tagged release workflow publishes the package to GitHub Packages; diff --git a/Cargo.lock b/Cargo.lock index edc940ea68..174c25aba8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4133,6 +4133,7 @@ dependencies = [ "openshell-driver-mxc", "openshell-driver-podman", "openshell-otel", + "openshell-policy", "openshell-server", "rustix 1.1.4", "serde", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ac09d9758d..0df620df2b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -290,10 +290,12 @@ is driver-owned supervisor input and is removed from workload environments. Kubernetes, Docker, and Podman share one AppArmor configuration model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates -that model to its native API and rejects a requested confined profile when its -backend reports AppArmor unavailable. Docker and Podman use explicit -`Unconfined` by default because their runtime-default profiles commonly block -the supervisor's namespace mount setup; the Helm chart uses the same default. +that model to its native API and rejects an explicitly requested confined +profile when its backend reports AppArmor unavailable. Docker keeps its +historical explicit `Unconfined` default. Podman sends no override when the +field is omitted, preserving the runtime-selected profile; development paths +that require the supervisor's namespace mount setup opt into `Unconfined` +explicitly. The Helm chart independently uses `Unconfined` for Kubernetes. Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, @@ -322,10 +324,11 @@ For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file -ownership. Unmarked pre-migration overlays recover the account from their -persisted prepared rootfs before falling back to explicit configuration or the -legacy `10001:10001` default. The gateway-owned VM driver socket uses a private -`run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +ownership. Unmarked pre-migration overlays recover identity from concrete +overlay or prepared-rootfs state, an explicit operator override, or the current +image account. The driver never assumes `10001:10001`; it preserves that legacy +identity only when persisted state reports it. The gateway-owned VM driver +socket uses a private `run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/architecture/gateway.md b/architecture/gateway.md index 809de088ea..0fcdd57561 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -38,6 +38,20 @@ immediately without a grace period. Finalization is persisted separately from the exit result; the gateway deletes an ephemeral sandbox only after the finalized supervisor session disconnects. +## Configuration Boundary + +The gateway accepts exactly schema version 2. Missing, legacy, and future +versions fail before runtime construction, and driver settings belong only to +`[openshell.drivers.]`. The process does not migrate legacy files. +Package lifecycle code may replace an exact package-generated v1 default, but +it preserves edited configurations for explicit operator migration. + +Gateway listener TLS and sandbox callback TLS are separate inputs. A selected +local Docker, Podman, or VM driver requires a complete guest bundle whenever +the gateway listener uses TLS; package-managed local TLS can supply that bundle. +Kubernetes instead projects guest credentials through its configured Secret. +The gateway validates this requirement before constructing the selected driver. + ## Protocol and Auth The gateway listens on one service port and multiplexes gRPC and HTTP traffic. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 9d247eac7a..5c8c02f73d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -187,6 +187,7 @@ pub struct DockerComputeConfig { /// `AppArmor` confinement requested for sandbox containers. The explicit /// default preserves the prior supervisor-compatible Docker behavior. + #[serde(skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 991eb9ae46..fc4cfea305 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -150,6 +150,15 @@ fn docker_config_rejects_legacy_sandbox_namespace() { assert!(error.to_string().contains("sandbox_namespace")); } +#[test] +fn docker_config_keeps_explicit_unconfined_apparmor_default() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!(config.app_armor_profile, Some(AppArmorProfile::Unconfined)); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], "Unconfined"); +} + #[test] fn docker_config_defaults_to_driver_owned_pids_limit() { let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 6dad86e0c8..5781594158 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -415,8 +415,10 @@ explicit container-reachable `tcp:IP:port` endpoint. The driver sets the supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. `app_armor_profile` shares the canonical `RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and -Kubernetes. Podman defaults to explicit `Unconfined` for the supervisor mount -setup; confined choices fail early when Podman reports AppArmor unavailable. +Kubernetes. When omitted, the driver sends no override and preserves Podman's +runtime-selected profile. Set `Unconfined` explicitly only when the deployment +requires the supervisor's mount setup to bypass that profile. Explicit confined +choices fail early when Podman reports AppArmor unavailable. This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index c71df3a727..855b7f0e24 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -91,9 +91,9 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, - /// `AppArmor` confinement requested for sandbox containers. The default - /// explicitly opts out because the supervisor needs mount operations that - /// the runtime default profile denies. + /// `AppArmor` confinement requested for sandbox containers. Omission sends + /// no override and preserves Podman's runtime-selected profile. + #[serde(default, skip_serializing_if = "Option::is_none")] pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// @@ -452,7 +452,7 @@ impl Default for PodmanComputeConfig { sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - app_armor_profile: Some(AppArmorProfile::Unconfined), + app_armor_profile: None, health_check_interval_secs: None, https_proxy: None, no_proxy: None, @@ -541,6 +541,32 @@ mod tests { ); } + #[test] + fn omitted_apparmor_profile_preserves_runtime_default() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("omitted AppArmor profile should deserialize"); + assert_eq!(config.app_armor_profile, None); + + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert!(serialized.get("app_armor_profile").is_none()); + } + + #[test] + fn explicit_apparmor_profiles_round_trip() { + for value in [ + "RuntimeDefault", + "Unconfined", + "Localhost/openshell-supervisor", + ] { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "app_armor_profile": value, + })) + .expect("explicit AppArmor profile should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], value); + } + } + #[test] fn default_config_sets_podman_stop_timeout() { let cfg = PodmanComputeConfig::default(); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 1b0f48e90f..e14bf2d9d6 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2257,6 +2257,8 @@ mod tests { } validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) .expect("Unconfined does not require AppArmor support"); + validate_apparmor_support(None, false) + .expect("an omitted profile preserves Podman's runtime behavior"); } #[test] diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a66b2e88e0..7a57893692 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -152,7 +152,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers that identity from its persisted prepared rootfs, then falls back to explicit configuration or the legacy `10001:10001` default. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers identity from its upper layer or prepared rootfs, an explicit override, or the current image. Legacy `10001:10001` is retained only when persisted state reports it. | | `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend has no such NAT, so a gateway-host proxy URL is rejected before GPU sandbox launch; use an address routable from the guest's masqueraded egress. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | | `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | @@ -262,7 +262,7 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain -- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection +- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation, identity inspection, and QEMU environment injection. Explicit `sandbox_uid`/`sandbox_gid` values do not remove this runtime prerequisite. - Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 4e30be40ca..df8fc3dad6 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -155,7 +155,10 @@ ensure_target_runtime() { fi local owner owner="$(sandbox_owner_for_root "$image_root")" - chown -R "$owner" "$image_root/sandbox" 2>/dev/null || chown -R 1000:1000 "$image_root/sandbox" || true + if ! chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then + ts "FATAL: failed to apply sandbox image ownership (${owner})" + exit 1 + fi chmod 0755 "$image_root/sandbox" } @@ -677,7 +680,8 @@ setup_sandbox_workdir() { fi if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 1000:1000 "$sandbox_dir" + ts "FATAL: failed to apply sandbox ownership (${owner})" + exit 1 fi fi chmod 0755 "$sandbox_dir" diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 39eee60ba2..6656a4dab8 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,7 +11,8 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - sandbox_guest_user_ids_from_image, set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_user_ids_from_image, sandbox_guest_user_ids_from_overlay_image, + set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; use bollard::Docker; @@ -187,7 +188,6 @@ const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; -const LEGACY_SANDBOX_UID: u32 = 10001; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -203,6 +203,7 @@ const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; const IMAGE_PREP_INIT_MODE: &str = "image-prep"; static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0); +static OWNER_STATE_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone)] struct VmDriverTlsPaths { @@ -2230,6 +2231,14 @@ impl VmDriver { preparation, ) .await?; + let owner_state_written_before_prepare = + write_owner_state && preparation == OverlayPreparation::Fresh; + if owner_state_written_before_prepare { + // Persist the selected identity before creating the overlay. A + // crash during preparation can then retry without misclassifying + // the partial overlay as legacy state. + write_sandbox_owner_state(state_dir, owner_state).await?; + } let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2256,7 +2265,7 @@ impl VmDriver { .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; result?; - if write_owner_state { + if write_owner_state && !owner_state_written_before_prepare { write_sandbox_owner_state(state_dir, owner_state).await?; } span_status.finish(Ok(owner_state)) @@ -5050,9 +5059,9 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { /// Select the exact identity the guest must use for this overlay and whether a /// successful preparation must create or upgrade its state marker. /// -/// For an unmarked persisted overlay, inspect the prepared rootfs recorded by -/// the previous driver before falling back to the old 10001 default. This -/// preserves images and explicit configurations that used another UID/GID. +/// Persisted overlays are resolved only from concrete state. In particular, +/// absence of a marker is not evidence that an overlay used the historical +/// 10001 identity: it can also mean fresh provisioning was interrupted. async fn sandbox_owner_state_for_launch( state_dir: &Path, overlay_disk: &Path, @@ -5063,11 +5072,8 @@ async fn sandbox_owner_state_for_launch( let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); match tokio::fs::read_to_string(&marker_path).await { Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { - let identity = match persisted_sandbox_owner_identity(state_dir, config).await? { - Some(identity) => identity, - None => sandbox_owner_identity_from_image(owner_source_disk).await?, - }; - return Ok((identity, true)); + // The v1 marker recorded no identity. Resolve it through the same + // evidence-based migration path as an unmarked overlay. } Ok(contents) => { let identity = parse_sandbox_owner_state(&contents).map_err(|error| { @@ -5099,29 +5105,32 @@ async fn sandbox_owner_state_for_launch( }; if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + match sandbox_owner_identity_from_overlay(overlay_disk).await { + Ok(Some(identity)) => return Ok((identity, true)), + Ok(None) => {} + Err(error) => warn!( + overlay_path = %overlay_disk.display(), + error = %error, + "could not read sandbox identity from VM overlay upper layer" + ), + } if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { return Ok((identity, true)); } if let Some((uid, gid)) = configured_sandbox_identity(config) { return Ok((SandboxOwnerIdentity { uid, gid }, true)); } - return Ok(( - SandboxOwnerIdentity { - uid: LEGACY_SANDBOX_UID, - gid: LEGACY_SANDBOX_UID, - }, - true, - )); + return sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)); } - let identity = sandbox_owner_identity_from_image(owner_source_disk) + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + sandbox_owner_identity_from_image(owner_source_disk) .await - .or_else(|error| { - configured_sandbox_identity(config) - .map(|(uid, gid)| SandboxOwnerIdentity { uid, gid }) - .ok_or(error) - })?; - Ok((identity, true)) + .map(|identity| (identity, true)) } async fn persisted_sandbox_owner_identity( @@ -5157,17 +5166,26 @@ async fn sandbox_owner_identity_from_image( image_path: &Path, ) -> Result { let image_path = image_path.to_path_buf(); - let display = image_path.display().to_string(); let identity = tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) .await .map_err(|error| format!("read sandbox identity task failed: {error}"))??; - let (uid, gid) = identity.ok_or_else(|| { - format!("prepared VM rootfs {display} does not contain a sandbox account") - })?; + let (uid, gid) = identity.unwrap_or((DEFAULT_SANDBOX_UID, DEFAULT_SANDBOX_UID)); Ok(SandboxOwnerIdentity { uid, gid }) } +async fn sandbox_owner_identity_from_overlay( + overlay_path: &Path, +) -> Result, String> { + let overlay_path = overlay_path.to_path_buf(); + let identity = tokio::task::spawn_blocking(move || { + sandbox_guest_user_ids_from_overlay_image(&overlay_path) + }) + .await + .map_err(|error| format!("read sandbox overlay identity task failed: {error}"))??; + Ok(identity.map(|(uid, gid)| SandboxOwnerIdentity { uid, gid })) +} + fn parse_sandbox_owner_state(contents: &str) -> Result { let mut fields = contents.trim().split(':'); if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { @@ -5186,6 +5204,7 @@ fn parse_sandbox_owner_state(contents: &str) -> Result Result<(), String> { - write_private_file( - &state_dir.join(SANDBOX_OWNER_STATE_FILE), - identity.marker_contents().into_bytes(), - ) - .await - .map_err(|err| format!("write sandbox owner state: {err}")) + validate_sandbox_owner_identity(identity.uid, identity.gid)?; + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + let sequence = OWNER_STATE_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary_path = state_dir.join(format!( + ".{SANDBOX_OWNER_STATE_FILE}.{}.{sequence}.tmp", + std::process::id() + )); + write_private_file(&temporary_path, identity.marker_contents().into_bytes()) + .await + .map_err(|err| format!("write temporary sandbox owner state: {err}"))?; + if let Err(error) = tokio::fs::rename(&temporary_path, &marker_path).await { + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(format!("install sandbox owner state: {error}")); + } + Ok(()) +} + +fn validate_sandbox_owner_identity(uid: u32, gid: u32) -> Result<(), String> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + if !range.contains(&uid) { + return Err(format!( + "uid {uid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + if !range.contains(&gid) { + return Err(format!( + "gid {gid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + Ok(()) } #[allow(clippy::result_large_err)] @@ -7485,17 +7532,58 @@ mod tests { } #[tokio::test] - async fn unmarked_legacy_overlay_falls_back_to_legacy_default_identity() { + async fn unmarked_overlay_uses_current_image_instead_of_blind_legacy_identity() { let dir = unique_temp_dir(); std::fs::create_dir_all(&dir).unwrap(); let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); - std::fs::write(&overlay, b"legacy overlay").unwrap(); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let source = dir.join("current-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let current_rootfs = dir.join("current-rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, ¤t_rootfs, 32 * 1024 * 1024).unwrap(); let config = VmDriverConfig { state_dir: dir.clone(), ..Default::default() }; let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + ¤t_rootfs, + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_without_identity_evidence_fails_safely() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let error = sandbox_owner_state_for_launch( &dir, &overlay, Path::new("/missing-current-rootfs"), @@ -7503,22 +7591,74 @@ mod tests { OverlayPreparation::PreserveExisting, ) .await + .expect_err("ambiguous overlay must not receive a guessed identity"); + + assert!(error.contains("missing-current-rootfs")); + assert!(!error.contains("10001")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_overlay_uses_explicit_identity_without_image_inspection() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::Fresh, + ) + .await .unwrap(); assert_eq!( identity, SandboxOwnerIdentity { - uid: LEGACY_SANDBOX_UID, - gid: LEGACY_SANDBOX_UID, + uid: 2000, + gid: 3000 } ); assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_image_without_sandbox_account_uses_default_identity() { + let dir = unique_temp_dir(); + let source = dir.join("rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write(source.join("etc/passwd"), "root:x:0:0:root:/root:/bin/sh\n").unwrap(); + let rootfs = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, &rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, _) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + &rootfs, + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + assert_eq!( - identity.guest_environment(), - [ - "OPENSHELL_VM_SANDBOX_UID=10001".to_string(), - "OPENSHELL_VM_SANDBOX_GID=10001".to_string(), - ] + identity, + SandboxOwnerIdentity { + uid: DEFAULT_SANDBOX_UID, + gid: DEFAULT_SANDBOX_UID, + } ); let _ = std::fs::remove_dir_all(dir); } @@ -7563,6 +7703,8 @@ mod tests { "sandbox-owner-v3:1000:1000", "sandbox-owner-v2", "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:0:1000", + "sandbox-owner-v2:1000:0", "sandbox-owner-v2:1000:1000:extra", ] { assert!( @@ -7607,6 +7749,44 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_upper_passwd() { + let dir = unique_temp_dir(); + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:10001:10001:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 10001, + gid: 10001, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { let root = unique_temp_dir(); diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 5e38c28919..4821844a7d 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -663,7 +663,33 @@ fn debugfs_quote_argument(argument: &str) -> Option { /// created them. Reading the matching old lower disk lets the driver preserve /// that overlay's real ownership contract during an upgrade. pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { - let quoted_path = debugfs_quote_absolute_path("/etc/passwd") + sandbox_guest_user_ids_from_image_path(image_path, "/etc/passwd") +} + +/// Read a sandbox account copied into an overlay upper layer. +/// +/// An upper-layer passwd file is the most direct evidence of the identity an +/// existing overlay observed, so migration consults it before any lower image. +pub fn sandbox_guest_user_ids_from_overlay_image( + image_path: &Path, +) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/upper/etc/passwd") +} + +fn sandbox_guest_user_ids_from_image_path( + image_path: &Path, + guest_path: &str, +) -> Result, String> { + let metadata = fs::metadata(image_path) + .map_err(|error| format!("stat rootfs image {}: {error}", image_path.display()))?; + if !metadata.is_file() { + return Err(format!( + "rootfs image {} is not a regular file", + image_path.display() + )); + } + + let quoted_path = debugfs_quote_absolute_path(guest_path) .expect("the static passwd path is a valid debugfs path"); let command = format!("cat {quoted_path}"); let mut last_error = None; @@ -679,7 +705,7 @@ pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result { let passwd = String::from_utf8(output.stdout).map_err(|error| { format!( - "read /etc/passwd from {} as UTF-8: {error}", + "read {guest_path} from {} as UTF-8: {error}", image_path.display() ) })?; diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index f9d02027e0..e9685adad4 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true } openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +openshell-policy = { path = "../openshell-policy", optional = true } hyper-util = { workspace = true, optional = true } nix = { workspace = true, optional = true } serde = { workspace = true, optional = true } @@ -43,6 +44,7 @@ in-tree-compute-drivers = [ "dep:openshell-driver-docker", "dep:openshell-driver-kubernetes", "dep:openshell-driver-podman", + "dep:openshell-policy", "dep:openshell-otel", "dep:hyper-util", "dep:nix", diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index f091960a8e..5645eba695 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -198,6 +198,7 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls( &mut config.guest_tls_ca, &mut config.guest_tls_cert, @@ -230,6 +231,7 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "podman")?; config.gateway_port = context.gateway_port(); if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { config.socket_path = Some(path.into()); @@ -268,6 +270,7 @@ impl openshell_server::ComputeDriverFactory for VmFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: vm::VmComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "vm")?; if config.state_dir.as_os_str().is_empty() { config.state_dir = vm::VmComputeConfig::default_state_dir(); } @@ -300,6 +303,32 @@ impl openshell_server::ComputeDriverFactory for VmFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn require_guest_tls_for_local_driver( + context: &openshell_server::ComputeDriverBuildContext<'_>, + driver_name: &str, +) -> openshell_core::Result<()> { + validate_local_driver_guest_tls( + context.gateway_tls_enabled(), + context.guest_tls_paths().is_some(), + driver_name, + ) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn validate_local_driver_guest_tls( + gateway_tls_enabled: bool, + has_guest_tls: bool, + driver_name: &str, +) -> openshell_core::Result<()> { + if gateway_tls_enabled && !has_guest_tls { + return Err(openshell_core::Error::config(format!( + "gateway TLS requires guest_tls_ca, guest_tls_cert, and guest_tls_key in [openshell.gateway] when using the {driver_name} compute driver" + ))); + } + Ok(()) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn apply_guest_tls( ca: &mut Option, @@ -318,6 +347,47 @@ fn apply_guest_tls( } } +#[cfg(all(test, not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod local_driver_tests { + use super::{apply_guest_tls, validate_local_driver_guest_tls}; + use std::path::{Path, PathBuf}; + + #[test] + fn tls_enabled_local_drivers_require_a_guest_bundle() { + for driver_name in ["docker", "podman", "vm"] { + let error = validate_local_driver_guest_tls(true, false, driver_name) + .expect_err("TLS-enabled local driver must require guest TLS"); + let message = error.to_string(); + assert!(message.contains(driver_name)); + assert!(message.contains("guest_tls_ca")); + } + validate_local_driver_guest_tls(true, true, "docker") + .expect("a complete guest bundle satisfies the requirement"); + validate_local_driver_guest_tls(false, false, "docker") + .expect("plaintext gateways do not require guest TLS"); + } + + #[test] + fn package_managed_guest_bundle_is_injected_when_driver_paths_are_absent() { + let mut ca = None; + let mut cert = None; + let mut key = None; + apply_guest_tls( + &mut ca, + &mut cert, + &mut key, + Some(( + Path::new("/managed/ca.pem"), + Path::new("/managed/client.pem"), + Path::new("/managed/client-key.pem"), + )), + ); + assert_eq!(ca, Some(PathBuf::from("/managed/ca.pem"))); + assert_eq!(cert, Some(PathBuf::from("/managed/client.pem"))); + assert_eq!(key, Some(PathBuf::from("/managed/client-key.pem"))); + } +} + #[cfg(all(test, target_os = "windows", feature = "in-tree-compute-drivers"))] mod windows_tests { use super::*; diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 8ca8a4b5f1..70d4e266cd 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -91,6 +91,12 @@ pub struct VmComputeConfig { /// Writable overlay disk size for each VM sandbox, in MiB. pub overlay_disk_mib: u64, + /// Optional UID override for the VM guest sandbox account. + pub sandbox_uid: Option, + + /// Optional GID override for the VM guest sandbox account. + pub sandbox_gid: Option, + /// Host-side CA certificate for the guest's mTLS client bundle. pub guest_tls_ca: Option, @@ -171,6 +177,8 @@ impl Default for VmComputeConfig { vcpus: Self::default_vcpus(), mem_mib: Self::default_mem_mib(), overlay_disk_mib: Self::default_overlay_disk_mib(), + sandbox_uid: None, + sandbox_gid: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, @@ -474,6 +482,7 @@ pub async fn spawn( )); } + validate_vm_sandbox_identity(vm_config)?; vm_config.upstream_proxy.validate().map_err(Error::config)?; if let Some(endpoint) = vm_config .provider_spiffe_workload_api_tcp_endpoint @@ -523,6 +532,7 @@ pub async fn spawn( command .arg("--overlay-disk-mib") .arg(vm_config.overlay_disk_mib.to_string()); + append_vm_identity_args(&mut command, vm_config); if let Some(tls) = guest_tls_paths { command.arg("--guest-tls-ca").arg(tls.ca); command.arg("--guest-tls-cert").arg(tls.cert); @@ -543,6 +553,35 @@ pub async fn spawn( )) } +fn validate_vm_sandbox_identity(config: &VmComputeConfig) -> Result<()> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + for (field, value) in [ + ("sandbox_uid", config.sandbox_uid), + ("sandbox_gid", config.sandbox_gid), + ] { + if let Some(value) = value + && !range.contains(&value) + { + return Err(Error::config(format!( + "{field} {value} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + ))); + } + } + Ok(()) +} + +#[cfg(unix)] +fn append_vm_identity_args(command: &mut Command, config: &VmComputeConfig) { + if let Some(uid) = config.sandbox_uid { + command.arg("--sandbox-uid").arg(uid.to_string()); + } + if let Some(gid) = config.sandbox_gid { + command.arg("--sandbox-gid").arg(gid.to_string()); + } +} + #[cfg(unix)] fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { let proxy = &config.upstream_proxy; @@ -659,10 +698,11 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, append_vm_proxy_and_spiffe_args, - compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, - prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, - resolve_driver_search_dirs, + VmComputeConfig, append_otlp_args, append_vm_identity_args, + append_vm_proxy_and_spiffe_args, compute_driver_guest_tls_paths, + compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, + prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, + validate_vm_sandbox_identity, }; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; @@ -749,8 +789,6 @@ mod tests { #[test] fn invalid_corporate_proxy_config_is_rejected_before_the_driver_starts() { - // Without this the operator would see an opaque driver-readiness - // timeout instead of an error naming the offending key. let err = openshell_core::UpstreamProxyConfig { https_proxy: Some("socks5://proxy.corp.com:1080".to_string()), ..Default::default() @@ -767,6 +805,36 @@ mod tests { .expect("a lone proxy URL is a complete configuration"); } + #[test] + fn vm_driver_command_includes_configured_sandbox_identity() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + let config = VmComputeConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + validate_vm_sandbox_identity(&config).expect("valid identity"); + append_vm_identity_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(args, ["--sandbox-uid", "2000", "--sandbox-gid", "3000"]); + } + + #[test] + fn vm_gateway_config_rejects_root_identity() { + let config = VmComputeConfig { + sandbox_uid: Some(0), + ..Default::default() + }; + let error = validate_vm_sandbox_identity(&config).expect_err("root UID must fail"); + assert!(error.to_string().contains("sandbox_uid 0")); + } + #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 9dc10b8401..0a16000999 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -84,7 +84,7 @@ pub struct SandboxJwtIssuer { kid: String, issuer: String, audience: String, - ttl: Duration, + ttl: Option, } impl std::fmt::Debug for SandboxJwtIssuer { @@ -110,10 +110,14 @@ impl SandboxJwtIssuer { signing_key_pem: &[u8], kid: String, gateway_id: &str, - ttl: Duration, + ttl: Option, ) -> Result { crate::install_jsonwebtoken_crypto_provider(); + if ttl.is_some_and(|ttl| ttl.is_zero()) { + return Err("sandbox token TTL must be positive when configured".to_string()); + } + let encoding_key = EncodingKey::from_ed_pem(signing_key_pem) .map_err(|e| format!("failed to parse Ed25519 signing key PEM: {e}"))?; let identity = format!("openshell-gateway:{gateway_id}"); @@ -132,11 +136,9 @@ impl SandboxJwtIssuer { crate::install_jsonwebtoken_crypto_provider(); let now = now_secs(); - let exp = if self.ttl.is_zero() { - 0 - } else { - now.saturating_add(i64::try_from(self.ttl.as_secs()).unwrap_or(3_600)) - }; + let exp = self.ttl.map_or(0, |ttl| { + now.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(3_600)) + }); let claims = SandboxJwtClaims { sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), iss: self.issuer.clone(), @@ -226,7 +228,7 @@ impl SandboxJwtIssuer { }) } - pub fn ttl(&self) -> Duration { + pub fn sandbox_token_ttl(&self) -> Option { self.ttl } } @@ -417,10 +419,10 @@ mod tests { } fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { - pair_with_ttl(Duration::from_secs(3600)) + pair_with_ttl(Some(Duration::from_secs(3600))) } - fn pair_with_ttl(ttl: Duration) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { + fn pair_with_ttl(ttl: Option) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { let mat = generate_jwt_key().expect("jwt key"); let issuer = SandboxJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), @@ -446,6 +448,7 @@ mod tests { async fn mint_and_validate_round_trip() { let (issuer, auth) = pair(); let minted = issuer.mint("sandbox-a").unwrap(); + assert!(minted.expires_at_ms > 0); let principal = auth .authenticate(&header_map_with_bearer(&minted.token), "/anything") .await @@ -472,7 +475,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); let auth = SandboxJwtAuthenticator::from_pem( @@ -514,8 +517,8 @@ mod tests { } #[tokio::test] - async fn ttl_zero_mints_non_expiring_token() { - let (issuer, auth) = pair_with_ttl(Duration::ZERO); + async fn ttl_none_mints_non_expiring_token() { + let (issuer, auth) = pair_with_ttl(None); let minted = issuer.mint("sandbox-never").unwrap(); assert_eq!(minted.expires_at_ms, 0); @@ -537,6 +540,19 @@ mod tests { assert_eq!(decoded.claims.exp, 0); } + #[test] + fn ttl_some_zero_is_rejected() { + let mat = generate_jwt_key().expect("jwt key"); + let error = SandboxJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), + mat.kid, + "test-gateway", + Some(Duration::ZERO), + ) + .expect_err("Some(Duration::ZERO) must not reintroduce a sentinel"); + assert!(error.contains("must be positive")); + } + #[tokio::test] async fn token_signed_by_other_key_is_rejected() { let (_, auth_a) = pair(); @@ -582,7 +598,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "g", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); let auth = @@ -613,7 +629,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::ZERO, + None, ) .expect("issuer"); let decoding_key = DecodingKey::from_ed_pem(mat.public_key_pem.as_bytes()).unwrap(); @@ -662,7 +678,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); @@ -692,13 +708,9 @@ mod tests { #[test] fn extension_token_rejects_wrong_audience() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid, - "gateway-a", - Duration::ZERO, - ) - .expect("issuer"); + let issuer = + SandboxJwtIssuer::from_pem(mat.signing_key_pem.as_bytes(), mat.kid, "gateway-a", None) + .expect("issuer"); let minted = issuer .mint_extension_token( &extension_audience("service-a"), @@ -734,7 +746,7 @@ mod tests { #[test] fn extension_token_enforces_positive_bounded_ttl_and_caller_shape() { - let (issuer, _) = pair_with_ttl(Duration::ZERO); + let (issuer, _) = pair_with_ttl(None); for ttl in [ Duration::ZERO, MAX_EXTENSION_TOKEN_TTL + Duration::from_secs(1), diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index c4e4d91ecd..a9d7faa519 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -204,6 +204,7 @@ fn validate_remote_driver_config(cfg: &RemoteDriverConfig, name: &str) -> Result mod tests { use super::*; use std::collections::BTreeMap; + use std::path::Path; fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = @@ -225,14 +226,95 @@ mod tests { } #[test] - fn gateway_guest_tls_requires_complete_bundle() { + fn gateway_guest_tls_resolves_explicit_complete_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let ca = dir.path().join("ca.pem"); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + for path in [&ca, &cert, &key] { + std::fs::write(path, b"test").expect("write TLS fixture"); + } let gateway = config_file::GatewayFileSection { - guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_ca: Some(ca.clone()), + guest_tls_cert: Some(cert.clone()), + guest_tls_key: Some(key.clone()), + ..Default::default() + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect("complete guest TLS should resolve") + .expect("guest TLS bundle"); + + assert_eq!( + resolved.as_paths(), + (ca.as_path(), cert.as_path(), key.as_path()) + ); + } + + #[test] + fn gateway_guest_tls_rejects_every_partial_bundle() { + let path = PathBuf::from("/tmp/guest-tls.pem"); + for (ca, cert, key) in [ + (Some(path.clone()), None, None), + (None, Some(path.clone()), None), + (None, None, Some(path.clone())), + (Some(path.clone()), Some(path.clone()), None), + (Some(path.clone()), None, Some(path.clone())), + (None, Some(path.clone()), Some(path)), + ] { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: ca, + guest_tls_cert: cert, + guest_tls_key: key, + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + } + + #[test] + fn gateway_guest_tls_rejects_missing_explicit_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(dir.path().join("missing-ca.pem")), + guest_tls_cert: Some(dir.path().join("missing-cert.pem")), + guest_tls_key: Some(dir.path().join("missing-key.pem")), ..Default::default() }; let error = GuestTlsPaths::resolve(Some(&gateway), None, false) - .expect_err("partial guest TLS must fail"); - assert!(error.contains("one complete bundle")); + .expect_err("missing explicit file must fail"); + assert!(error.contains("guest_tls_ca")); + assert!(error.contains("does not exist")); + } + + #[test] + fn gateway_guest_tls_uses_package_managed_bundle() { + let local = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + let resolved = GuestTlsPaths::resolve(None, Some(&local), false) + .expect("managed bundle should resolve") + .expect("guest TLS bundle"); + assert_eq!( + resolved.as_paths(), + ( + Path::new("/managed/ca.pem"), + Path::new("/managed/client-cert.pem"), + Path::new("/managed/client-key.pem"), + ) + ); + } + + #[test] + fn gateway_guest_tls_can_be_absent() { + assert!(GuestTlsPaths::resolve(None, None, false).unwrap().is_none()); + assert!(GuestTlsPaths::resolve(None, None, true).unwrap().is_none()); } #[test] @@ -268,6 +350,9 @@ socket_path = "/run/openshell/kyma.sock" fn remote_driver_config_reads_only_socket_path() { let file: config_file::ConfigFile = toml::from_str( r#" +[openshell] +version = 2 + [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" workspace_mode = "shared" diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 1e717c17c4..47dede5c89 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -986,6 +986,30 @@ ssh_gateway_port = 8080 )); } + #[test] + fn rejects_missing_version_in_nonempty_file() { + let tmp = write_raw_tmp("[openshell]\n\n[openshell.gateway]\nname = \"test\"\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); + } + + #[test] + fn rejects_future_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 3\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::UnsupportedVersion { version: 3 }) + )); + } + + #[test] + fn accepts_current_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 2\n"); + load(tmp.path()).expect("schema version 2 must be accepted"); + } + #[test] fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index 104d639584..35325f3bbf 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -226,11 +226,11 @@ fn mint_extension_credentials( .iter() .map(|service| (service.name.as_str(), service)) .collect(); - let ttl = if issuer.ttl().is_zero() { - DEFAULT_EXTENSION_TOKEN_TTL - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - }; + let ttl = issuer + .sandbox_token_ttl() + .map_or(DEFAULT_EXTENSION_TOKEN_TTL, |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }); requested_names .iter() @@ -323,7 +323,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid, "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); state.sandbox_jwt_issuer = Some(Arc::new(issuer)); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 363f00e561..c5c76548f3 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -97,11 +97,11 @@ struct GatewayExtensionCredential { } fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration { - if issuer.ttl().is_zero() { - Duration::from_secs(15 * 60) - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - } + issuer + .sandbox_token_ttl() + .map_or(Duration::from_secs(15 * 60), |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }) } /// Mint the gateway-caller credential for one extension registration. @@ -496,7 +496,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - jwt.sandbox_token_ttl().unwrap_or_default(), + jwt.sandbox_token_ttl(), ) .map_err(Error::config)?, ); @@ -1586,7 +1586,7 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, + configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ @@ -1632,18 +1632,30 @@ mod tests { } fn extension_test_issuer() -> Arc { + extension_test_issuer_with_ttl(Some(Duration::from_secs(900))) + } + + fn extension_test_issuer_with_ttl( + ttl: Option, + ) -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( crate::auth::sandbox_jwt::SandboxJwtIssuer::from_pem( material.signing_key_pem.as_bytes(), material.kid, "gateway-a", - Duration::from_secs(900), + ttl, ) .expect("issuer"), ) } + #[test] + fn non_expiring_sandbox_tokens_use_finite_extension_ttl() { + let issuer = extension_test_issuer_with_ttl(None); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(15 * 60)); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index ebe57ae1bf..7130871f22 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -204,6 +204,9 @@ tests: - notMatchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'guest_tls_(ca|cert|key)\s*=' - it: renders user namespace enablement under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index ce20d28e25..f6889e4fb9 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -34,9 +34,11 @@ prevents unexpected driver selection if Docker is also installed on the host. ### Customizing the configuration -Edit `~/.config/openshell/gateway.toml` directly. The template at -`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime -and is not overwritten by RPM upgrades. +Edit `~/.config/openshell/gateway.toml` directly. The package-owned template at +`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime and +may change during an RPM upgrade. The active user copy is preserved. During a +schema-v2 upgrade, the service replaces only an exact package-generated v1 +copy; it never rewrites an edited configuration. To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index a8460a473e..58de9e7856 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -227,9 +227,14 @@ non-functional until restarted, causing the gateway to fail with a connection error on `/run/user//podman/podman.sock`. The gateway retries briefly on startup, but a stale socket will not recover on its own. -Package upgrades do not overwrite `~/.config/openshell/gateway.toml` when you -create one. New gateway process options can be added manually by referencing -CONFIGURATION.md or running `openshell-gateway --help`. +Package upgrades preserve edited `~/.config/openshell/gateway.toml` files. On +the schema-v2 upgrade, the user service replaces only an exact copy of the v1 +file previously seeded by the RPM. If you edited that file, migrate it manually +before restarting the service; direct `dnf` or `rpm` upgrades do not use the +breaking-upgrade guard in `install.sh`. See the +[Gateway Configuration File](https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2) +for the field-by-field migration steps. New gateway process options are listed +in CONFIGURATION.md and `openshell-gateway --help`. To pick up new container images after an upgrade: @@ -238,6 +243,17 @@ podman pull ghcr.io/nvidia/openshell/supervisor:latest podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ``` +### Migrating a TLS-enabled local driver to schema version 2 + +Docker, Podman, and VM sandboxes connect back to the gateway with a guest TLS +bundle. Package-managed installs use the complete bundle generated under +`~/.local/state/openshell/tls`, so the RPM default requires no additional TOML. +If you override the listener with custom `--tls-cert` and `--tls-key` inputs and +do not use that managed bundle, configure all three `guest_tls_ca`, +`guest_tls_cert`, and `guest_tls_key` paths under `[openshell.gateway]`. The +gateway now fails at startup instead of allowing sandboxes to fail later. Omit +all three fields when TLS is disabled. + ### Migrating from gateway.env Previous releases generated `~/.config/openshell/gateway.env` on first diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 new file mode 100644 index 0000000000..ba76f873b2 --- /dev/null +++ b/deploy/rpm/gateway.toml.default.v1 @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_driver = "podman" diff --git a/deploy/rpm/migrate-gateway-config.sh b/deploy/rpm/migrate-gateway-config.sh new file mode 100755 index 0000000000..5c75af1f9f --- /dev/null +++ b/deploy/rpm/migrate-gateway-config.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +if [ "$#" -ne 3 ]; then + echo "usage: $0 DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT" >&2 + exit 2 +fi + +destination=$1 +current_default=$2 +legacy_default=$3 + +for source in "$current_default" "$legacy_default"; do + if [ ! -f "$source" ]; then + echo "gateway config migration source is not a regular file: $source" >&2 + exit 1 + fi +done + +if [ -L "$destination" ] || { [ -e "$destination" ] && [ ! -f "$destination" ]; }; then + echo "refusing to replace non-regular gateway config: $destination" >&2 + exit 1 +fi + +if [ ! -e "$destination" ]; then + install -Dm 0644 "$current_default" "$destination" + exit 0 +fi + +# Replace only the exact config seeded by the schema-v1 RPM. Any edit, +# including whitespace or comments, makes the operator-owned file authoritative. +if ! cmp -s "$legacy_default" "$destination"; then + exit 0 +fi + +destination_dir=$(dirname "$destination") +temporary=$(mktemp "$destination_dir/.gateway.toml.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM +install -m 0644 "$current_default" "$temporary" +mv -f "$temporary" "$destination" +trap - EXIT HUP INT TERM diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 9026f939a6..7064fe7a56 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -44,7 +44,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew preserves user-edited prefix and user configs during upgrades; it removes the IPv6 bind only from an unchanged config generated by the affected formula. +The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. @@ -63,7 +63,7 @@ On Debian and Ubuntu, the install script uses a Debian package. The Debian packa Linux packages require glibc 2.28 or newer. The installer checks libc before downloading packages and exits with an error on older glibc versions, Alpine, musl-based distributions, or unknown libc environments. -The Linux user service listens on `https://127.0.0.1:17670`, starts from built-in defaults, and generates a local mTLS bundle before the gateway starts. Create `~/.config/openshell/gateway.toml` only when you need to override those defaults. +The Linux user service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle before the gateway starts. Debian uses built-in gateway defaults unless you create a config. RPM seeds `~/.config/openshell/gateway.toml` from its packaged Podman template on first start. RPM upgrades migrate only an unchanged package-generated schema-v1 file; they preserve edited files. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) when upgrading an edited v1 configuration. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index de8702117d..3b3cff69b1 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,18 +22,20 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Optional Gateway TOML location | +| Package | Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. + +Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. ## Layout @@ -74,7 +76,10 @@ future version. To migrate an existing file: 3. Move every compute-driver option into `[openshell.drivers.]`. Schema version 2 does not inherit driver defaults from `[openshell.gateway]`. Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway - scope; set all three or omit all three when TLS is disabled. + scope. A TLS-enabled Docker, Podman, or VM gateway requires one complete + guest bundle. Set all three paths unless the package-managed local TLS + bundle supplies them. When TLS is disabled, omit all three. Kubernetes + projects sandbox TLS through `client_tls_secret_name` instead. 4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM `openshell_endpoint` to `grpc_endpoint`. @@ -90,7 +95,8 @@ future version. To migrate an existing file: TOML requires an explicit endpoint; Helm derives one from the release's gateway Service. New VM root filesystems use an image-provided `sandbox` account when present and otherwise use UID/GID 1000. Existing persisted VM - state using 10001 remains compatible. + state retains its recorded or recoverable identity, including legacy 10001; + the driver does not assign 10001 to an overlay without supporting state. Unknown fields and non-table `[openshell.drivers.]` values fail startup. This strict validation prevents misspelled or misplaced security-sensitive @@ -139,8 +145,11 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Guest TLS paths remain gateway settings. Set all three for TLS, or omit all -# three only when TLS is disabled. Driver tables must not repeat these fields. +# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM +# gateways require a complete bundle unless package-managed local TLS supplies +# it automatically. Omit all three when TLS is disabled. Kubernetes projects +# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat +# these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -832,13 +841,18 @@ health_check_interval_secs = 10 # explicit container-reachable TCP endpoint, for provider token exchange. # provider_spiffe_workload_api_socket = "/run/spire/agent.sock" # provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" -# Explicit supervisor-compatible default. RuntimeDefault and Localhost/ -# require Podman to report AppArmor support. -app_armor_profile = "Unconfined" +# Omit app_armor_profile to preserve Podman's runtime-selected profile. +# Set Unconfined only when the supervisor's mount setup requires it. +# Explicit RuntimeDefault and Localhost/ require Podman to report +# AppArmor support. +# app_armor_profile = "Unconfined" ``` Use `ssh_socket_path` for Podman configurations. The legacy -`sandbox_ssh_socket_path` key is rejected. +`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, +OpenShell sends no override and Podman applies its runtime-selected profile. +Set `Unconfined` explicitly only when the deployment requires the supervisor's +mount setup to bypass that profile. ### MicroVM @@ -873,21 +887,22 @@ vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 # Resolved sandbox UID/GID for new rootfs /etc/passwd entries. -# Defaults to 1000 when unset; matching GID is used if sandbox_gid is empty. -# Existing persisted VM rootfs/overlays with UID 10001 retain that identity. -# Any non-root Linux UID/GID is valid. +# Defaults to the image's sandbox account, or 1000 when the account is absent; +# matching GID is used if sandbox_gid is empty. Persisted overlays recover their +# recorded identity, including 10001, rather than receiving a legacy fallback. +# Values must fall within OpenShell's allowed non-root sandbox identity range. # sandbox_uid = 20001 +# sandbox_gid = 20001 # Corporate forward proxy for sandbox egress. The keys, their semantics, and # the fail-closed contract are identical to the Podman driver above: only TLS # (CONNECT) egress is chained, plain-HTTP destination requests always dial # directly, credentials must come from proxy_auth_file rather than the URL, # an http:// proxy with credentials requires proxy_auth_allow_insecure, and # any present-but-invalid value is rejected at gateway startup rather than -# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on -# the gateway host. +# degrading to a direct dial. proxy_auth_file is a path on the gateway host. # # The sandbox cannot select or override these settings. They reach the guest -# supervisor on its command line through a per-sandbox file the driver writes +# supervisor through a protected per-sandbox argument file the driver writes # into the overlay upperdir on every launch, so a sandbox image cannot supply # its own values or disable the operator's by baking a file at that path. # @@ -907,20 +922,18 @@ overlay_disk_mib = 4096 # address routable from the guest's masqueraded egress. # # Because a microVM has no bind mounts or container secrets, the driver stages -# the credential and the CA into the per-sandbox overlay disk: the credential -# root-only inside the guest, and both removed with the sandbox. The -# credential is therefore at rest in that overlay image on the gateway host — -# the same delivery the per-sandbox gateway token already uses, and a -# difference from the Podman secret model worth noting when choosing where to -# keep proxy credentials. +# the credential into the per-sandbox overlay disk, root-only inside the guest, +# and removes it with the sandbox. The credential is therefore at rest in that +# overlay image on the gateway host — the same delivery the per-sandbox gateway +# token already uses, and a difference from the Podman secret model worth noting +# when choosing where to keep proxy credentials. # https_proxy = "http://host.openshell.internal:8080" # no_proxy = "10.0.0.0/8,.internal.example" # proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs; see the Podman section above. # proxy_connect_by_hostname = true -# Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. -# proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" # VM guests cannot mount a host Workload API Unix socket. Configure only a # separately operated guest-reachable TCP listener and explicitly acknowledge # the exposure; host-only sockets are never exposed automatically. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index db3eac1a60..9dc708f62b 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -64,7 +64,7 @@ Common gateway options: |---|---| | `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. For gateway-managed Docker, Podman, and VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` together in `[openshell.gateway]`; driver tables reject those gateway-owned fields. See the [Gateway Configuration File](./gateway-config) reference for the full schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -255,6 +255,12 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. +Podman preserves its runtime-selected AppArmor profile when +`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the +supervisor's mount setup requires it. Explicit `RuntimeDefault` and +`Localhost/` selections fail startup when Podman reports that AppArmor +is unavailable. + On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -597,7 +603,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created so a driver upgrade does not rewrite their ownership contract. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. ### Custom Images diff --git a/openshell.spec b/openshell.spec index ac57d29ee9..13591a1014 100644 --- a/openshell.spec +++ b/openshell.spec @@ -110,6 +110,8 @@ install -Dpm 0755 "${OPENSHELL_PREBUILT_BINARIES_DIR}/%{name}-gateway" %{buildro # Shipped as a read-only reference in %{_datadir}. The systemd unit seeds a # user-level copy at ~/.config/openshell/gateway.toml on first start. install -Dpm 0644 deploy/rpm/gateway.toml.default %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default +install -Dpm 0644 deploy/rpm/gateway.toml.default.v1 %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default.v1 +install -Dpm 0755 deploy/rpm/migrate-gateway-config.sh %{buildroot}%{_libexecdir}/%{name}-gateway-migrate-config # --- Gateway systemd user unit --- # Installed to the systemd user unit directory so any user can run: @@ -129,11 +131,10 @@ Type=exec # the CLI discovers them automatically. # See /usr/share/doc/openshell-gateway/ for details. -# Seed a default TOML config on first start if the user has not created one. -# The template ships at /usr/share/openshell-gateway/gateway.toml.default. -# Edit ~/.config/openshell/gateway.toml to customize. +# Seed a default TOML config on first start. On upgrade, replace only the exact +# schema-v1 config previously seeded by this package; preserve edited files. # %%E expands to $XDG_CONFIG_HOME (~/.config) in user units. -ExecStartPre=/bin/sh -c 'test -f %%E/openshell/gateway.toml || install -Dm644 /usr/share/openshell-gateway/gateway.toml.default %%E/openshell/gateway.toml' +ExecStartPre=%{_libexecdir}/%{name}-gateway-migrate-config %%E/openshell/gateway.toml /usr/share/openshell-gateway/gateway.toml.default /usr/share/openshell-gateway/gateway.toml.default.v1 # Auto-generate PKI on first start if not present. # The default local TLS dir uses %%h because %%S resolves differently across @@ -217,10 +218,12 @@ PYTHONPATH=%{buildroot}%{python3_sitelib} %{python3} -c "from importlib.metadata # A missing template means first-start seeding silently falls back to the # binary default of 127.0.0.1, which breaks Podman sandbox connectivity. test -f %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default +test -f %{buildroot}%{_datadir}/%{name}-gateway/gateway.toml.default.v1 +test -x %{buildroot}%{_libexecdir}/%{name}-gateway-migrate-config -# Verify the systemd unit references the template in its ExecStartPre seed step. -# If this grep fails, the first-start seeding logic was removed from the unit. -grep -q 'gateway.toml.default' %{buildroot}%{_userunitdir}/%{name}-gateway.service +# Verify the systemd unit invokes exact-default migration before startup. +grep -q '%{name}-gateway-migrate-config' %{buildroot}%{_userunitdir}/%{name}-gateway.service +grep -q 'gateway.toml.default.v1' %{buildroot}%{_userunitdir}/%{name}-gateway.service %post gateway %systemd_user_post %{name}-gateway.service @@ -248,7 +251,9 @@ grep -q 'gateway.toml.default' %{buildroot}%{_userunitdir}/%{name}-gateway.servi %doc %{_docdir}/%{name}-gateway/TROUBLESHOOTING.md %{_bindir}/%{name}-gateway %{_userunitdir}/%{name}-gateway.service +%{_libexecdir}/%{name}-gateway-migrate-config %{_datadir}/%{name}-gateway/gateway.toml.default +%{_datadir}/%{name}-gateway/gateway.toml.default.v1 %{_mandir}/man8/openshell-gateway.8* %files -n python3-%{name} diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index 9f30de1b76..f15b7da800 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -64,9 +64,28 @@ def test_generate_homebrew_formula_uses_tagged_macos_driver_asset_without_defaul flags=re.DOTALL, ) assert generated_config is not None + assert "version = 2" in generated_config.group("contents") assert "[openshell.gateway]" in generated_config.group("contents") assert "bind_address =" not in generated_config.group("contents") - assert 'bind_address = "[::1]:17670"' in formula + + legacy_empty_config = re.search( + r"legacy_empty_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_empty_config is not None + assert "version = 1" in legacy_empty_config.group("contents") + assert "bind_address =" not in legacy_empty_config.group("contents") + + legacy_ipv6_config = re.search( + r"legacy_ipv6_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_ipv6_config is not None + assert "version = 1" in legacy_ipv6_config.group("contents") + assert 'bind_address = "[::1]:17670"' in legacy_ipv6_config.group("contents") + assert "gateway_config.read == legacy_empty_gateway_config_contents ||" in formula assert "gateway_config.read == legacy_ipv6_gateway_config_contents" in formula assert "gateway_config.write gateway_config_contents" in formula assert '# compute_driver = "vm"' not in formula @@ -142,12 +161,15 @@ def test_snap_docker_connect_hook_restarts_gateway() -> None: ) -def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: +def test_rpm_spec_seeds_and_migrates_gateway_defaults() -> None: repo_root = Path(__file__).resolve().parents[2] spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") assert "init-gateway-config.sh" not in spec assert "init-pki.sh" not in spec + assert "migrate-gateway-config.sh" in spec + assert "gateway.toml.default.v1" in spec + assert "%{name}-gateway-migrate-config" in spec assert "Environment=OPENSHELL_LOCAL_TLS_DIR=%%h/.local/state/openshell/tls" in spec assert ( "openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR}" diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py new file mode 100644 index 0000000000..af846678f3 --- /dev/null +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MIGRATOR = REPO_ROOT / "deploy/rpm/migrate-gateway-config.sh" +CURRENT = REPO_ROOT / "deploy/rpm/gateway.toml.default" +LEGACY = REPO_ROOT / "deploy/rpm/gateway.toml.default.v1" + + +def run_migrator( + destination: Path, *, check: bool = True +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(CURRENT), str(LEGACY)], + check=check, + text=True, + capture_output=True, + ) + + +def test_migrator_seeds_missing_config_and_is_idempotent(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_replaces_only_exact_legacy_default(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + destination.write_bytes(LEGACY.read_bytes()) + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_edited_legacy_and_current_configs(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\n" + destination.write_text(edited, encoding="utf-8") + + run_migrator(destination) + assert destination.read_text(encoding="utf-8") == edited + + destination.write_bytes(CURRENT.read_bytes()) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_refuses_symlink_destination(tmp_path: Path) -> None: + target = tmp_path / "target.toml" + target.write_text("operator-owned\n", encoding="utf-8") + destination = tmp_path / "gateway.toml" + destination.symlink_to(target) + + result = run_migrator(destination, check=False) + + assert result.returncode != 0 + assert "non-regular gateway config" in result.stderr + assert target.read_text(encoding="utf-8") == "operator-owned\n" diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index e544e844da..8007236d7a 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. ## Motivation -The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: +Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: -- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. -- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. -- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. +- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. +- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. +- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. ## Non-goals @@ -87,10 +87,10 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) +# TLS / mTLS — package-managed local TLS may supply listener defaults. # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway -# ignores the [openshell.gateway.tls] table below. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a +# plaintext listener; guest TLS fields must then be omitted. disable_tls = false # Gateway-owned TLS bundle injected into the selected local driver. @@ -202,9 +202,9 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. +- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver name with no matching `[openshell.drivers.]` table runs with its built-in defaults. The legacy `compute_drivers` list is rejected. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. ### Schema compatibility @@ -220,7 +220,8 @@ version = 2 bind_address = "0.0.0.0:8080" compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). -# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). +# The gateway runs plaintext behind Envoy / ingress. +disable_tls = true [openshell.drivers.kubernetes] namespace = "agents" @@ -231,12 +232,7 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: - -1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. -2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. -3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. -4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. +The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. ```yaml # values.yaml excerpt @@ -255,23 +251,15 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation plan - -No part of this RFC has shipped yet. The work breaks down as: +## Implementation -1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. -2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. -3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. -4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. -5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. -6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. -7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. +The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. +- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -290,8 +278,7 @@ No part of this RFC has shipped yet. The work breaks down as: ## Open questions -1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. -2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index c981ec9d93..e7918e6ab0 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -94,9 +94,15 @@ Gateway configuration requires `[openshell] version = 2`, a singular `compute_driver` selector, and driver-owned settings under `[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, `--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating -them. Guest TLS CA, certificate, and key paths are the exception: configure the -complete bundle under `[openshell.gateway]`, and the gateway injects it only -into the selected local driver. +them. Homebrew and RPM package startup migrates only exact package-generated v1 +defaults. If an upgraded package still reports an unsupported version, inspect +the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must +follow the published schema-v2 migration steps and must not be overwritten. +Guest TLS CA, certificate, and key paths are the exception to driver ownership: +configure the complete bundle under `[openshell.gateway]`, and the gateway +injects it only into the selected local driver. TLS-enabled Docker, Podman, and +VM drivers fail startup when neither those paths nor the package-managed local +bundle is available; Kubernetes projects its bundle through a Secret. Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. @@ -618,6 +624,11 @@ Use the VM driver logs and host diagnostics available in the user's environment. - The VM driver process is running and reachable by the gateway. - The runtime rootfs exists and matches the expected architecture. +- `mke2fs` or `mkfs.ext4` and `debugfs` from e2fsprogs are installed; explicit + `sandbox_uid`/`sandbox_gid` does not remove this prerequisite. +- A persisted overlay identity error is resolved from its owner marker, overlay + upper layer, prepared rootfs, explicit config, or current image. Do not assign + `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. - The sandbox supervisor can establish its callback connection to the gateway. diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index d6c6e1efe6..188c698962 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -24,12 +24,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index d1d86ad4a2..4d990d629f 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -21,12 +21,14 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -223,6 +225,9 @@ ttl_secs = 3600 default_image = "${SANDBOX_IMAGE}" supervisor_image = "${SUPERVISOR_IMAGE}" image_pull_policy = "${SANDBOX_IMAGE_PULL_POLICY}" +# Local development requires supervisor mount setup that Podman's runtime +# profile may deny. Production configs preserve Podman's default when omitted. +app_armor_profile = "Unconfined" health_check_interval_secs = 10 EOF diff --git a/tasks/scripts/gateway-pull-policy.sh b/tasks/scripts/gateway-pull-policy.sh new file mode 100755 index 0000000000..7c0d1bd741 --- /dev/null +++ b/tasks/scripts/gateway-pull-policy.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Normalize compatibility inputs at the development-script boundary while +# keeping schema-v2 TOML and backend validation strict. +normalize_image_pull_policy() { + local value + value="$(printf '%s' "${1:-}" | LC_ALL=C tr '[:upper:]' '[:lower:]')" + case "${value}" in + always) + printf '%s\n' "always" + ;; + if_not_present|ifnotpresent|missing) + printf '%s\n' "if_not_present" + ;; + never) + printf '%s\n' "never" + ;; + newer) + printf '%s\n' "newer" + ;; + *) + printf 'unsupported image pull policy: %s\n' "${1:-}" >&2 + return 2 + ;; + esac +} diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index da7f91fb68..300b6a8c7e 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -16,6 +16,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" usage() { @@ -206,7 +208,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index 29a503567a..3b0afe5ecd 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -416,9 +416,17 @@ def post_install [openshell.gateway] TOML + # These are the only v1 configurations emitted by pre-schema-v2 formulas. + # Do not migrate a configuration unless it exactly matches one of them. + legacy_empty_gateway_config_contents = <<~TOML + [openshell] + version = 1 + + [openshell.gateway] + TOML legacy_ipv6_gateway_config_contents = <<~TOML [openshell] - version = 2 + version = 1 [openshell.gateway] bind_address = "[::1]:{LOCAL_GATEWAY_PORT}" @@ -426,9 +434,9 @@ def post_install unless gateway_config.exist? gateway_config.write gateway_config_contents else - # Migrate only the exact config generated by the affected formula. Keep - # any user-edited config untouched. - if gateway_config.read == legacy_ipv6_gateway_config_contents + # Keep any user-edited config untouched. + if gateway_config.read == legacy_empty_gateway_config_contents || + gateway_config.read == legacy_ipv6_gateway_config_contents gateway_config.write gateway_config_contents end end diff --git a/tasks/scripts/test-gateway-pull-policy.sh b/tasks/scripts/test-gateway-pull-policy.sh new file mode 100755 index 0000000000..267c302fd0 --- /dev/null +++ b/tasks/scripts/test-gateway-pull-policy.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" + +assert_policy() { + local input=$1 + local expected=$2 + local actual + actual="$(normalize_image_pull_policy "${input}")" + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %q -> %q, got %q\n' "${input}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +for input in always Always ALWAYS; do + assert_policy "${input}" always +done +for input in if_not_present IfNotPresent ifnotpresent IFNOTPRESENT missing MISSING; do + assert_policy "${input}" if_not_present +done +for input in never Never NEVER; do + assert_policy "${input}" never +done +for input in newer Newer NEWER; do + assert_policy "${input}" newer +done + +if normalize_image_pull_policy sometimes >/dev/null 2>&1; then + echo "unsupported policy unexpectedly succeeded" >&2 + exit 1 +fi + +for script in gateway.sh gateway-docker.sh gateway-podman.sh; do + if ! grep -q 'normalize_image_pull_policy' "${ROOT}/tasks/scripts/${script}"; then + echo "${script} does not normalize image pull policy" >&2 + exit 1 + fi +done + +echo "gateway pull-policy tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index 4a5cda0890..a310e1e229 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -12,6 +12,7 @@ depends = [ "test:sbom", "test:install-sh", "test:build-env", + "test:gateway-pull-policy", "test:packaging-assets", "test:codex-security-release-range", "test:docs-website", @@ -40,6 +41,12 @@ run = "tasks/scripts/test-build-env.sh" run_windows = "echo Skipping test:build-env: the Unix build-env.sh helper does not apply on Windows." hide = true +["test:gateway-pull-policy"] +description = "Test development gateway image pull-policy normalization" +run = "tasks/scripts/test-gateway-pull-policy.sh" +run_windows = "echo Skipping test:gateway-pull-policy: Unix gateway scripts do not apply on Windows." +hide = true + ["test:packaging-assets"] description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh" From fef7275972591cd408368e4cca608e2975a9e9de Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 09:59:36 -0400 Subject: [PATCH 07/42] test(config): expand schema v2 regression coverage Signed-off-by: Jesse Jaggars --- .github/workflows/branch-checks.yml | 21 ++ crates/openshell-core/src/config.rs | 192 +++++++++++++++++- crates/openshell-driver-docker/src/tests.rs | 94 +++++++++ .../openshell-driver-kubernetes/src/config.rs | 35 +++- crates/openshell-driver-podman/src/config.rs | 12 ++ crates/openshell-driver-podman/src/watcher.rs | 19 ++ crates/openshell-driver-vm/src/driver.rs | 131 +++++++++++- crates/openshell-gateway/src/lib.rs | 14 +- crates/openshell-gateway/src/vm.rs | 42 ++++ crates/openshell-server/src/cli.rs | 67 ++++++ .../src/compute/driver_config.rs | 96 +++++++++ crates/openshell-server/src/lib.rs | 9 + .../openshell/tests/gateway_config_test.yaml | 95 +++++++++ deploy/rpm/gateway.toml.default.v1 | 2 +- deploy/rpm/migrate-gateway-config.sh | 3 +- .../openshell/gateway_config_fixture_test.py | 49 +++++ python/openshell/release_formula_test.py | 12 ++ .../rpm_gateway_config_migration_test.py | 156 +++++++++++++- tasks/scripts/gateway-docker.sh | 14 +- tasks/scripts/gateway-podman.sh | 15 +- tasks/scripts/gateway-toml.sh | 19 ++ tasks/scripts/gateway-vm.sh | 15 +- tasks/scripts/gateway.sh | 2 +- tasks/scripts/test-gateway-config.sh | 35 ++++ tasks/scripts/test-gateway-pull-policy.sh | 29 +++ tasks/test.toml | 7 + 26 files changed, 1127 insertions(+), 58 deletions(-) create mode 100644 python/openshell/gateway_config_fixture_test.py create mode 100644 tasks/scripts/gateway-toml.sh create mode 100755 tasks/scripts/test-gateway-config.sh diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 958d0b53c0..8148618d0f 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -300,3 +300,24 @@ jobs: run: | OPENSHELL_NPM_VERSION="$(uv run python tasks/scripts/release.py get-version --npm)" \ mise run sdk:ts:publish + + gateway-config: + name: Gateway configuration fixtures + needs: pr_metadata + if: needs.pr_metadata.outputs.should_run == 'true' + runs-on: linux-amd64-cpu8 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install tools + run: mise install --locked + + - name: Test local gateway configuration helpers + run: | + bash tasks/scripts/test-gateway-pull-policy.sh + bash tasks/scripts/test-gateway-config.sh diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index b5429d6d19..f28c8368e3 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -1046,10 +1046,10 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ - Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, + AppArmorProfile, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, GatewayProviderProfileSourceConfig, ImagePullPolicy, PolicyValidationFailureMode, - normalize_compute_driver_name, + UpstreamProxyConfig, default_sandbox_pids_limit, normalize_compute_driver_name, }; use std::net::SocketAddr; use std::time::Duration; @@ -1193,6 +1193,194 @@ mod tests { assert!("IfNotPresent".parse::().is_err()); } + #[test] + fn app_armor_profiles_round_trip_and_translate_for_each_backend() { + for (value, expected, kubernetes_type, localhost_profile, oci_security_opt) in [ + ( + "RuntimeDefault", + AppArmorProfile::RuntimeDefault, + "RuntimeDefault", + None, + None, + ), + ( + "Unconfined", + AppArmorProfile::Unconfined, + "Unconfined", + None, + Some("apparmor=unconfined"), + ), + ( + "Localhost/openshell-supervisor", + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "Localhost", + Some("openshell-supervisor"), + Some("apparmor=openshell-supervisor"), + ), + ] { + let parsed = value.parse::().expect("valid profile"); + assert_eq!(parsed, expected); + assert_eq!(parsed.to_string(), value); + assert_eq!(parsed.kubernetes_type(), kubernetes_type); + assert_eq!(parsed.localhost_profile(), localhost_profile); + assert_eq!(parsed.oci_security_opt().as_deref(), oci_security_opt); + + let json = serde_json::to_value(&parsed).expect("profile serializes"); + assert_eq!(json, value); + assert_eq!( + serde_json::from_value::(json).expect("profile deserializes"), + parsed + ); + } + + for invalid in [ + "Localhost/", + "Localhost/openshell profile", + "runtimeDefault", + "unconfined", + "Unknown", + ] { + assert!( + invalid.parse::().is_err(), + "{invalid} should be rejected" + ); + } + } + + #[test] + fn upstream_proxy_validation_enforces_cross_field_contract() { + let auth_file = Some("/run/secrets/proxy-auth".into()); + let cases = [ + ("default", UpstreamProxyConfig::default(), None), + ( + "https auth", + UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + proxy_auth_file: auth_file.clone(), + ..Default::default() + }, + None, + ), + ( + "acknowledged http auth", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_file: auth_file.clone(), + proxy_auth_allow_insecure: Some(true), + ..Default::default() + }, + None, + ), + ( + "unacknowledged http auth", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_file: auth_file.clone(), + ..Default::default() + }, + Some("proxy_auth_allow_insecure"), + ), + ( + "no_proxy without proxy", + UpstreamProxyConfig { + no_proxy: Some("localhost".to_string()), + ..Default::default() + }, + Some("no_proxy"), + ), + ( + "blank no_proxy", + UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(" ".to_string()), + ..Default::default() + }, + Some("no_proxy"), + ), + ( + "auth file without proxy", + UpstreamProxyConfig { + proxy_auth_file: auth_file, + ..Default::default() + }, + Some("proxy_auth_file"), + ), + ( + "ack without auth file", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_allow_insecure: Some(true), + ..Default::default() + }, + Some("proxy_auth_allow_insecure"), + ), + ( + "hostname mode without proxy", + UpstreamProxyConfig { + proxy_connect_by_hostname: Some(true), + ..Default::default() + }, + Some("proxy_connect_by_hostname"), + ), + ( + "empty proxy", + UpstreamProxyConfig { + https_proxy: Some(String::new()), + ..Default::default() + }, + Some("https_proxy"), + ), + ( + "inline credentials", + UpstreamProxyConfig { + https_proxy: Some( + "https://secret-user:secret-password@proxy.example:8443".to_string(), + ), + ..Default::default() + }, + Some("must not embed credentials"), + ), + ]; + + for (name, config, expected_error) in cases { + match expected_error { + None => config + .validate() + .unwrap_or_else(|error| panic!("{name}: {error}")), + Some(expected) => { + let error = match config.validate() { + Ok(()) => panic!("{name} should fail validation"), + Err(error) => error, + }; + assert!(error.contains(expected), "{name}: {error}"); + assert!(!error.contains("secret-user"), "{name}: {error}"); + assert!(!error.contains("secret-password"), "{name}: {error}"); + } + } + } + } + + #[test] + fn config_defaults_and_builder_use_singular_compute_driver() { + let config = Config::new(None); + assert_eq!(config.compute_driver, None); + assert_eq!( + Config::new(None) + .with_compute_driver("podman") + .compute_driver + .as_deref(), + Some("podman") + ); + } + + #[test] + fn typed_sandbox_pids_default_matches_positive_constant() { + assert_eq!( + default_sandbox_pids_limit().map(std::num::NonZeroI64::get), + Some(super::DEFAULT_SANDBOX_PIDS_LIMIT) + ); + } + #[test] fn name_defaults_and_can_be_overridden() { assert_eq!(Config::new(None).name, "openshell"); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index fc4cfea305..e257a8594b 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -191,6 +191,57 @@ fn docker_rejects_newer_image_pull_policy() { assert!(error.to_string().contains("supported only by the Podman")); } +#[test] +fn docker_apparmor_profiles_render_and_require_daemon_capability() { + for (profile, expected) in [ + (AppArmorProfile::RuntimeDefault, None), + ( + AppArmorProfile::Unconfined, + Some(vec!["apparmor=unconfined".to_string()]), + ), + ( + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + Some(vec!["apparmor=openshell-supervisor".to_string()]), + ), + ] { + let mut config = runtime_config(); + config.app_armor_profile = Some(profile.clone()); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + assert_eq!(body.host_config.unwrap().security_opt, expected); + } + + let unavailable = SystemInfo::default(); + assert!( + validate_docker_app_armor_profile(Some(&AppArmorProfile::Unconfined), &unavailable).is_ok() + ); + for confined in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + let error = validate_docker_app_armor_profile(Some(&confined), &unavailable) + .expect_err("confined profile requires daemon AppArmor support"); + assert!( + error + .to_string() + .contains("Docker reports it is unavailable") + ); + } + + let available = SystemInfo { + security_options: Some(vec!["name=apparmor".to_string()]), + ..Default::default() + }; + assert!( + validate_docker_app_armor_profile( + Some(&AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )), + &available + ) + .is_ok() + ); +} + #[test] fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { let config: DockerComputeConfig = toml::from_str( @@ -1304,6 +1355,17 @@ fn container_create_body_omits_pids_limit_by_default() { assert_eq!(host_config.pids_limit, None); } +#[test] +fn container_create_body_emits_configured_positive_pids_limit() { + let mut config = runtime_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(4096); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + assert_eq!( + body.host_config.expect("host config").pids_limit, + Some(4096) + ); +} + #[test] fn build_environment_sets_docker_tls_paths() { let env = build_environment(&test_sandbox(), &runtime_config()); @@ -2260,6 +2322,12 @@ fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { .windows(2) .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-no-proxy", ".svc"]) + ); + assert!(command.contains(&"--upstream-proxy-connect-by-hostname".to_string())); let binds = body.host_config.unwrap().binds.unwrap(); assert!( binds @@ -3131,6 +3199,32 @@ fn docker_guest_tls_paths_allows_plain_http_without_tls_flags() { assert!(result.is_none()); } +#[test] +fn docker_automatic_tls_detection_is_fail_closed_for_partial_bundles() { + for mask in 0_u8..8 { + let config = DockerComputeConfig { + guest_tls_ca: (mask & 1 != 0).then(|| PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: (mask & 2 != 0).then(|| PathBuf::from("/tmp/cert.pem")), + guest_tls_key: (mask & 4 != 0).then(|| PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + assert_eq!( + docker_guest_tls_configured(&config), + mask != 0, + "TLS presence mask {mask:03b}" + ); + + if mask != 0 && mask != 7 { + let mut inferred = config; + inferred.grpc_endpoint = "https://host.openshell.internal:8080".to_string(); + assert!( + docker_guest_tls_paths(&inferred).is_err(), + "partial TLS presence mask {mask:03b} must fail" + ); + } + } +} + #[test] fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { let image = openshell_core::config::default_supervisor_image(); diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index bfdceff9d4..56b553c351 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -903,20 +903,33 @@ mod tests { cfg.supervisor_image_pull_policy, Some(ImagePullPolicy::Never) ); - assert_eq!( - KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::IfNotPresent), - "IfNotPresent" - ); + + for (policy, expected) in [ + (ImagePullPolicy::Always, "Always"), + (ImagePullPolicy::IfNotPresent, "IfNotPresent"), + (ImagePullPolicy::Never, "Never"), + ] { + assert_eq!( + KubernetesComputeConfig::image_pull_policy_value(policy), + expected + ); + } } #[test] - fn image_pull_policy_rejects_newer() { - let cfg = KubernetesComputeConfig { - image_pull_policy: Some(ImagePullPolicy::Newer), - ..KubernetesComputeConfig::default() - }; - let error = cfg.validate_image_pull_policies().unwrap_err(); - assert!(error.contains("supported only by the Podman")); + fn image_pull_policy_rejects_newer_for_sandbox_and_supervisor_images() { + for (sandbox, supervisor) in [ + (Some(ImagePullPolicy::Newer), None), + (None, Some(ImagePullPolicy::Newer)), + ] { + let cfg = KubernetesComputeConfig { + image_pull_policy: sandbox, + supervisor_image_pull_policy: supervisor, + ..KubernetesComputeConfig::default() + }; + let error = cfg.validate_image_pull_policies().unwrap_err(); + assert!(error.contains("supported only by the Podman")); + } } #[test] diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 855b7f0e24..ba6881800a 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -512,6 +512,18 @@ impl std::fmt::Debug for PodmanComputeConfig { mod tests { use super::*; + #[test] + fn shared_image_pull_policies_map_to_podman_vocabulary() { + for (policy, expected) in [ + (ImagePullPolicy::Always, "always"), + (ImagePullPolicy::IfNotPresent, "missing"), + (ImagePullPolicy::Never, "never"), + (ImagePullPolicy::Newer, "newer"), + ] { + assert_eq!(podman_image_pull_policy(policy), expected); + } + } + #[test] fn config_uses_canonical_ssh_socket_path_name() { let config: PodmanComputeConfig = diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 257d649a76..b8b6827394 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -602,6 +602,25 @@ mod tests { assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); } + #[test] + fn condition_running_with_pending_healthcheck_is_not_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: Some(HealthState { + status: "starting".to_string(), + }), + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let condition = condition_from_state(&state); + assert_eq!(condition.r#type, "Ready"); + assert_eq!(condition.status, "False"); + assert_eq!(condition.reason, "HealthCheckStarting"); + } + #[test] fn condition_oom_killed() { let state = ContainerState { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 6656a4dab8..677e44d6c3 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -5171,6 +5171,7 @@ async fn sandbox_owner_identity_from_image( .await .map_err(|error| format!("read sandbox identity task failed: {error}"))??; let (uid, gid) = identity.unwrap_or((DEFAULT_SANDBOX_UID, DEFAULT_SANDBOX_UID)); + validate_sandbox_owner_identity(uid, gid)?; Ok(SandboxOwnerIdentity { uid, gid }) } @@ -5183,7 +5184,12 @@ async fn sandbox_owner_identity_from_overlay( }) .await .map_err(|error| format!("read sandbox overlay identity task failed: {error}"))??; - Ok(identity.map(|(uid, gid)| SandboxOwnerIdentity { uid, gid })) + identity + .map(|(uid, gid)| { + validate_sandbox_owner_identity(uid, gid)?; + Ok(SandboxOwnerIdentity { uid, gid }) + }) + .transpose() } fn parse_sandbox_owner_state(contents: &str) -> Result { @@ -7714,6 +7720,87 @@ mod tests { } } + #[tokio::test] + async fn image_and_overlay_owner_evidence_rejects_root_identity() { + let dir = unique_temp_dir(); + let image_source = dir.join("image-source"); + std::fs::create_dir_all(image_source.join("etc")).unwrap(); + std::fs::write( + image_source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:0:0:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let image = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&image_source, &image, 32 * 1024 * 1024).unwrap(); + let image_error = sandbox_owner_identity_from_image(&image) + .await + .expect_err("root image identity must be rejected"); + assert!(image_error.contains("uid 0 is outside the allowed range")); + + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:0:0:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let overlay_error = sandbox_owner_identity_from_overlay(&overlay) + .await + .expect_err("root overlay identity must be rejected"); + assert!(overlay_error.contains("uid 0 is outside the allowed range")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn legacy_owner_marker_uses_evidence_migration_and_new_markers_are_private() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join(SANDBOX_OWNER_STATE_FILE), + format!("{SANDBOX_OWNER_STATE_V1}\n"), + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000 + } + ); + assert!(write_marker); + + write_sandbox_owner_state(&dir, identity).await.unwrap(); + let marker = dir.join(SANDBOX_OWNER_STATE_FILE); + assert_eq!( + std::fs::read_to_string(&marker).unwrap(), + "sandbox-owner-v2:2000:3000\n" + ); + assert_eq!( + std::fs::metadata(marker).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let _ = std::fs::remove_dir_all(dir); + } + #[tokio::test] async fn persisted_owner_marker_preserves_exact_identity() { let dir = unique_temp_dir(); @@ -8739,6 +8826,8 @@ mod tests { assert!( env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY=https://proxy.example:8443".to_string()) ); + assert!(env.contains(&"OPENSHELL_VM_UPSTREAM_NO_PROXY=.svc".to_string())); + assert!(env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME=true".to_string())); assert!(env.contains(&format!( "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE={GUEST_UPSTREAM_PROXY_AUTH_PATH}" ))); @@ -9028,14 +9117,48 @@ mod tests { } #[test] - fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { + fn prepared_image_cache_identity_includes_layout_version_and_owner_contract() { + let image = "sha256:local-image"; + let image_account = prepared_image_cache_identity(image, &VmDriverConfig::default()); assert_eq!( - prepared_image_cache_identity("sha256:local-image", &VmDriverConfig::default()), + image_account, format!( - "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:sha256:local-image", + "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:{image}", openshell_core::VERSION ) ); + + let identities = [ + VmDriverConfig { + sandbox_uid: Some(1000), + sandbox_gid: Some(1000), + ..Default::default() + }, + VmDriverConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }, + VmDriverConfig { + sandbox_uid: Some(2000), + ..Default::default() + }, + VmDriverConfig { + sandbox_gid: Some(3000), + ..Default::default() + }, + ] + .map(|config| prepared_image_cache_identity(image, &config)); + + assert!(identities.iter().all(|identity| identity != &image_account)); + for (index, identity) in identities.iter().enumerate() { + assert!( + identities[index + 1..] + .iter() + .all(|other| other != identity), + "owner contracts must use distinct cache keys" + ); + } } #[test] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 5645eba695..dc97fc9ffe 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -349,9 +349,21 @@ fn apply_guest_tls( #[cfg(all(test, not(target_os = "windows"), feature = "in-tree-compute-drivers"))] mod local_driver_tests { - use super::{apply_guest_tls, validate_local_driver_guest_tls}; + use super::{ + apply_guest_tls, install_default_compute_drivers, validate_local_driver_guest_tls, + }; use std::path::{Path, PathBuf}; + #[test] + fn linux_builtin_compute_driver_registry_has_expected_names() { + assert_eq!( + install_default_compute_drivers() + .installed_driver_names() + .collect::>(), + ["docker", "kubernetes", "podman", "vm"] + ); + } + #[test] fn tls_enabled_local_drivers_require_a_guest_bundle() { for driver_name in ["docker", "podman", "vm"] { diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 70d4e266cd..c26220e12d 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -704,6 +704,7 @@ mod tests { prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, validate_vm_sandbox_identity, }; + use openshell_core::UpstreamProxyConfig; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; @@ -825,6 +826,47 @@ mod tests { assert_eq!(args, ["--sandbox-uid", "2000", "--sandbox-gid", "3000"]); } + #[test] + fn vm_driver_command_forwards_proxy_and_spiffe_configuration_without_credentials() { + let config = VmComputeConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.internal:8443".to_string()), + no_proxy: Some("localhost,.svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/gateway/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_vm_proxy_and_spiffe_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "--upstream-proxy", + "https://proxy.internal:8443", + "--upstream-no-proxy", + "localhost,.svc", + "--upstream-proxy-auth-file", + "/gateway/secrets/proxy-auth", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--provider-spiffe-workload-api-tcp-endpoint", + "tcp:192.0.2.10:8081", + "--provider-spiffe-allow-guest-tcp", + ] + ); + assert!(!args.iter().any(|arg| arg.contains("user:password"))); + } + #[test] fn vm_gateway_config_rejects_root_identity() { let config = VmComputeConfig { diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index f357b8e3a1..c1a42d72a5 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -1123,6 +1123,36 @@ mod tests { assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); } + #[test] + fn command_rejects_legacy_compute_driver_flags() { + for flag in ["--driver", "--drivers"] { + let err = command() + .try_get_matches_from([ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + flag, + "docker", + ]) + .expect_err("legacy compute driver selector must be rejected"); + assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); + } + } + + #[test] + fn legacy_compute_driver_environment_is_rejected_even_when_empty() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for value in ["docker", ""] { + let guard = EnvVarGuard::set("OPENSHELL_DRIVERS", value); + let error = super::reject_legacy_driver_selector_env() + .expect_err("legacy environment selector must be rejected"); + assert!(error.to_string().contains("OPENSHELL_COMPUTE_DRIVER")); + drop(guard); + } + } + #[test] fn command_rejects_removed_ssh_endpoint_flags() { for flag in [ @@ -1591,6 +1621,43 @@ log_level = "debug" assert_eq!(args.name, "env-gateway"); } + #[test] + fn compute_driver_file_value_and_cli_environment_precedence_are_explicit() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let file = config_file_from_toml( + r#" +[openshell.gateway] +compute_driver = "podman" +"#, + ); + + let canonical_guard = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let (mut file_args, file_matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut file_args, &file.openshell.gateway, &file_matches); + assert_eq!(file_args.compute_driver.as_deref(), Some("podman")); + + let (mut cli_args, cli_matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "docker", + ]); + merge_file_into_args(&mut cli_args, &file.openshell.gateway, &cli_matches); + assert_eq!(cli_args.compute_driver.as_deref(), Some("docker")); + drop(canonical_guard); + + let _canonical = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "vm"); + let (mut env_args, env_matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut env_args, &file.openshell.gateway, &env_matches); + assert_eq!(env_args.compute_driver.as_deref(), Some("vm")); + } + #[test] fn file_oidc_block_populates_oidc_args() { let _lock = ENV_LOCK diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index a9d7faa519..40ab449c66 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -330,6 +330,102 @@ mod tests { assert!(error.contains("require gateway TLS")); } + #[derive(Debug, Default, Deserialize)] + struct EmptyDriverConfig {} + + #[test] + fn driver_owned_guest_tls_fields_are_rejected_for_local_and_remote_drivers() { + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + let source = format!( + r#" +[openshell] +version = 2 + +[openshell.drivers.kyma] +socket_path = "/run/openshell/kyma.sock" +{field} = "/run/openshell/guest.pem" +"# + ); + let file: config_file::ConfigFile = toml::from_str(&source).expect("valid TOML"); + + let local_error = + driver_config_from_context::(test_context(Some(&file)), "kyma") + .expect_err("local driver TLS field must be rejected"); + assert!(local_error.to_string().contains(field)); + assert!(local_error.to_string().contains("[openshell.gateway]")); + + let remote_error = remote_driver_config_from_context(test_context(Some(&file)), "kyma") + .expect_err("remote driver TLS field must be rejected"); + assert!(remote_error.to_string().contains(field)); + assert!(remote_error.to_string().contains("[openshell.gateway]")); + } + } + + #[test] + fn explicit_gateway_guest_tls_takes_precedence_over_package_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let explicit = [ + dir.path().join("explicit-ca.pem"), + dir.path().join("explicit-cert.pem"), + dir.path().join("explicit-key.pem"), + ]; + for path in &explicit { + std::fs::write(path, b"explicit").expect("write explicit TLS fixture"); + } + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(explicit[0].clone()), + guest_tls_cert: Some(explicit[1].clone()), + guest_tls_key: Some(explicit[2].clone()), + ..Default::default() + }; + let package = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), Some(&package), false) + .expect("explicit bundle resolves") + .expect("guest bundle"); + assert_eq!( + resolved.as_paths(), + ( + explicit[0].as_path(), + explicit[1].as_path(), + explicit[2].as_path() + ) + ); + } + + #[test] + fn gateway_guest_tls_rejects_directories_for_every_bundle_member() { + let dir = tempfile::tempdir().expect("temp dir"); + let files = [ + dir.path().join("ca.pem"), + dir.path().join("cert.pem"), + dir.path().join("key.pem"), + ]; + for path in &files { + std::fs::write(path, b"fixture").expect("write TLS fixture"); + } + + for index in 0..files.len() { + let mut paths = files.clone(); + paths[index] = dir.path().to_path_buf(); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(paths[0].clone()), + guest_tls_cert: Some(paths[1].clone()), + guest_tls_key: Some(paths[2].clone()), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("directory TLS input must be rejected"); + assert!(error.contains("not a file"), "{error}"); + } + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c5c76548f3..18c86efaed 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1656,6 +1656,15 @@ mod tests { assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(15 * 60)); } + #[test] + fn extension_token_ttl_is_capped_at_one_hour() { + let issuer = extension_test_issuer_with_ttl(Some(Duration::from_secs(24 * 60 * 60))); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(60 * 60)); + + let short = extension_test_issuer_with_ttl(Some(Duration::from_secs(5 * 60))); + assert_eq!(extension_token_ttl(&short), Duration::from_secs(5 * 60)); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 7130871f22..2878a80f61 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -807,3 +807,98 @@ tests: asserts: - failedTemplate: errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." + + - it: does not render the schema-v1 compute_drivers selector + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: "(?m)^\\s*compute_drivers\\s*=" + + - it: uses the release fullname and namespace for default Kubernetes settings + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)name\\s*=\\s*\\\"openshell\\\".*?\\[openshell\\.drivers\\.kubernetes\\].*?namespace\\s*=\\s*\\\"my-namespace\\\".*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + + - it: uses an explicit server grpc endpoint verbatim + template: templates/gateway-config.yaml + set: + server.grpcEndpoint: https://gateway.example.test:9443 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"https://gateway\\.example\\.test:9443\\\"" + + - it: uses HTTP callback and omits client TLS secret when TLS is disabled + template: templates/gateway-config.yaml + set: + server.disableTls: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"http://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + - notMatchRegex: + path: data["gateway.toml"] + pattern: "client_tls_secret_name\\s*=" + + - it: accepts Always pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: Always + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" + + - it: accepts IfNotPresent pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: IfNotPresent + supervisor.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" + + - it: accepts Never pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Never + supervisor.image.pullPolicy: Never + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" + + - it: accepts canonical lowercase pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: always + supervisor.image.pullPolicy: always + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" + + - it: accepts canonical if_not_present pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: if_not_present + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" + + - it: accepts canonical never pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: never + supervisor.image.pullPolicy: never + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 index ba76f873b2..cd7e0d99c3 100644 --- a/deploy/rpm/gateway.toml.default.v1 +++ b/deploy/rpm/gateway.toml.default.v1 @@ -25,4 +25,4 @@ version = 1 # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. -compute_driver = "podman" +compute_drivers = ["podman"] diff --git a/deploy/rpm/migrate-gateway-config.sh b/deploy/rpm/migrate-gateway-config.sh index 5c75af1f9f..4794ffa454 100755 --- a/deploy/rpm/migrate-gateway-config.sh +++ b/deploy/rpm/migrate-gateway-config.sh @@ -14,7 +14,8 @@ current_default=$2 legacy_default=$3 for source in "$current_default" "$legacy_default"; do - if [ ! -f "$source" ]; then + # Package-owned defaults must be ordinary files; never follow a symlink. + if [ -L "$source" ] || [ ! -f "$source" ]; then echo "gateway config migration source is not a regular file: $source" >&2 exit 1 fi diff --git a/python/openshell/gateway_config_fixture_test.py b/python/openshell/gateway_config_fixture_test.py new file mode 100644 index 0000000000..92f96ebfa6 --- /dev/null +++ b/python/openshell/gateway_config_fixture_test.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Schema-only checks for E2E gateway fixtures; these do not start a runtime.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_DIR = REPO_ROOT / "e2e/configs/gateway" + + +@pytest.mark.parametrize( + ("fixture_name", "driver"), + [("docker.toml", "docker"), ("podman.toml", "podman")], +) +def test_e2e_gateway_fixtures_use_schema_v2_scalar_driver( + fixture_name: str, driver: str +) -> None: + config = tomllib.loads((FIXTURE_DIR / fixture_name).read_text(encoding="utf-8")) + gateway = config["openshell"]["gateway"] + + assert config["openshell"]["version"] == 2 + assert gateway["compute_driver"] == driver + assert "compute_drivers" not in gateway + assert "sandbox_namespace" not in gateway + + +def test_docker_e2e_gateway_fixture_uses_canonical_policy_and_label() -> None: + config = tomllib.loads((FIXTURE_DIR / "docker.toml").read_text(encoding="utf-8")) + docker = config["openshell"]["drivers"]["docker"] + + assert docker["image_pull_policy"] == "if_not_present" + assert docker["sandbox_label"] == "openshell-e2e" + assert "sandbox_namespace" not in docker + + +def test_podman_e2e_gateway_fixture_uses_canonical_policy_and_callbacks() -> None: + config = tomllib.loads((FIXTURE_DIR / "podman.toml").read_text(encoding="utf-8")) + podman = config["openshell"]["drivers"]["podman"] + + assert podman["image_pull_policy"] == "if_not_present" + assert podman["health_check_interval_secs"] == 10 + assert podman["ssh_socket_path"] == "/run/openshell/ssh.sock" + assert "sandbox_namespace" not in podman diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index f15b7da800..aa7b56e432 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -202,3 +202,15 @@ def test_deb_user_service_uses_gateway_defaults_without_config_helper() -> None: assert "ExecStart=/usr/bin/openshell-gateway" in unit assert "--config" not in unit assert "--db-url" not in unit + + +def test_rpm_migration_exec_start_pre_argument_order() -> None: + repo_root = Path(__file__).resolve().parents[2] + spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") + + assert ( + "ExecStartPre=%{_libexecdir}/%{name}-gateway-migrate-config " + "%%E/openshell/gateway.toml " + "/usr/share/openshell-gateway/gateway.toml.default " + "/usr/share/openshell-gateway/gateway.toml.default.v1" + ) in spec diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py index af846678f3..d622f36bb1 100644 --- a/python/openshell/rpm_gateway_config_migration_test.py +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -10,6 +10,35 @@ MIGRATOR = REPO_ROOT / "deploy/rpm/migrate-gateway-config.sh" CURRENT = REPO_ROOT / "deploy/rpm/gateway.toml.default" LEGACY = REPO_ROOT / "deploy/rpm/gateway.toml.default.v1" +SHIPPED_V1_FIXTURE = b"""# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_drivers = ["podman"] +""" def run_migrator( @@ -35,7 +64,10 @@ def test_migrator_seeds_missing_config_and_is_idempotent(tmp_path: Path) -> None def test_migrator_replaces_only_exact_legacy_default(tmp_path: Path) -> None: destination = tmp_path / "gateway.toml" - destination.write_bytes(LEGACY.read_bytes()) + shipped_v1 = SHIPPED_V1_FIXTURE + assert b'compute_drivers = ["podman"]' in shipped_v1 + assert LEGACY.read_bytes() == shipped_v1 + destination.write_bytes(shipped_v1) run_migrator(destination) @@ -67,3 +99,125 @@ def test_migrator_refuses_symlink_destination(tmp_path: Path) -> None: assert result.returncode != 0 assert "non-regular gateway config" in result.stderr assert target.read_text(encoding="utf-8") == "operator-owned\n" + + +def test_shipped_rpm_defaults_have_expected_schema_shapes() -> None: + import tomllib + + current = tomllib.loads(CURRENT.read_text(encoding="utf-8")) + legacy = tomllib.loads(LEGACY.read_text(encoding="utf-8")) + + assert current["openshell"]["version"] == 2 + assert current["openshell"]["gateway"]["compute_driver"] == "podman" + assert "compute_drivers" not in current["openshell"]["gateway"] + assert current["openshell"]["drivers"]["podman"]["health_check_interval_secs"] == 10 + assert LEGACY.read_bytes() == SHIPPED_V1_FIXTURE + assert legacy["openshell"]["version"] == 1 + assert legacy["openshell"]["gateway"]["compute_drivers"] == ["podman"] + assert "compute_driver" not in legacy["openshell"]["gateway"] + + +def test_migrator_rejects_wrong_argument_count() -> None: + result = subprocess.run( + ["sh", str(MIGRATOR)], text=True, capture_output=True, check=False + ) + + assert result.returncode == 2 + assert result.stderr == ( + f"usage: {MIGRATOR} DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT\n" + ) + + +def test_migrator_seeds_with_mode_0644(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_operator_content_and_mode(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\\n" + destination.write_text(edited, encoding="utf-8") + destination.chmod(0o600) + + run_migrator(destination) + + assert destination.read_text(encoding="utf-8") == edited + assert destination.stat().st_mode & 0o777 == 0o600 + + destination.write_bytes(CURRENT.read_bytes()) + destination.chmod(0o640) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o640 + + +def test_migrator_handles_paths_containing_spaces(tmp_path: Path) -> None: + package = tmp_path / "package defaults" + package.mkdir() + current = package / "current default.toml" + legacy = package / "legacy default.toml" + current.write_bytes(CURRENT.read_bytes()) + legacy.write_bytes(LEGACY.read_bytes()) + destination = tmp_path / "operator config/gateway config.toml" + + subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(current), str(legacy)], + check=True, + ) + + assert destination.read_bytes() == current.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_rejects_missing_and_nonregular_sources(tmp_path: Path) -> None: + for source_name in ("current", "legacy"): + for source_kind in ("missing", "directory", "symlink"): + current = tmp_path / f"{source_name}-{source_kind}-current.toml" + legacy = tmp_path / f"{source_name}-{source_kind}-legacy.toml" + current.write_bytes(CURRENT.read_bytes()) + legacy.write_bytes(LEGACY.read_bytes()) + source = current if source_name == "current" else legacy + if source_kind == "missing": + source.unlink() + elif source_kind == "directory": + source.unlink() + source.mkdir() + else: + source.unlink() + source.symlink_to(CURRENT) + + result = subprocess.run( + [ + "sh", + str(MIGRATOR), + str(tmp_path / f"{source_name}-{source_kind}-destination.toml"), + str(current), + str(legacy), + ], + check=False, + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + assert ( + f"gateway config migration source is not a regular file: {source}" + in result.stderr + ) + + +def test_migrator_refuses_directory_destination(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + destination.mkdir() + + result = run_migrator(destination, check=False) + + assert result.returncode == 1 + assert ( + f"refusing to replace non-regular gateway config: {destination}" + in result.stderr + ) diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 188c698962..23c87e8704 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -24,6 +24,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" # shellcheck source=tasks/scripts/gateway-pull-policy.sh source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" @@ -54,18 +56,6 @@ linux_target_triple() { esac } -# Escape a value for a TOML basic string before copying an operator-provided -# proxy path or URL into the generated local configuration. -toml_escape() { - local s=$1 - s=${s//\\/\\\\} - s=${s//\"/\\\"} - s=${s//$'\n'/\\n} - s=${s//$'\r'/\\r} - s=${s//$'\t'/\\t} - printf '%s' "${s}" -} - port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index 4d990d629f..ccf30ca104 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -21,6 +21,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" # shellcheck source=tasks/scripts/gateway-pull-policy.sh source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" @@ -92,19 +94,6 @@ ensure_podman_supervisor_image() { fi } -# Escape a value for embedding in a double-quoted TOML basic string, so -# quotes, backslashes, or control characters in an environment value cannot -# corrupt gateway.toml or inject extra configuration keys. -toml_escape() { - local s=$1 - s=${s//\\/\\\\} - s=${s//\"/\\\"} - s=${s//$'\n'/\\n} - s=${s//$'\r'/\\r} - s=${s//$'\t'/\\t} - printf '%s' "${s}" -} - port_is_in_use() { local port=$1 if command_available lsof; then diff --git a/tasks/scripts/gateway-toml.sh b/tasks/scripts/gateway-toml.sh new file mode 100644 index 0000000000..f9b1e1916b --- /dev/null +++ b/tasks/scripts/gateway-toml.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shell helpers shared by local gateway launch scripts. This file is sourced, +# rather than executed, so helpers must not change shell options or process +# environment at import time. + +# Escape a value for a TOML basic string. This supports operator-provided paths +# and URLs without allowing a quote or control character to add TOML fields. +toml_escape() { + local value=$1 + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '%s' "${value}" +} diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 6853ad7e5d..76a38d4071 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -33,6 +33,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" PORT="${OPENSHELL_SERVER_PORT:-18081}" GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-vm-dev}" STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}" @@ -40,6 +42,8 @@ SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" VM_BOOTSTRAP_IMAGE="${OPENSHELL_VM_BOOTSTRAP_IMAGE:-}" SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +# VM currently has no image-pull-policy setting in its driver configuration; unlike +# Docker, Podman, and Kubernetes launch paths it intentionally does not normalize this input. LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_DIR_DEFAULT="${ROOT}/target/debug" @@ -70,17 +74,6 @@ normalize_bool() { esac } -# Escape values that are copied from the local environment to gateway TOML. -toml_escape() { - local s=$1 - s=${s//\\/\\\\} - s=${s//\"/\\\"} - s=${s//$'\n'/\\n} - s=${s//$'\r'/\\r} - s=${s//$'\t'/\\t} - printf '%s' "${s}" -} - port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 300b6a8c7e..ccd909c4f2 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -18,7 +18,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # shellcheck source=tasks/scripts/gateway-pull-policy.sh source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" -GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" usage() { cat <<'EOF' diff --git a/tasks/scripts/test-gateway-config.sh b/tasks/scripts/test-gateway-config.sh new file mode 100755 index 0000000000..850cccc0bb --- /dev/null +++ b/tasks/scripts/test-gateway-config.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +UV="$(mise which uv)" +trap 'rm -rf "${WORK}"' EXIT + +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" + +raw=$'quote" slash\\ newline\ncarriage\rtab\t' +printf 'value = "' > "${WORK}/escaped.toml" +toml_escape "${raw}" >> "${WORK}/escaped.toml" +printf '"\n' >> "${WORK}/escaped.toml" + +printf '%s\n' 'import sys, tomllib' 'from pathlib import Path' 'assert tomllib.loads(Path(sys.argv[1]).read_text())["value"] == sys.argv[2]' > "${WORK}/check_escape.py" +"${UV}" run --no-project python "${WORK}/check_escape.py" "${WORK}/escaped.toml" "${raw}" + +mkdir -p "${WORK}/bin" +ln -s /usr/bin/true "${WORK}/bin/mise" +ln -s /usr/bin/false "${WORK}/bin/lsof" +ln -s /bin/bash "${WORK}/bin/gateway" + +printf '%s\n' 'exec() {' ' config=""' ' while [ "$#" -gt 0 ]; do' ' if [ "$1" = "--config" ]; then config="$2"; shift 2; else shift; fi' ' done' ' cp "${config}" "${CAPTURED_CONFIG}"' ' builtin exit 0' '}' > "${WORK}/fake-gateway-env" + +CAPTURED_CONFIG="${WORK}/generated.toml" +BASH_ENV="${WORK}/fake-gateway-env" CAPTURED_CONFIG="${CAPTURED_CONFIG}" PATH="${WORK}/bin:${PATH}" KUBERNETES_SERVICE_HOST=fixture OPENSHELL_GATEWAY_BIN=/bin/true OPENSHELL_GATEWAY_STATE_DIR="${WORK}/state" OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent OPENSHELL_GRPC_ENDPOINT=https://callback.example.test:9443 bash "${ROOT}/tasks/scripts/gateway.sh" + +printf '%s\n' 'import sys, tomllib' 'from pathlib import Path' 'config = tomllib.loads(Path(sys.argv[1]).read_text())' 'gateway = config["openshell"]["gateway"]' 'driver = config["openshell"]["drivers"]["kubernetes"]' 'assert config["openshell"]["version"] == 2' 'assert gateway["compute_driver"] == "kubernetes"' 'assert "compute_drivers" not in gateway' 'assert driver["image_pull_policy"] == "if_not_present"' 'assert driver["grpc_endpoint"] == "https://callback.example.test:9443"' > "${WORK}/check_generated.py" +"${UV}" run --no-project python "${WORK}/check_generated.py" "${CAPTURED_CONFIG}" +echo "gateway generated-TOML tests passed" diff --git a/tasks/scripts/test-gateway-pull-policy.sh b/tasks/scripts/test-gateway-pull-policy.sh index 267c302fd0..7859b7c1d8 100755 --- a/tasks/scripts/test-gateway-pull-policy.sh +++ b/tasks/scripts/test-gateway-pull-policy.sh @@ -45,3 +45,32 @@ for script in gateway.sh gateway-docker.sh gateway-podman.sh; do done echo "gateway pull-policy tests passed" + +# Empty and whitespace values are invalid rather than silently falling back to a default. +# Callers must supply one of the documented names. +assert_rejected_policy() { + local input=$1 + if normalize_image_pull_policy "${input}" >/dev/null 2>&1; then + printf 'unsupported policy unexpectedly succeeded: %q\n' "${input}" >&2 + exit 1 + fi +} + +assert_rejected_policy "" +assert_rejected_policy " " +assert_rejected_policy $'\t' +assert_rejected_policy " if_not_present" +assert_rejected_policy "if_not_present " + +# The VM driver does not render an image-pull-policy field, so it is intentionally +# excluded from Docker/Podman/Kubernetes normalization. Do not add it until it consumes it. +if grep -Fq 'gateway-pull-policy.sh' "${ROOT}/tasks/scripts/gateway-vm.sh"; then + echo "gateway-vm.sh unexpectedly normalizes an unused image pull policy" >&2 + exit 1 +fi +if ! grep -Fq 'intentionally does not normalize this input' "${ROOT}/tasks/scripts/gateway-vm.sh"; then + echo "gateway-vm.sh does not document its pull-policy exclusion" >&2 + exit 1 +fi + +echo "gateway pull-policy edge-case tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index a310e1e229..8825c6ed1f 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -13,6 +13,7 @@ depends = [ "test:install-sh", "test:build-env", "test:gateway-pull-policy", + "test:gateway-config", "test:packaging-assets", "test:codex-security-release-range", "test:docs-website", @@ -278,3 +279,9 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu ["e2e:openshift"] description = "Run OpenShift database-backend integration scenarios against a live cluster (requires oc CLI authenticated to an OpenShift cluster)" run = "e2e/rust/e2e-openshift.sh" + +["test:gateway-config"] +description = "Test generated local gateway TOML without starting a runtime" +run = "bash tasks/scripts/test-gateway-config.sh" +run_windows = "echo Skipping test:gateway-config: Unix gateway scripts do not apply on Windows." +hide = true From d2d496e8e70bf746f225a900f7726a0b2103675d Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 15:43:07 -0400 Subject: [PATCH 08/42] test(config): add schema v2 parity manifest Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-capability-parity.toml | 398 ++++++++++++++++++ ...ateway_schema_v2_capability_parity_test.py | 242 +++++++++++ 2 files changed, 640 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-capability-parity.toml create mode 100644 python/openshell/gateway_schema_v2_capability_parity_test.py diff --git a/e2e/configs/gateway/schema-v2-capability-parity.toml b/e2e/configs/gateway/schema-v2-capability-parity.toml new file mode 100644 index 0000000000..ab51bbf433 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-capability-parity.toml @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Inventory for the schema-v2 live capability-parity campaign. Every entry is a +# test case contract, not a result. `status = "not_run"` deliberately means no +# live assertion has been made. Keep the oracle observable and put the runtime +# prerequisites in `required_environment`; this lets later waves select a lane +# without rediscovering the migration's intended behavior. +manifest_version = 1 +baseline_ref = "origin/main" +baseline_schema_version = 1 +candidate_ref = "HEAD" +candidate_schema_version = 2 + +# Allowed lane names: deterministic, e2e-docker, e2e-podman, e2e-kubernetes, +# e2e-vm, windows-mxc, extension-driver, auth-oidc, observability, packaging. +# Allowed statuses: not_run, blocked, planned. A live pass is intentionally not +# a status in this first-wave manifest. + +[[capabilities]] +id = "configuration-source-precedence" +topics = ["configuration_producers"] +origin_main_access_paths = ["--config", "OPENSHELL_GATEWAY_CONFIG", "CLI > OPENSHELL_* > [openshell.gateway] > defaults"] +schema_v2_access_paths = ["--config", "OPENSHELL_GATEWAY_CONFIG", "CLI > OPENSHELL_* > [openshell.gateway] > defaults"] +behavioral_oracle = "A CLI or environment value wins over the corresponding file value; an unset value uses the file value." +required_environment = "none; parse and startup-config construction only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "schema-version-and-strict-layout" +topics = ["configuration_producers"] +origin_main_access_paths = ["[openshell] version = 1", "[openshell.gateway] inherited driver defaults"] +schema_v2_access_paths = ["[openshell] version = 2", "[openshell.gateway] gateway-only fields", "[openshell.drivers.] driver-owned fields"] +behavioral_oracle = "Missing, v1, future, unknown, misplaced, and non-table driver values fail before runtime construction." +required_environment = "none; TOML parser only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "gateway-identity-and-logging" +topics = ["configuration_producers", "observability"] +origin_main_access_paths = ["--name / OPENSHELL_GATEWAY_NAME", "--log-level / OPENSHELL_LOG_LEVEL", "[openshell.gateway].name", "[openshell.gateway].log_level"] +schema_v2_access_paths = ["[openshell.gateway].name", "[openshell.gateway].log_level", "same CLI and environment variables"] +behavioral_oracle = "The configured name and log filter are retained after v2 file merge and identify emitted gateway telemetry." +required_environment = "gateway process with captured logs" +test_lane = "observability" +status = "not_run" + +[[capabilities]] +id = "primary-health-and-metrics-listeners" +topics = ["listeners"] +origin_main_access_paths = ["--bind-address / OPENSHELL_BIND_ADDRESS", "--port / OPENSHELL_SERVER_PORT", "--health-port / OPENSHELL_HEALTH_PORT", "--metrics-port / OPENSHELL_METRICS_PORT", "[openshell.gateway].{bind_address,health_bind_address,metrics_bind_address}"] +schema_v2_access_paths = ["[openshell.gateway].bind_address", "[openshell.gateway].health_bind_address", "[openshell.gateway].metrics_bind_address", "same CLI and environment variables"] +behavioral_oracle = "The primary multiplexed endpoint and optional health/metrics endpoints bind their configured addresses; a zero auxiliary port disables only that auxiliary listener." +required_environment = "loopback TCP ports" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "database-url-and-persistence-backends" +topics = ["database"] +origin_main_access_paths = ["--db-url / OPENSHELL_DB_URL", "[openshell.gateway].database_url (rejected)"] +schema_v2_access_paths = ["--db-url / OPENSHELL_DB_URL only", "[openshell.gateway].database_url (rejected)"] +behavioral_oracle = "A SQLite or Postgres URL supplied outside TOML opens the store; a URL embedded in TOML is rejected without exposing credentials." +required_environment = "SQLite temporary directory; Postgres service for backend variant" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "ssh-rate-limit-and-policy-posture" +topics = ["listeners"] +origin_main_access_paths = ["[openshell.gateway].ssh_session_ttl_secs", "[openshell.gateway].grpc_rate_limit_{requests,window_seconds}", "[openshell.gateway].policy_validation_failure_mode"] +schema_v2_access_paths = ["same [openshell.gateway] fields"] +behavioral_oracle = "SSH TTL, paired rate-limit behavior, and fail_closed versus retain_last_valid policy posture survive migration with their documented validation rules." +required_environment = "gateway with a controllable clock and policy fixture" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "sandbox-service-routing" +topics = ["listeners"] +origin_main_access_paths = ["--server-san / OPENSHELL_SERVER_SAN", "--enable-loopback-service-http / OPENSHELL_ENABLE_LOOPBACK_SERVICE_HTTP", "[openshell.gateway].{server_sans,enable_loopback_service_http}"] +schema_v2_access_paths = ["[openshell.gateway].server_sans", "[openshell.gateway].enable_loopback_service_http", "same CLI and environment variables"] +behavioral_oracle = "Wildcard SAN routing and loopback-only plaintext sandbox-service HTTP remain available while gateway APIs stay unavailable on that plaintext route." +required_environment = "TLS certificate with sandbox wildcard SAN and loopback HTTP client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "gateway-listener-tls-and-sni" +topics = ["auth_tls_jwt", "listeners"] +origin_main_access_paths = ["--tls-cert / OPENSHELL_TLS_CERT", "--tls-key / OPENSHELL_TLS_KEY", "--tls-client-ca / OPENSHELL_TLS_CLIENT_CA", "[openshell.gateway.tls]"] +schema_v2_access_paths = ["[openshell.gateway.tls].{cert_path,key_path,client_ca_path,require_client_auth,external_cert_path,external_key_path,external_server_names}", "same CLI and environment variables for primary bundle"] +behavioral_oracle = "The listener presents the primary or configured SNI certificate, validates client certificates when required, and rejects incomplete TLS bundles." +required_environment = "test CA, primary and external certificates, TLS client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "plaintext-listener-mode" +topics = ["auth_tls_jwt", "listeners"] +origin_main_access_paths = ["--disable-tls / OPENSHELL_DISABLE_TLS", "[openshell.gateway].disable_tls"] +schema_v2_access_paths = ["[openshell.gateway].disable_tls", "same CLI and environment variable"] +behavioral_oracle = "An explicit plaintext listener starts without a listener TLS table and is usable behind a trusted TLS-terminating proxy." +required_environment = "loopback HTTP client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "guest-callback-tls-ownership" +topics = ["auth_tls_jwt", "docker", "podman", "vm"] +origin_main_access_paths = ["[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by local drivers", "driver-local guest_tls_* overrides"] +schema_v2_access_paths = ["[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key} injected only into selected Docker, Podman, or VM driver", "driver tables reject guest_tls_*"] +behavioral_oracle = "A TLS-enabled selected local driver receives one complete guest bundle; partial, misplaced, and plaintext-incompatible bundles fail closed." +required_environment = "test CA and client certificate bundle" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "oidc-bearer-authentication" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["--oidc-* / OPENSHELL_OIDC_*", "[openshell.gateway.oidc].{issuer,audience,jwks_ttl_secs,roles_claim,admin_role,user_role,scopes_claim}"] +schema_v2_access_paths = ["same [openshell.gateway.oidc] fields", "same CLI and environment variables"] +behavioral_oracle = "A token from the configured issuer is accepted only with the expected audience, role, and optional scope; invalid JWTs are rejected." +required_environment = "OIDC issuer with JWKS and signed test tokens" +test_lane = "auth-oidc" +status = "not_run" + +[[capabilities]] +id = "mtls-user-authentication" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["--enable-mtls-auth / OPENSHELL_ENABLE_MTLS_AUTH", "[openshell.gateway.mtls_auth].enabled"] +schema_v2_access_paths = ["[openshell.gateway.mtls_auth].enabled", "same CLI and environment variable"] +behavioral_oracle = "A verified local client certificate maps to a user principal only when mTLS user auth is enabled and no stronger auth policy replaces it." +required_environment = "local driver, test CA, client certificate" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "unsafe-unauthenticated-user-mode" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["[openshell.gateway.auth].allow_unauthenticated_users"] +schema_v2_access_paths = ["same [openshell.gateway.auth].allow_unauthenticated_users"] +behavioral_oracle = "The explicit trusted-development switch admits user requests without a credential while sandbox callbacks retain sandbox authentication." +required_environment = "isolated gateway with no public exposure" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "gateway-minted-sandbox-jwt" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["[openshell.gateway.gateway_jwt].{signing_key_path,public_key_path,kid_path,gateway_id,ttl_secs}"] +schema_v2_access_paths = ["same [openshell.gateway.gateway_jwt] fields"] +behavioral_oracle = "The gateway mints sandbox and extension tokens with the configured identity, key ID, audience, and positive or omitted TTL semantics; explicit zero is rejected." +required_environment = "Ed25519 keypair and sandbox callback fixture" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "otlp-observability" +topics = ["observability"] +origin_main_access_paths = ["[openshell.gateway.otlp].{endpoint,service_name}", "OTEL_* SDK tuning environment"] +schema_v2_access_paths = ["same [openshell.gateway.otlp] fields", "same OTEL_* tuning environment"] +behavioral_oracle = "A configured OTLP/gRPC collector receives gateway and in-tree driver traces with gateway name and selected-driver resource attributes; collector failure does not prevent serving." +required_environment = "loopback OTLP/gRPC collector" +test_lane = "observability" +status = "not_run" + +[[capabilities]] +id = "gateway-interceptor-registration" +topics = ["interceptors", "auth_tls_jwt"] +origin_main_access_paths = ["[[openshell.gateway.interceptors]] including endpoint, TLS CA, audience, ordering, failure, timeout, limits, binding policy, and bindings"] +schema_v2_access_paths = ["same [[openshell.gateway.interceptors]] table and [[openshell.gateway.interceptors.bindings]]"] +behavioral_oracle = "Startup validates Describe metadata and configured binding policy; an allowed interceptor observes or modifies only its selected non-secret unary RPC phases." +required_environment = "test interceptor gRPC service; optional private CA or Unix socket" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "supervisor-middleware-registration" +topics = ["middleware", "auth_tls_jwt"] +origin_main_access_paths = ["[[openshell.supervisor.middleware]] including endpoint, TLS CA, audience, payload limit, timeout, and insecure transport"] +schema_v2_access_paths = ["same [[openshell.supervisor.middleware]] table"] +behavioral_oracle = "Gateway startup validates middleware Describe and policy configuration, applies payload/time limits, and distributes only validated registration data to supervisors." +required_environment = "test supervisor middleware gRPC service and sandbox supervisor" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "provider-profile-sources" +topics = ["inference", "interceptors"] +origin_main_access_paths = ["[openshell.gateway].provider_profile_sources"] +schema_v2_access_paths = ["same [openshell.gateway].provider_profile_sources"] +behavioral_oracle = "Configured builtin, user, and interceptor sources form one ordered validated catalog; duplicate normalized profile IDs or invalid interceptor source selections fail closed." +required_environment = "provider records and profile-capable interceptor fixture" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "inference-control-plane-configuration" +topics = ["inference"] +origin_main_access_paths = ["OpenShell inference configuration RPCs and persisted gateway settings; not a startup TOML field"] +schema_v2_access_paths = ["unchanged OpenShell inference configuration RPCs and persisted gateway settings; not a startup TOML field"] +behavioral_oracle = "A provider and route configured through the control plane resolves the same effective inference bundle after a schema-v2 gateway starts." +required_environment = "gateway, provider fixture, and sandbox supervisor" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "credential-driver-selection-and-kek" +topics = ["credentials"] +origin_main_access_paths = ["[openshell.gateway].credential_drivers", "[openshell.gateway].default_credential_driver", "[openshell.gateway.credential_storage]"] +schema_v2_access_paths = ["same [openshell.gateway] fields"] +behavioral_oracle = "Omission selects encrypted database storage; an explicit empty or multi-driver selection fails, and KEK path/environment configuration preserves credential confidentiality." +required_environment = "temporary gateway database and KEK fixture" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "credential-driver-backend-tables" +topics = ["credentials", "external_drivers"] +origin_main_access_paths = ["[openshell.credential_drivers.kubernetes-secrets]", "[openshell.credential_drivers.vault]", "[openshell.credential_drivers.]"] +schema_v2_access_paths = ["same credential-driver tables, including in_tree or uds transport, socket_path, command, args, and startup_timeout_secs"] +behavioral_oracle = "Built-in and remote credential driver tables validate their transport contract and store/retrieve opaque credential material without inline secrets in TOML." +required_environment = "credential-driver mock over UDS; Kubernetes or Vault for backend variants" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "docker-image-and-callback-configuration" +topics = ["docker"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,host_gateway_ip} inherited by Docker", "[openshell.drivers.docker].{socket_path,default_image,image_pull_policy,sandbox_namespace,grpc_endpoint,supervisor_bin,supervisor_image,network_name,host_gateway_ip,ssh_socket_path}"] +schema_v2_access_paths = ["[openshell.drivers.docker].{socket_path,default_image,image_pull_policy,sandbox_label,grpc_endpoint,supervisor_bin,supervisor_image,network_name,host_gateway_ip,ssh_socket_path}"] +behavioral_oracle = "Docker starts a sandbox using its driver table, maps the canonical sandbox label, honors image policy, and derives or honors a callback endpoint." +required_environment = "Linux Docker daemon, bridge network, sandbox and supervisor images" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "docker-security-and-provider-configuration" +topics = ["docker", "credentials"] +origin_main_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_socket,app_armor_profile}"] +schema_v2_access_paths = ["same [openshell.drivers.docker] fields"] +behavioral_oracle = "Docker rejects unsafe proxy or PID inputs, preserves explicit bind-mount and AppArmor posture, and mounts authorized SPIFFE/proxy material only into the supervisor." +required_environment = "Linux Docker daemon, proxy fixture, optional SPIFFE Unix socket and AppArmor support" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "podman-image-and-callback-configuration" +topics = ["podman"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,host_gateway_ip} inherited by Podman", "[openshell.drivers.podman].{socket_path,default_image,image_pull_policy,grpc_endpoint,gateway_port,network_name,host_gateway_ip,stop_timeout_secs,supervisor_image}"] +schema_v2_access_paths = ["same [openshell.drivers.podman] fields; gateway_port is runtime-derived"] +behavioral_oracle = "Podman starts a sandbox with its selected socket, image and policy, derives the callback route, and applies stop timeout without v1 gateway inheritance." +required_environment = "Podman service socket, user bridge network, sandbox and supervisor images" +test_lane = "e2e-podman" +status = "not_run" + +[[capabilities]] +id = "podman-runtime-security-and-health" +topics = ["podman", "credentials"] +origin_main_access_paths = ["[openshell.drivers.podman].{sandbox_ssh_socket_path,sandbox_pids_limit,enable_bind_mounts,provider_spiffe_workload_api_socket,app_armor_profile,health_check_interval_secs,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,proxy_ca_bundle,userns,uidmap,gidmap}"] +schema_v2_access_paths = ["[openshell.drivers.podman].{ssh_socket_path,sandbox_pids_limit,enable_bind_mounts,provider_spiffe_workload_api_socket,app_armor_profile,health_check_interval_secs,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,proxy_ca_bundle,userns,uidmap,gidmap}"] +behavioral_oracle = "Podman uses the renamed SSH path and validates PID, health, user namespace mappings, AppArmor, proxy, CA, and SPIFFE contracts before a sandbox can run." +required_environment = "Podman service socket, optional proxy/CA/SPIFFE fixture, rootless and rootful variants" +test_lane = "e2e-podman" +status = "not_run" + +[[capabilities]] +id = "kubernetes-core-placement-and-images" +topics = ["kubernetes"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,client_tls_secret_name,service_account_name,host_gateway_ip,enable_user_namespaces,sa_token_ttl_secs} inherited by Kubernetes", "[openshell.drivers.kubernetes].{namespace,default_image,image_pull_policy,image_pull_secrets,service_account_name,supervisor_image,supervisor_image_pull_policy,grpc_endpoint,ssh_socket_path,client_tls_secret_name,host_gateway_ip,enable_user_namespaces,sa_token_ttl_secs}"] +schema_v2_access_paths = ["same fields exclusively in [openshell.drivers.kubernetes]"] +behavioral_oracle = "The Kubernetes driver creates a sandbox Pod with driver-owned namespace, service account, image, pull policy, callback endpoint, SSH socket, TLS Secret, and token TTL." +required_environment = "Kubernetes cluster, namespace, service account, image pull secret, and client TLS Secret" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-workspace-isolation" +topics = ["kubernetes"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{workspace_mode,gateway_id,operator_namespace_label,operator_namespace_file,workspace_default_storage_size,workspace_storage_class,default_runtime_class_name}"] +schema_v2_access_paths = ["same [openshell.drivers.kubernetes] fields"] +behavioral_oracle = "Shared, managed, and operator workspace modes select or create the expected namespace and workspace storage with configured discovery, class, and runtime class." +required_environment = "Kubernetes cluster with namespaces, PVC provisioning, and optional RuntimeClass" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-supervisor-topology" +topics = ["kubernetes", "middleware"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{supervisor_sideload_method,topology}", "[openshell.drivers.kubernetes.sidecar].{proxy_uid,process_binary_aware_network_policy}", "[openshell.drivers.kubernetes.managed_ssh_ingress].{enabled,gateway_namespace,gateway_pod_selector}"] +schema_v2_access_paths = ["same Kubernetes driver subtables and fields"] +behavioral_oracle = "The rendered sandbox Pod matches the selected combined or sidecar supervisor topology, sideload method, ingress selector, and sidecar security posture." +required_environment = "Kubernetes cluster supporting selected image-volume or init-container method" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-egress-spiffe-and-security" +topics = ["kubernetes", "credentials"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{https_proxy,no_proxy,proxy_auth_secret_name,proxy_auth_secret_key,proxy_auth_allow_insecure,proxy_connect_by_hostname,app_armor_profile,provider_spiffe_workload_api_socket_path,sandbox_uid,sandbox_gid}"] +schema_v2_access_paths = ["same [openshell.drivers.kubernetes] fields"] +behavioral_oracle = "Kubernetes validates proxy/Secret and sidecar-topology relationships, projects SPIFFE only from an allowed path, and applies valid non-root identity and AppArmor settings." +required_environment = "Kubernetes cluster, proxy credential Secret, optional SPIFFE socket, AppArmor-capable node" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "vm-launch-and-resource-configuration" +topics = ["vm"] +origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{openshell_endpoint,state_dir,launcher_bin,default_image,bootstrap_image,log_level,krun_log_level,vcpus,mem_mib,overlay_disk_mib,gpu_enabled,gpu_mem_mib,gpu_vcpus}"] +schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,launcher_bin,default_image,bootstrap_image,log_level,krun_log_level,vcpus,mem_mib,overlay_disk_mib,gpu_enabled,gpu_mem_mib,gpu_vcpus}"] +behavioral_oracle = "The VM driver receives its renamed callback endpoint and launches a guest with selected state, images, resources, and optional GPU configuration." +required_environment = "Linux libkrun/KVM host, VM driver binary, OCI image, optional GPU/VFIO hardware" +test_lane = "e2e-vm" +status = "not_run" + +[[capabilities]] +id = "vm-guest-security-and-spiffe" +topics = ["vm", "credentials"] +origin_main_access_paths = ["[openshell.drivers.vm].{sandbox_uid,sandbox_gid}"] +schema_v2_access_paths = ["[openshell.drivers.vm].{sandbox_uid,sandbox_gid,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_tcp_endpoint,provider_spiffe_allow_guest_tcp}", "gateway-owned guest_tls_* bundle"] +behavioral_oracle = "VM validates non-root guest ownership, callback TLS, proxy safety, and requires an explicit opt-in before exposing a guest-reachable SPIFFE TCP endpoint." +required_environment = "Linux libkrun/KVM host, TLS and optional proxy/SPIFFE TCP fixture" +test_lane = "e2e-vm" +status = "not_run" + +[[capabilities]] +id = "mxc-windows-driver-configuration" +topics = ["mxc"] +origin_main_access_paths = ["[openshell.drivers.mxc].{wxc_exec_path,backend,pc_least_privilege,pc_capabilities,default_configuration_id,debug}"] +schema_v2_access_paths = ["same [openshell.drivers.mxc] fields"] +behavioral_oracle = "The Windows gateway selects MXC and passes the configured wxc-exec path, backend, AppContainer capabilities, isolation configuration, and debug flag to MXC." +required_environment = "Windows host with MXC/wxc-exec; mock fixture for deterministic smoke variant" +test_lane = "windows-mxc" +status = "not_run" + +[[capabilities]] +id = "external-compute-driver-socket" +topics = ["external_drivers"] +origin_main_access_paths = ["--drivers / --driver / OPENSHELL_DRIVERS plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.] remote socket table"] +schema_v2_access_paths = ["--compute-driver / OPENSHELL_COMPUTE_DRIVER plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.].socket_path"] +behavioral_oracle = "A selected non-reserved external driver connects through its configured Unix socket; legacy plural selector flags and environment variables are rejected." +required_environment = "external compute-driver gRPC fixture over Unix socket" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "helm-configuration-producer" +topics = ["configuration_producers", "kubernetes"] +origin_main_access_paths = ["Helm rendered schema-v1 gateway TOML and environment-backed overrides"] +schema_v2_access_paths = ["deploy/helm/openshell/templates/gateway-config.yaml renders [openshell] version = 2 and Kubernetes driver table", "Secret-backed OPENSHELL_DB_URL and certificate inputs"] +behavioral_oracle = "helm template renders schema-v2 TOML, derives the Kubernetes callback endpoint, and keeps database and secret material outside the ConfigMap." +required_environment = "Helm and chart test plugin; Kubernetes cluster only for live install variant" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "local-launch-script-producers" +topics = ["configuration_producers", "docker", "podman", "vm", "kubernetes"] +origin_main_access_paths = ["tasks/scripts/gateway{,-docker,-podman,-vm}.sh generated schema-v1 TOML"] +schema_v2_access_paths = ["tasks/scripts/gateway{,-docker,-podman,-vm}.sh generated schema-v2 TOML with per-driver tables"] +behavioral_oracle = "Each local launch script emits parseable v2 TOML with a scalar driver and its canonical driver-owned values before it executes the gateway binary." +required_environment = "shell, uv, and fake gateway executable; no runtime daemon required" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "e2e-fixture-producers" +topics = ["configuration_producers", "docker", "podman"] +origin_main_access_paths = ["e2e/configs/gateway/docker.toml and podman.toml schema-v1 fixtures"] +schema_v2_access_paths = ["e2e/configs/gateway/docker.toml and podman.toml schema-v2 fixtures"] +behavioral_oracle = "Docker and Podman E2E fixtures parse as v2, select one scalar driver, and contain canonical renamed policy and callback fields." +required_environment = "none; TOML parser only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "rpm-schema-upgrade" +topics = ["packaging_upgrades", "configuration_producers"] +origin_main_access_paths = ["deploy/rpm/gateway.toml.default version 1", "systemd user config seeded from package default"] +schema_v2_access_paths = ["deploy/rpm/gateway.toml.default version 2", "deploy/rpm/migrate-gateway-config.sh exact-v1 replacement"] +behavioral_oracle = "RPM first start seeds the v2 default and upgrade replaces only an exact package-generated v1 file, preserving edited files, modes, and symlink safety." +required_environment = "temporary filesystem and shell; RPM install test host for package variant" +test_lane = "packaging" +status = "not_run" + +[[capabilities]] +id = "homebrew-debian-and-snap-upgrades" +topics = ["packaging_upgrades"] +origin_main_access_paths = ["Homebrew prefix config and migration helper", "Debian XDG user-service config", "Snap $SNAP_COMMON/gateway.toml"] +schema_v2_access_paths = ["package-managed schema-v2 defaults and documented exact-v1/manual migration paths"] +behavioral_oracle = "Each package resolves its documented config location, starts from v2 defaults, and never overwrites operator-edited legacy configuration during upgrade." +required_environment = "macOS Homebrew, Debian/Ubuntu user-service, and Snap test environments" +test_lane = "packaging" +status = "not_run" diff --git a/python/openshell/gateway_schema_v2_capability_parity_test.py b/python/openshell/gateway_schema_v2_capability_parity_test.py new file mode 100644 index 0000000000..4ef12fb58b --- /dev/null +++ b/python/openshell/gateway_schema_v2_capability_parity_test.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate the schema-v2 live capability-parity manifest without a runtime.""" + +from __future__ import annotations + +import tomllib +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" + +REQUIRED_MANIFEST_FIELDS = { + "manifest_version", + "baseline_ref", + "baseline_schema_version", + "candidate_ref", + "candidate_schema_version", + "capabilities", +} +REQUIRED_CAPABILITY_FIELDS = { + "id", + "topics", + "origin_main_access_paths", + "schema_v2_access_paths", + "behavioral_oracle", + "required_environment", + "test_lane", + "status", +} +REQUIRED_TOPICS = { + "auth_tls_jwt", + "configuration_producers", + "credentials", + "database", + "docker", + "external_drivers", + "inference", + "interceptors", + "kubernetes", + "listeners", + "middleware", + "mxc", + "observability", + "packaging_upgrades", + "podman", + "vm", +} +REQUIRED_CAPABILITY_IDS = { + "configuration-source-precedence", + "schema-version-and-strict-layout", + "gateway-identity-and-logging", + "primary-health-and-metrics-listeners", + "database-url-and-persistence-backends", + "ssh-rate-limit-and-policy-posture", + "sandbox-service-routing", + "gateway-listener-tls-and-sni", + "plaintext-listener-mode", + "guest-callback-tls-ownership", + "oidc-bearer-authentication", + "mtls-user-authentication", + "unsafe-unauthenticated-user-mode", + "gateway-minted-sandbox-jwt", + "otlp-observability", + "gateway-interceptor-registration", + "supervisor-middleware-registration", + "provider-profile-sources", + "inference-control-plane-configuration", + "credential-driver-selection-and-kek", + "credential-driver-backend-tables", + "docker-image-and-callback-configuration", + "docker-security-and-provider-configuration", + "podman-image-and-callback-configuration", + "podman-runtime-security-and-health", + "kubernetes-core-placement-and-images", + "kubernetes-workspace-isolation", + "kubernetes-supervisor-topology", + "kubernetes-egress-spiffe-and-security", + "vm-launch-and-resource-configuration", + "vm-guest-security-and-spiffe", + "mxc-windows-driver-configuration", + "external-compute-driver-socket", + "helm-configuration-producer", + "local-launch-script-producers", + "e2e-fixture-producers", + "rpm-schema-upgrade", + "homebrew-debian-and-snap-upgrades", +} +ALLOWED_LANES = { + "deterministic", + "e2e-docker", + "e2e-podman", + "e2e-kubernetes", + "e2e-vm", + "windows-mxc", + "extension-driver", + "auth-oidc", + "observability", + "packaging", +} +# This inventory plans execution. Live results belong in the execution record, +# not in this baseline manifest, so a PASS cannot be accidentally implied. +ALLOWED_STATUSES = {"not_run", "blocked", "planned"} + + +def load_manifest() -> dict[str, Any]: + with MANIFEST_PATH.open("rb") as manifest_file: + return tomllib.load(manifest_file) + + +def require_nonempty_string(value: Any, field: str, entry_id: str) -> None: + assert isinstance(value, str) and value.strip(), ( + f"{entry_id}: {field} is required" + ) + + +def require_string_list(value: Any, field: str, entry_id: str) -> None: + assert isinstance(value, list) and value, f"{entry_id}: {field} must be non-empty" + assert all(isinstance(item, str) and item.strip() for item in value), ( + f"{entry_id}: {field} must contain only non-empty strings" + ) + assert len(value) == len(set(value)), f"{entry_id}: {field} contains duplicates" + + +def validate_manifest(manifest: dict[str, Any]) -> None: + assert set(manifest) == REQUIRED_MANIFEST_FIELDS, ( + "unexpected or missing manifest metadata" + ) + assert manifest["manifest_version"] == 1 + assert manifest["baseline_ref"] == "origin/main" + assert manifest["baseline_schema_version"] == 1 + assert manifest["candidate_ref"] == "HEAD" + assert manifest["candidate_schema_version"] == 2 + + capabilities = manifest["capabilities"] + assert isinstance(capabilities, list) and capabilities, ( + "capabilities must be non-empty" + ) + ids: list[str] = [] + topics: set[str] = set() + for capability in capabilities: + assert isinstance(capability, dict), "each capability must be a TOML table" + assert set(capability) == REQUIRED_CAPABILITY_FIELDS, ( + "capability has unexpected or missing metadata" + ) + entry_id = capability["id"] + require_nonempty_string(entry_id, "id", "capability") + ids.append(entry_id) + require_string_list(capability["topics"], "topics", entry_id) + topics.update(capability["topics"]) + require_string_list( + capability["origin_main_access_paths"], + "origin_main_access_paths", + entry_id, + ) + require_string_list( + capability["schema_v2_access_paths"], + "schema_v2_access_paths", + entry_id, + ) + require_nonempty_string( + capability["behavioral_oracle"], "behavioral_oracle", entry_id + ) + require_nonempty_string( + capability["required_environment"], "required_environment", entry_id + ) + assert capability["test_lane"] in ALLOWED_LANES, ( + f"{entry_id}: unknown test lane {capability['test_lane']!r}" + ) + assert capability["status"] in ALLOWED_STATUSES, ( + f"{entry_id}: live PASS results are not valid in this planning manifest" + ) + + assert len(ids) == len(set(ids)), "capability IDs must be unique" + assert set(ids) == REQUIRED_CAPABILITY_IDS, ( + "capability inventory is incomplete or stale" + ) + assert topics == REQUIRED_TOPICS, ( + "topic inventory is incomplete or contains an unknown topic" + ) + + +def test_schema_v2_capability_parity_manifest_is_well_formed() -> None: + validate_manifest(load_manifest()) + + +@pytest.mark.parametrize("field", sorted(REQUIRED_MANIFEST_FIELDS - {"capabilities"})) +def test_manifest_rejects_missing_header_metadata(field: str) -> None: + manifest = deepcopy(load_manifest()) + del manifest[field] + + with pytest.raises(AssertionError, match="missing manifest metadata"): + validate_manifest(manifest) + + +@pytest.mark.parametrize("field", sorted(REQUIRED_CAPABILITY_FIELDS - {"id"})) +def test_manifest_rejects_missing_capability_metadata(field: str) -> None: + manifest = deepcopy(load_manifest()) + del manifest["capabilities"][0][field] + + with pytest.raises(AssertionError, match="metadata|required|must be"): + validate_manifest(manifest) + + +@pytest.mark.parametrize( + ("field", "value", "match"), + [ + ("topics", [], "must be non-empty"), + ("behavioral_oracle", "", "is required"), + ("required_environment", "", "is required"), + ("test_lane", "not-a-lane", "unknown test lane"), + ], +) +def test_manifest_rejects_malformed_capability_metadata( + field: str, value: Any, match: str +) -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][0][field] = value + + with pytest.raises(AssertionError, match=match): + validate_manifest(manifest) + + +def test_manifest_rejects_duplicate_capability_id() -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][1]["id"] = manifest["capabilities"][0]["id"] + + with pytest.raises(AssertionError, match="unique|incomplete"): + validate_manifest(manifest) + + +def test_manifest_does_not_claim_live_pass_results() -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][0]["status"] = "pass" + + with pytest.raises(AssertionError, match="live PASS"): + validate_manifest(manifest) From dd636f7ce86b1c9459535234eee5b9f9111e1fa0 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 15:53:52 -0400 Subject: [PATCH 09/42] fix(config): correct parity manifest inventory Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-capability-parity.toml | 18 +++--- ...ateway_schema_v2_capability_parity_test.py | 61 +++++++++++++++++-- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/e2e/configs/gateway/schema-v2-capability-parity.toml b/e2e/configs/gateway/schema-v2-capability-parity.toml index ab51bbf433..56afce90a0 100644 --- a/e2e/configs/gateway/schema-v2-capability-parity.toml +++ b/e2e/configs/gateway/schema-v2-capability-parity.toml @@ -8,8 +8,10 @@ # without rediscovering the migration's intended behavior. manifest_version = 1 baseline_ref = "origin/main" +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" baseline_schema_version = 1 candidate_ref = "HEAD" +candidate_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" candidate_schema_version = 2 # Allowed lane names: deterministic, e2e-docker, e2e-podman, e2e-kubernetes, @@ -32,7 +34,7 @@ id = "schema-version-and-strict-layout" topics = ["configuration_producers"] origin_main_access_paths = ["[openshell] version = 1", "[openshell.gateway] inherited driver defaults"] schema_v2_access_paths = ["[openshell] version = 2", "[openshell.gateway] gateway-only fields", "[openshell.drivers.] driver-owned fields"] -behavioral_oracle = "Missing, v1, future, unknown, misplaced, and non-table driver values fail before runtime construction." +behavioral_oracle = "Missing, v1, future, unknown gateway, misplaced selected-driver, unknown selected-driver, and non-table driver values fail before runtime construction; unselected driver tables are validated when selected." required_environment = "none; TOML parser only" test_lane = "deterministic" status = "not_run" @@ -240,9 +242,9 @@ status = "not_run" [[capabilities]] id = "docker-security-and-provider-configuration" topics = ["docker", "credentials"] -origin_main_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_socket,app_armor_profile}"] -schema_v2_access_paths = ["same [openshell.drivers.docker] fields"] -behavioral_oracle = "Docker rejects unsafe proxy or PID inputs, preserves explicit bind-mount and AppArmor posture, and mounts authorized SPIFFE/proxy material only into the supervisor." +origin_main_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts}", "no origin/main Docker equivalent for proxy, SPIFFE, or AppArmor configuration"] +schema_v2_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_socket,app_armor_profile}"] +behavioral_oracle = "Docker preserves PID and bind-mount behavior while schema v2 adds fail-closed proxy, SPIFFE, and AppArmor configuration whose material is mounted only into the supervisor." required_environment = "Linux Docker daemon, proxy fixture, optional SPIFFE Unix socket and AppArmor support" test_lane = "e2e-docker" status = "not_run" @@ -310,10 +312,10 @@ status = "not_run" [[capabilities]] id = "vm-launch-and-resource-configuration" topics = ["vm"] -origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{openshell_endpoint,state_dir,launcher_bin,default_image,bootstrap_image,log_level,krun_log_level,vcpus,mem_mib,overlay_disk_mib,gpu_enabled,gpu_mem_mib,gpu_vcpus}"] -schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,launcher_bin,default_image,bootstrap_image,log_level,krun_log_level,vcpus,mem_mib,overlay_disk_mib,gpu_enabled,gpu_mem_mib,gpu_vcpus}"] -behavioral_oracle = "The VM driver receives its renamed callback endpoint and launches a guest with selected state, images, resources, and optional GPU configuration." -required_environment = "Linux libkrun/KVM host, VM driver binary, OCI image, optional GPU/VFIO hardware" +origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{openshell_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}"] +schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key}"] +behavioral_oracle = "The gateway finds and launches the VM driver from driver_dir, forwards the renamed callback endpoint, and launches a guest with the selected state, images, resources, and identity." +required_environment = "Linux libkrun/KVM host, VM driver binary, and OCI image" test_lane = "e2e-vm" status = "not_run" diff --git a/python/openshell/gateway_schema_v2_capability_parity_test.py b/python/openshell/gateway_schema_v2_capability_parity_test.py index 4ef12fb58b..9d3ff80b65 100644 --- a/python/openshell/gateway_schema_v2_capability_parity_test.py +++ b/python/openshell/gateway_schema_v2_capability_parity_test.py @@ -5,6 +5,7 @@ from __future__ import annotations +import re import tomllib from copy import deepcopy from pathlib import Path @@ -18,8 +19,10 @@ REQUIRED_MANIFEST_FIELDS = { "manifest_version", "baseline_ref", + "baseline_commit", "baseline_schema_version", "candidate_ref", + "candidate_commit", "candidate_schema_version", "capabilities", } @@ -114,9 +117,7 @@ def load_manifest() -> dict[str, Any]: def require_nonempty_string(value: Any, field: str, entry_id: str) -> None: - assert isinstance(value, str) and value.strip(), ( - f"{entry_id}: {field} is required" - ) + assert isinstance(value, str) and value.strip(), f"{entry_id}: {field} is required" def require_string_list(value: Any, field: str, entry_id: str) -> None: @@ -133,8 +134,10 @@ def validate_manifest(manifest: dict[str, Any]) -> None: ) assert manifest["manifest_version"] == 1 assert manifest["baseline_ref"] == "origin/main" + assert manifest["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" assert manifest["baseline_schema_version"] == 1 assert manifest["candidate_ref"] == "HEAD" + assert manifest["candidate_commit"] == "8c868e430e9cd3284d7e274628419ab484ebcee0" assert manifest["candidate_schema_version"] == 2 capabilities = manifest["capabilities"] @@ -185,10 +188,58 @@ def validate_manifest(manifest: dict[str, Any]) -> None: ) +def capability_by_id(manifest: dict[str, Any], capability_id: str) -> dict[str, Any]: + return next( + capability + for capability in manifest["capabilities"] + if capability["id"] == capability_id + ) + + def test_schema_v2_capability_parity_manifest_is_well_formed() -> None: validate_manifest(load_manifest()) +def test_frozen_comparison_commits_are_full_git_object_ids() -> None: + manifest = load_manifest() + + for field in ("baseline_commit", "candidate_commit"): + commit = manifest[field] + assert len(commit) == 40 + assert all(character in "0123456789abcdef" for character in commit) + + +def test_vm_gateway_inventory_excludes_standalone_driver_only_fields() -> None: + capability = capability_by_id( + load_manifest(), "vm-launch-and-resource-configuration" + ) + gateway_paths = " ".join( + capability["origin_main_access_paths"] + capability["schema_v2_access_paths"] + ) + + gateway_fields = set(re.findall(r"[a-z][a-z0-9_]*", gateway_paths)) + for standalone_field in ("launcher_bin", "log_level", "gpu_enabled", "gpu_mem_mib"): + assert standalone_field not in gateway_fields + assert "driver_dir" in gateway_fields + + +def test_new_docker_capabilities_are_not_attributed_to_origin_main() -> None: + capability = capability_by_id( + load_manifest(), "docker-security-and-provider-configuration" + ) + origin_paths = " ".join(capability["origin_main_access_paths"]) + candidate_paths = " ".join(capability["schema_v2_access_paths"]) + + assert "no origin/main Docker equivalent" in origin_paths + for added_field in ( + "https_proxy", + "provider_spiffe_workload_api_socket", + "app_armor_profile", + ): + assert added_field not in origin_paths + assert added_field in candidate_paths + + @pytest.mark.parametrize("field", sorted(REQUIRED_MANIFEST_FIELDS - {"capabilities"})) def test_manifest_rejects_missing_header_metadata(field: str) -> None: manifest = deepcopy(load_manifest()) @@ -203,7 +254,7 @@ def test_manifest_rejects_missing_capability_metadata(field: str) -> None: manifest = deepcopy(load_manifest()) del manifest["capabilities"][0][field] - with pytest.raises(AssertionError, match="metadata|required|must be"): + with pytest.raises(AssertionError, match=r"metadata|required|must be"): validate_manifest(manifest) @@ -230,7 +281,7 @@ def test_manifest_rejects_duplicate_capability_id() -> None: manifest = deepcopy(load_manifest()) manifest["capabilities"][1]["id"] = manifest["capabilities"][0]["id"] - with pytest.raises(AssertionError, match="unique|incomplete"): + with pytest.raises(AssertionError, match=r"unique|incomplete"): validate_manifest(manifest) From de067b232369072683fb402d7b568a78ae8e54c0 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 15:56:58 -0400 Subject: [PATCH 10/42] docs(config): record schema v2 intentional changes Signed-off-by: Jesse Jaggars --- .../schema-v2-intentional-changes.toml | 160 ++++++++++++++++++ ...eway_schema_v2_intentional_changes_test.py | 150 ++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-intentional-changes.toml create mode 100644 python/openshell/gateway_schema_v2_intentional_changes_test.py diff --git a/e2e/configs/gateway/schema-v2-intentional-changes.toml b/e2e/configs/gateway/schema-v2-intentional-changes.toml new file mode 100644 index 0000000000..b6d92f29ce --- /dev/null +++ b/e2e/configs/gateway/schema-v2-intentional-changes.toml @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Explicit compatibility exceptions for the schema-v2 parity campaign. These +# entries document behavior that is not expected to match schema v1 byte for +# byte. They do not waive the requirement that the replacement behavior work. +ledger_version = 1 +issue = 2792 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +[[intentional_changes]] +id = "schema-version-cutover" +category = "schema_cutover" +origin_main_contract = "Gateway configuration uses schema version 1; omitted versions were accepted in some paths." +schema_v2_contract = "Every loaded gateway configuration must declare [openshell] version = 2; missing, v1, and future versions fail." +migration = "Set version = 2 and apply every field relocation, rename, and type migration in this ledger before restart." +rationale = "An explicit version boundary prevents a partially migrated security-sensitive configuration from being interpreted with mixed ownership rules." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "rpm-schema-upgrade", "homebrew-debian-and-snap-upgrades"] + +[[intentional_changes]] +id = "singular-compute-driver-selector" +category = "cardinality" +origin_main_contract = "Configuration spells the selector compute_drivers as a list even though runtime selection rejects more than one configured driver." +schema_v2_contract = "Configuration uses one optional scalar compute_driver; omission retains built-in auto-detection." +migration = "Replace compute_drivers = [\"name\"] with compute_driver = \"name\"." +rationale = "The schema now represents the existing one-driver runtime invariant instead of implying unsupported multi-driver operation." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "external-compute-driver-socket"] + +[[intentional_changes]] +id = "driver-table-exclusive-ownership" +category = "ownership" +origin_main_contract = "Selected driver tables inherit an allowlisted set of values from [openshell.gateway]." +schema_v2_contract = "Driver-specific values are read only from [openshell.drivers.]; gateway scope contains only gateway-owned values." +migration = "Move every driver option to the selected driver table instead of relying on gateway inheritance." +rationale = "Only one compute driver is active, so inheritance adds ambiguity and can map one gateway key to different backend meanings." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "docker-image-and-callback-configuration", "podman-image-and-callback-configuration", "kubernetes-core-placement-and-images", "vm-launch-and-resource-configuration"] + +[[intentional_changes]] +id = "kubernetes-fields-relocated" +category = "relocation" +origin_main_contract = "Kubernetes namespace, image, client TLS Secret, ServiceAccount, host gateway, user namespace, and token TTL values may be inherited from gateway scope." +schema_v2_contract = "Those values exist exclusively under [openshell.drivers.kubernetes]." +migration = "Move namespace, default_image, supervisor_image, client_tls_secret_name, service_account_name, host_gateway_ip, enable_user_namespaces, and sa_token_ttl_secs into the Kubernetes driver table." +rationale = "Kubernetes-only deployment controls are not gateway-wide concerns." +parity_disposition = "intentional_change" +validation_capability_ids = ["kubernetes-core-placement-and-images"] + +[[intentional_changes]] +id = "docker-sandbox-label-rename" +category = "rename" +origin_main_contract = "Docker identifies OpenShell containers with sandbox_namespace." +schema_v2_contract = "Docker uses sandbox_label." +migration = "Rename [openshell.drivers.docker].sandbox_namespace to sandbox_label." +rationale = "The value is a Docker container label, not a Kubernetes-style namespace." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-image-and-callback-configuration"] + +[[intentional_changes]] +id = "podman-ssh-socket-rename" +category = "rename" +origin_main_contract = "Podman uses sandbox_ssh_socket_path in gateway TOML." +schema_v2_contract = "Podman uses ssh_socket_path in gateway TOML." +migration = "Rename [openshell.drivers.podman].sandbox_ssh_socket_path to ssh_socket_path." +rationale = "The canonical name now matches the Docker and Kubernetes driver tables." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "vm-grpc-endpoint-rename" +category = "rename" +origin_main_contract = "The VM driver callback override is named openshell_endpoint." +schema_v2_contract = "The VM driver callback override is named grpc_endpoint." +migration = "Rename [openshell.drivers.vm].openshell_endpoint and the standalone flag to grpc_endpoint and --grpc-endpoint." +rationale = "All compute drivers use the same name for the gateway gRPC callback endpoint." +parity_disposition = "intentional_change" +validation_capability_ids = ["vm-launch-and-resource-configuration", "external-compute-driver-socket"] + +[[intentional_changes]] +id = "canonical-image-pull-policy" +category = "type_normalization" +origin_main_contract = "Drivers accept different pull-policy types and spellings, including Podman missing and Kubernetes-style capitalization." +schema_v2_contract = "Gateway TOML uses always, if_not_present, never, and Podman-only newer through the shared typed policy." +migration = "Translate legacy or runtime-native spellings to canonical lowercase values." +rationale = "A shared validated type rejects typos before contacting a container runtime while preserving Podman's additional newer behavior." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-image-and-callback-configuration", "podman-image-and-callback-configuration", "kubernetes-core-placement-and-images", "local-launch-script-producers"] + +[[intentional_changes]] +id = "gateway-jwt-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "gateway_jwt.ttl_secs = 0 means that sandbox tokens do not expire." +schema_v2_contract = "Omitting gateway_jwt.ttl_secs means non-expiring sandbox tokens; an explicit zero is invalid." +migration = "Remove ttl_secs when non-expiring local tokens are intended, or set a positive duration." +rationale = "Omission distinguishes a deliberate absence of expiry from an invalid duration." +parity_disposition = "intentional_change" +validation_capability_ids = ["gateway-minted-sandbox-jwt"] + +[[intentional_changes]] +id = "sandbox-pid-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "Docker and Podman sandbox_pids_limit use zero as a backend-default sentinel." +schema_v2_contract = "Omitting sandbox_pids_limit uses the OpenShell default of 2048; configured values must be positive." +migration = "Remove a zero sandbox_pids_limit or replace it with a positive explicit limit." +rationale = "A typed optional non-zero limit removes ambiguous zero handling and gives both local container drivers one default." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-security-and-provider-configuration", "podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "podman-health-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "Podman health_check_interval_secs = 0 disables health checks." +schema_v2_contract = "Omitting health_check_interval_secs disables gateway-managed Podman health checks; configured values must be positive." +migration = "Remove a zero health_check_interval_secs or set a positive interval." +rationale = "Optional non-zero duration expresses enabled and disabled states without a sentinel." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "guest-tls-centralized" +category = "ownership" +origin_main_contract = "Local driver tables may override guest_tls_ca, guest_tls_cert, and guest_tls_key inherited from gateway scope." +schema_v2_contract = "One complete guest TLS bundle is gateway-owned and injected into the selected Docker, Podman, or VM driver; driver-local fields are rejected." +migration = "Keep guest_tls_ca, guest_tls_cert, and guest_tls_key under [openshell.gateway] only." +rationale = "A single active local driver needs one callback client identity, and centralized validation prevents partial or conflicting bundles." +parity_disposition = "intentional_change" +validation_capability_ids = ["guest-callback-tls-ownership"] + +[[intentional_changes]] +id = "middleware-payload-name-normalized" +category = "rename_with_alias" +origin_main_contract = "Supervisor middleware uses max_body_bytes while gateway interceptors use max_response_bytes." +schema_v2_contract = "Supervisor middleware uses max_payload_bytes and continues accepting max_body_bytes as a compatibility alias." +migration = "Prefer max_payload_bytes in new configuration; existing max_body_bytes remains accepted." +rationale = "Payload describes middleware data more accurately without forcing an immediate compatibility break." +parity_disposition = "intentional_change" +validation_capability_ids = ["supervisor-middleware-registration"] + +[[intentional_changes]] +id = "vm-sandbox-identity-selection" +category = "default_behavior" +origin_main_contract = "New VM sandboxes default to UID and GID 10001 when no explicit identity is configured." +schema_v2_contract = "New VM root filesystems use the image sandbox account when present and otherwise UID/GID 1000; persisted overlays retain recorded or recoverable identity and ambiguous state fails closed." +migration = "Set sandbox_uid and sandbox_gid for a fixed identity, or retain the generated owner marker with persistent VM state." +rationale = "Image-derived identity preserves filesystem ownership, while explicit state evidence avoids silently reassigning legacy overlays." +parity_disposition = "intentional_change" +validation_capability_ids = ["vm-launch-and-resource-configuration", "vm-guest-security-and-spiffe"] + +[[intentional_changes]] +id = "package-default-only-auto-migration" +category = "migration_policy" +origin_main_contract = "Package-managed and operator-edited schema-v1 configuration may exist at upgrade time." +schema_v2_contract = "RPM and Homebrew automatically replace only recognized byte-identical package defaults; operator-edited files require explicit migration." +migration = "Automatically migrate recognized defaults and preserve every edited file for the operator to convert using the schema-v2 guide." +rationale = "An upgrade must not overwrite operator intent merely because the old schema no longer parses." +parity_disposition = "intentional_change" +validation_capability_ids = ["rpm-schema-upgrade", "homebrew-debian-and-snap-upgrades"] diff --git a/python/openshell/gateway_schema_v2_intentional_changes_test.py b/python/openshell/gateway_schema_v2_intentional_changes_test.py new file mode 100644 index 0000000000..e604aedc1c --- /dev/null +++ b/python/openshell/gateway_schema_v2_intentional_changes_test.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate the explicit schema-v2 intentional-change ledger.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +LEDGER_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-intentional-changes.toml" +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" + +REQUIRED_HEADER_FIELDS = { + "ledger_version", + "issue", + "baseline_commit", + "candidate_start_commit", + "intentional_changes", +} +REQUIRED_CHANGE_FIELDS = { + "id", + "category", + "origin_main_contract", + "schema_v2_contract", + "migration", + "rationale", + "parity_disposition", + "validation_capability_ids", +} +REQUIRED_CHANGE_IDS = { + "canonical-image-pull-policy", + "docker-sandbox-label-rename", + "driver-table-exclusive-ownership", + "gateway-jwt-zero-sentinel-removed", + "guest-tls-centralized", + "kubernetes-fields-relocated", + "middleware-payload-name-normalized", + "package-default-only-auto-migration", + "podman-health-zero-sentinel-removed", + "podman-ssh-socket-rename", + "sandbox-pid-zero-sentinel-removed", + "schema-version-cutover", + "singular-compute-driver-selector", + "vm-grpc-endpoint-rename", + "vm-sandbox-identity-selection", +} +ALLOWED_CATEGORIES = { + "cardinality", + "default_behavior", + "migration_policy", + "ownership", + "relocation", + "rename", + "rename_with_alias", + "schema_cutover", + "sentinel_removal", + "type_normalization", +} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def require_nonempty_string(value: Any, field: str, change_id: str) -> None: + assert isinstance(value, str) and value.strip(), f"{change_id}: {field} is required" + + +def test_intentional_change_ledger_is_complete_and_well_formed() -> None: + ledger = load_toml(LEDGER_PATH) + + assert set(ledger) == REQUIRED_HEADER_FIELDS + assert ledger["ledger_version"] == 1 + assert ledger["issue"] == 2792 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert ledger["candidate_start_commit"] == ( + "8c868e430e9cd3284d7e274628419ab484ebcee0" + ) + + changes = ledger["intentional_changes"] + assert isinstance(changes, list) and changes + ids: list[str] = [] + for change in changes: + assert set(change) == REQUIRED_CHANGE_FIELDS + change_id = change["id"] + require_nonempty_string(change_id, "id", "intentional change") + ids.append(change_id) + assert change["category"] in ALLOWED_CATEGORIES + assert change["parity_disposition"] == "intentional_change" + for field in ( + "origin_main_contract", + "schema_v2_contract", + "migration", + "rationale", + ): + require_nonempty_string(change[field], field, change_id) + capability_ids = change["validation_capability_ids"] + assert isinstance(capability_ids, list) and capability_ids + assert len(capability_ids) == len(set(capability_ids)) + + assert len(ids) == len(set(ids)) + assert set(ids) == REQUIRED_CHANGE_IDS + + +def test_every_intentional_change_links_to_known_capabilities() -> None: + ledger = load_toml(LEDGER_PATH) + capabilities = load_toml(CAPABILITY_PATH)["capabilities"] + known_capability_ids = {capability["id"] for capability in capabilities} + + for change in ledger["intentional_changes"]: + assert set(change["validation_capability_ids"]) <= known_capability_ids, ( + f"{change['id']}: unknown validation capability" + ) + + +def test_ledger_does_not_hide_known_unresolved_parity_gaps() -> None: + ledger = load_toml(LEDGER_PATH) + change_ids = {change["id"] for change in ledger["intentional_changes"]} + + assert "legacy-environment-selector-upgrade" not in change_ids + assert "tls-require-client-auth-ignored" not in change_ids + assert "debian-snap-v1-upgrade" not in change_ids + + +def test_singular_selector_ledger_preserves_auto_detection() -> None: + ledger = load_toml(LEDGER_PATH) + selector = next( + change + for change in ledger["intentional_changes"] + if change["id"] == "singular-compute-driver-selector" + ) + + assert "omission retains built-in auto-detection" in selector["schema_v2_contract"] + assert "unsupported multi-driver operation" in selector["rationale"] + + +def test_operator_edited_package_configuration_is_never_auto_rewritten() -> None: + ledger = load_toml(LEDGER_PATH) + package_policy = next( + change + for change in ledger["intentional_changes"] + if change["id"] == "package-default-only-auto-migration" + ) + + assert "byte-identical package defaults" in package_policy["schema_v2_contract"] + assert "preserve every edited file" in package_policy["migration"] From 3d1ff83759716591102565b4dc79b0f9b0efd05e Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 15:59:21 -0400 Subject: [PATCH 11/42] docs(config): disposition schema v2 parity gaps Signed-off-by: Jesse Jaggars --- .../schema-v2-parity-gap-dispositions.toml | 106 +++++++++++++ ..._schema_v2_parity_gap_dispositions_test.py | 145 ++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml create mode 100644 python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py diff --git a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml new file mode 100644 index 0000000000..0b4dd110c3 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Reviewed potential blockers for the schema-v2 parity campaign. A disposition +# records whether the branch must change before parity can be claimed; it does +# not record test execution results. +ledger_version = 1 +issue = 2792 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +[[gaps]] +id = "legacy-environment-selector-upgrade" +severity = "blocker" +parity_relation = "regression" +disposition = "must_fix_before_parity" +origin_main_behavior = "OPENSHELL_DRIVERS accepts one driver name and is loaded by RPM, Debian, and Homebrew gateway services through gateway.env." +candidate_behavior = "Any presence of OPENSHELL_DRIVERS aborts startup before the canonical selector is resolved." +impact = "An otherwise valid in-place package upgrade can make the gateway unavailable without changing the operator-owned environment file." +resolution = "Accept one non-empty OPENSHELL_DRIVERS value as a deprecated environment-only alias when OPENSHELL_COMPUTE_DRIVER is absent; reject multiple values and conflicting canonical and legacy values, and emit a migration warning without logging secrets. Keep removed CLI flags rejected." +validation = "Start the candidate through package-style environment loading with one legacy driver, a conflicting canonical driver, multiple legacy drivers, and the canonical replacement; assert selection, diagnostics, and readiness." +owner_step = 6 + +[[gaps]] +id = "debian-snap-v1-upgrade" +severity = "blocker" +parity_relation = "upgrade_regression" +disposition = "must_fix_before_release_gate" +origin_main_behavior = "Debian auto-discovers a persistent schema-v1 XDG gateway.toml and Snap passes a persistent schema-v1 SNAP_COMMON gateway.toml." +candidate_behavior = "The schema-v2 gateway rejects those files, and neither package currently performs a package-specific migration or preflight." +impact = "Upgrading a working installation can leave its gateway service unable to start." +resolution = "Add package-specific preflight behavior that never overwrites edited configuration, recognizes any package-generated v1 default that is safe to migrate, and reports an actionable manual migration diagnostic for every preserved v1 file." +validation = "Upgrade real prior Debian and Snap artifacts with generated and edited v1 files; verify safe migration or deterministic operator guidance, file preservation, and successful restart after conversion." +owner_step = 12 + +[[gaps]] +id = "tls-require-client-auth-ignored" +severity = "major" +parity_relation = "preexisting_inaccessible_option" +disposition = "must_fix_or_remove_claim" +origin_main_behavior = "The TOML schema accepts and documents require_client_auth, but runtime derives the value from client CA presence and OIDC instead of using the configured boolean." +candidate_behavior = "Schema v2 retains the field and the same derived runtime behavior, so an explicit value remains silently ignored." +impact = "A security-sensitive option appears configurable but cannot change listener authentication behavior." +resolution = "Represent file-level require_client_auth as an optional value, honor an explicit value with documented precedence and OIDC interaction, or remove the field from the file schema and documentation. Require security review before changing runtime authentication semantics." +validation = "With a client CA, test explicit true, explicit false, omission, and OIDC combinations using TLS clients with and without certificates." +owner_step = 6 + +[[gaps]] +id = "unselected-driver-validation-claim" +severity = "minor" +parity_relation = "documentation_gap" +disposition = "documentation_fix_required" +origin_main_behavior = "Only the selected driver table receives driver-specific deserialization; unknown fields inside an unselected table are ignored." +candidate_behavior = "Schema v2 validates every driver entry is a table but still deserializes only the selected driver, while documentation broadly claims unknown driver fields fail startup." +impact = "Operators may believe a dormant driver configuration was fully validated when only its TOML table shape was checked." +resolution = "Narrow documentation to gateway fields, all driver-table shapes, and selected-driver fields unless registry-level validation for every recognized table is intentionally added." +validation = "Start Podman with an unknown Docker field present, then select Docker with the same field; assert the former is accepted and the latter fails." +owner_step = 6 + +[[gaps]] +id = "multi-driver-runtime-loss" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Schema v1 accepts a list-shaped selector but runtime rejects configurations containing more than one driver." +candidate_behavior = "Schema v2 represents the existing invariant as one optional scalar driver." +impact = "No working multi-driver runtime capability was removed." +resolution = "Keep the singular selector in the intentional-change ledger and validate explicit selection plus auto-detection." +validation = "Confirm origin/main rejects two configured drivers and the candidate supports one scalar or omission." +owner_step = 5 + +[[gaps]] +id = "generated-e2e-selector-shape" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Committed E2E fixtures select one driver through the schema-v1 list." +candidate_behavior = "Committed Docker and Podman fixtures and e2e/run.sh consistently use and require the schema-v2 scalar selector." +impact = "No committed fixture mismatch was found." +resolution = "Use the existing fixtures in live Docker and Podman parity lanes." +validation = "Run both driver E2E wrappers and confirm the selected runtime creates a sandbox." +owner_step = 5 + +[[gaps]] +id = "rpm-exact-default-migration" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "The RPM service seeds its package-owned schema-v1 default." +candidate_behavior = "The RPM pre-start migrator replaces only a byte-identical package v1 default and preserves edited or unsafe paths." +impact = "The implemented exact-default path does not block parity, but environment-file and edited-config behavior remain covered by separate gaps." +resolution = "Retain deterministic migration tests and add a real package upgrade lane." +validation = "Upgrade an installed prior RPM with exact and edited defaults and verify restart, bytes, ownership, and mode." +owner_step = 12 + +[[gaps]] +id = "gateway-owned-guest-tls" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Guest callback TLS may be inherited from gateway scope or overridden in the selected local driver table." +candidate_behavior = "One complete gateway-owned bundle is validated and injected into the selected local driver; misplaced driver fields fail explicitly." +impact = "The ownership change fails closed rather than silently disabling callback TLS." +resolution = "Keep the ownership change in the intentional-change ledger and validate callback connectivity on Docker, Podman, and VM." +validation = "Exercise complete, partial, misplaced, and plaintext combinations, then establish a real sandbox callback on each local driver." +owner_step = 10 diff --git a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py new file mode 100644 index 0000000000..1060414208 --- /dev/null +++ b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate schema-v2 parity-gap dispositions and release blockers.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +GAP_LEDGER_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml" +) + +REQUIRED_HEADER_FIELDS = { + "ledger_version", + "issue", + "baseline_commit", + "candidate_start_commit", + "gaps", +} +REQUIRED_GAP_FIELDS = { + "id", + "severity", + "parity_relation", + "disposition", + "origin_main_behavior", + "candidate_behavior", + "impact", + "resolution", + "validation", + "owner_step", +} +REQUIRED_GAP_IDS = { + "debian-snap-v1-upgrade", + "gateway-owned-guest-tls", + "generated-e2e-selector-shape", + "legacy-environment-selector-upgrade", + "multi-driver-runtime-loss", + "rpm-exact-default-migration", + "tls-require-client-auth-ignored", + "unselected-driver-validation-claim", +} +ALLOWED_SEVERITIES = {"blocker", "major", "minor", "none"} +ALLOWED_RELATIONS = { + "documentation_gap", + "non_finding", + "preexisting_inaccessible_option", + "regression", + "upgrade_regression", +} +ALLOWED_DISPOSITIONS = { + "documentation_fix_required", + "must_fix_before_parity", + "must_fix_before_release_gate", + "must_fix_or_remove_claim", + "no_action", +} + + +def load_ledger() -> dict[str, Any]: + with GAP_LEDGER_PATH.open("rb") as ledger_file: + return tomllib.load(ledger_file) + + +def test_gap_disposition_ledger_is_complete_and_well_formed() -> None: + ledger = load_ledger() + + assert set(ledger) == REQUIRED_HEADER_FIELDS + assert ledger["ledger_version"] == 1 + assert ledger["issue"] == 2792 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert ledger["candidate_start_commit"] == ( + "8c868e430e9cd3284d7e274628419ab484ebcee0" + ) + + gaps = ledger["gaps"] + assert isinstance(gaps, list) and gaps + ids: list[str] = [] + for gap in gaps: + assert set(gap) == REQUIRED_GAP_FIELDS + gap_id = gap["id"] + assert isinstance(gap_id, str) and gap_id.strip() + ids.append(gap_id) + assert gap["severity"] in ALLOWED_SEVERITIES + assert gap["parity_relation"] in ALLOWED_RELATIONS + assert gap["disposition"] in ALLOWED_DISPOSITIONS + assert isinstance(gap["owner_step"], int) and 1 <= gap["owner_step"] <= 15 + for field in ( + "origin_main_behavior", + "candidate_behavior", + "impact", + "resolution", + "validation", + ): + assert isinstance(gap[field], str) and gap[field].strip(), ( + f"{gap_id}: {field} is required" + ) + + assert len(ids) == len(set(ids)) + assert set(ids) == REQUIRED_GAP_IDS + + +def test_every_blocker_has_a_required_fix_and_validation() -> None: + for gap in load_ledger()["gaps"]: + if gap["severity"] != "blocker": + continue + + assert gap["disposition"].startswith("must_fix") + assert gap["owner_step"] in {6, 12} + assert len(gap["resolution"]) >= 80 + assert len(gap["validation"]) >= 80 + + +def test_non_findings_do_not_require_product_changes() -> None: + for gap in load_ledger()["gaps"]: + if gap["parity_relation"] == "non_finding": + assert gap["severity"] == "none" + assert gap["disposition"] == "no_action" + + +def test_security_sensitive_tls_gap_requires_fix_or_removed_claim() -> None: + tls_gap = next( + gap + for gap in load_ledger()["gaps"] + if gap["id"] == "tls-require-client-auth-ignored" + ) + + assert tls_gap["parity_relation"] == "preexisting_inaccessible_option" + assert tls_gap["disposition"] == "must_fix_or_remove_claim" + assert "security review" in tls_gap["resolution"] + + +def test_legacy_environment_resolution_preserves_singular_semantics() -> None: + legacy_gap = next( + gap + for gap in load_ledger()["gaps"] + if gap["id"] == "legacy-environment-selector-upgrade" + ) + + assert "one non-empty OPENSHELL_DRIVERS value" in legacy_gap["resolution"] + assert "reject multiple values" in legacy_gap["resolution"] + assert "conflicting canonical and legacy values" in legacy_gap["resolution"] From d876531020d48750f299fbb7f33cd999b66cb55b Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Wed, 2 Sep 2026 16:06:30 -0400 Subject: [PATCH 12/42] test(e2e): add dual schema parity harness Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 214 +++++++++++++++++++++++++++ e2e/parity/test.sh | 138 +++++++++++++++++ e2e/support/podman-gateway-config.sh | 135 +++++++++++++++++ e2e/with-podman-gateway.sh | 95 +++--------- tasks/parity.toml | 11 ++ tasks/test.toml | 1 + 6 files changed, 522 insertions(+), 72 deletions(-) create mode 100755 e2e/parity/run.sh create mode 100755 e2e/parity/test.sh create mode 100755 e2e/support/podman-gateway-config.sh create mode 100644 tasks/parity.toml diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh new file mode 100755 index 0000000000..c61b411492 --- /dev/null +++ b/e2e/parity/run.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compare the frozen schema-v1 gateway contract with the checkout's schema-v2 +# contract. This intentionally begins with one small semantic scenario; later +# parity waves add scenarios without changing the isolated variant runner. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MANIFEST="${OPENSHELL_PARITY_CAPABILITY_MANIFEST:-${ROOT}/e2e/configs/gateway/schema-v2-capability-parity.toml}" +DRIVER="" +BASELINE_WORKTREE="${OPENSHELL_PARITY_BASELINE_WORKTREE:-}" +RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" +WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" +TEMP_WORKTREE="" +RUN_DIR="" + +usage() { + cat >&2 <&2; exit 2; } + DRIVER=$2 + shift 2 + ;; + --baseline-worktree) + [ "$#" -ge 2 ] || { echo "ERROR: --baseline-worktree requires a path." >&2; exit 2; } + BASELINE_WORKTREE=$2 + shift 2 + ;; + --results-dir) + [ "$#" -ge 2 ] || { echo "ERROR: --results-dir requires a path." >&2; exit 2; } + RESULTS_DIR=$2 + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; usage; exit 2 ;; + esac +done + +if [ "${DRIVER}" != "podman" ]; then + echo "ERROR: only --driver podman is supported by the schema parity harness (got ${DRIVER:-})." >&2 + echo " Docker, Kubernetes, and VM backends are reserved for later parity waves." >&2 + exit 2 +fi + +if [ ! -f "${MANIFEST}" ]; then + echo "ERROR: parity capability manifest not found: ${MANIFEST}" >&2 + exit 2 +fi +BASELINE_SHA="$(awk -F '"' '/^[[:space:]]*baseline_commit[[:space:]]*=/ { print $2; exit }' "${MANIFEST}")" +if ! [[ "${BASELINE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: manifest baseline_commit must be a full 40-character SHA: ${MANIFEST}" >&2 + exit 2 +fi +BASELINE_SHA="${BASELINE_SHA,,}" +CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" + +cleanup() { + local status=$? + if [ -n "${TEMP_WORKTREE}" ]; then + git -C "${ROOT}" worktree remove --force "${TEMP_WORKTREE}" >/dev/null 2>&1 || true + fi + if [ -n "${RUN_DIR}" ]; then + rm -rf "${RUN_DIR}" || true + fi + exit "${status}" +} +trap cleanup EXIT + +if [ -n "${BASELINE_WORKTREE}" ]; then + if [ ! -d "${BASELINE_WORKTREE}" ]; then + echo "ERROR: baseline worktree does not exist: ${BASELINE_WORKTREE}" >&2 + exit 2 + fi + resolved_baseline_sha="$(git -C "${BASELINE_WORKTREE}" rev-parse HEAD 2>/dev/null || true)" + if [ "${resolved_baseline_sha}" != "${BASELINE_SHA}" ]; then + echo "ERROR: baseline worktree is not frozen manifest commit ${BASELINE_SHA}: ${BASELINE_WORKTREE}" >&2 + exit 2 + fi +else + if ! git -C "${ROOT}" cat-file -e "${BASELINE_SHA}^{commit}" 2>/dev/null; then + echo "ERROR: frozen baseline ${BASELINE_SHA} is unavailable locally; fetch it before running parity." >&2 + exit 2 + fi + TEMP_WORKTREE="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-baseline.XXXXXX")" + # Remove mktemp's directory so git worktree can create and register it. + rmdir "${TEMP_WORKTREE}" + git -C "${ROOT}" worktree add --detach "${TEMP_WORKTREE}" "${BASELINE_SHA}" >/dev/null + BASELINE_WORKTREE="${TEMP_WORKTREE}" +fi + +RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-run.XXXXXX")" +RESULTS_DIR="${RESULTS_DIR:-${ROOT}/target/parity/results}" +mkdir -p "${RESULTS_DIR}" + +require_executable() { + local label=$1 + local binary=$2 + if [ ! -x "${binary}" ]; then + echo "ERROR: ${label} binary is not executable: ${binary}" >&2 + exit 2 + fi +} + +build_variant() { + local variant=$1 source_root=$2 target_dir=$3 gateway_override=$4 cli_override=$5 conformance_override=$6 + local gateway_var=$7 cli_var=$8 conformance_var=$9 + local gateway cli conformance jobs=() + + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then jobs=(-j "${CARGO_BUILD_JOBS}"); fi + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + gateway="${gateway_override:-${target_dir}/debug/openshell-gateway}" + cli="${cli_override:-${target_dir}/debug/openshell}" + conformance="${conformance_override:-${target_dir}/debug/openshell-conformance}" + + if [ -z "${gateway_override}" ]; then + echo "Building ${variant} gateway in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-gateway --bin openshell-gateway) + fi + if [ -z "${cli_override}" ]; then + echo "Building ${variant} CLI in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-cli) + fi + if [ -z "${conformance_override}" ]; then + echo "Building ${variant} conformance CLI in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-conformance-cli) + fi + require_executable "${variant} gateway" "${gateway}" + require_executable "${variant} CLI" "${cli}" + require_executable "${variant} conformance" "${conformance}" + printf -v "${gateway_var}" '%s' "${gateway}" + printf -v "${cli_var}" '%s' "${cli}" + printf -v "${conformance_var}" '%s' "${conformance}" +} + +BASELINE_GATEWAY="" BASELINE_CLI="" BASELINE_CONFORMANCE="" +CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" +build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE +build_variant candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE +require_executable "Podman parity wrapper" "${WRAPPER}" + +write_result() { + local variant=$1 source_sha=$2 schema=$3 status=$4 + cat >"${RESULTS_DIR}/${variant}.json" <"${RESULTS_DIR}/comparison.json" < schema parity ${variant} (schema v${schema}, ${DRIVER})" + if env \ + OPENSHELL_PARITY_VARIANT="${variant}" \ + OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ + OPENSHELL_GATEWAY_BIN="${gateway}" \ + OPENSHELL_BIN="${cli}" \ + OPENSHELL_CONFORMANCE_BIN="${conformance}" \ + XDG_CONFIG_HOME="${variant_home}/config" \ + XDG_STATE_HOME="${variant_home}/state" \ + XDG_CACHE_HOME="${variant_home}/cache" \ + XDG_DATA_HOME="${variant_home}/data" \ + "${WRAPPER}" "${conformance}" run --openshell-bin "${cli}" --output json \ + 2>&1 | tee "${RESULTS_DIR}/${variant}.log"; then + result_status=true + else + result_status=false + fi + write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" + [ "${result_status}" = true ] +} + +baseline_exit=0 +candidate_exit=0 +run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" || baseline_exit=$? +# Do not short-circuit: a candidate result is useful even when the frozen +# baseline failed, and two equal failures must never constitute parity. +run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" || candidate_exit=$? + +baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) +candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) +write_comparison "${baseline_success}" "${candidate_success}" + +if [ "${baseline_exit}" -ne 0 ] || [ "${candidate_exit}" -ne 0 ]; then + echo "ERROR: schema parity requires both baseline and candidate conformance smoke runs to succeed." >&2 + exit 1 +fi +echo "Schema parity passed: baseline schema v1 and candidate schema v2 succeeded." diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh new file mode 100755 index 0000000000..4b9026b61f --- /dev/null +++ b/e2e/parity/test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Deterministic contract tests for e2e/parity/run.sh. No container runtime is +# invoked; the Podman wrapper and all three artifacts are tiny local fakes. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-test.XXXXXX")" +trap 'rm -rf "${WORKDIR}"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } +assert_contains() { grep -F -- "$2" "$1" >/dev/null || fail "expected $1 to contain: $2"; } +assert_not_contains() { ! grep -F -- "$2" "$1" >/dev/null || fail "expected $1 not to contain: $2"; } +assert_status() { [ "$1" -eq "$2" ] || fail "expected status $2, got $1"; } + +# Schema generator behavior is separately deterministic and does not need a +# gateway, certificates, or Podman. +# shellcheck source=e2e/support/gateway-common.sh +source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck source=e2e/support/podman-gateway-config.sh +source "${ROOT}/e2e/support/podman-gateway-config.sh" +mkdir -p "${WORKDIR}/pki/client" "${WORKDIR}/jwt" +e2e_write_podman_gateway_config "${WORKDIR}/v1.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +e2e_write_podman_gateway_config "${WORKDIR}/v2.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +assert_contains "${WORKDIR}/v1.toml" 'version = 1' +assert_contains "${WORKDIR}/v1.toml" 'compute_drivers = ["podman"]' +assert_contains "${WORKDIR}/v1.toml" 'image_pull_policy = "missing"' +assert_contains "${WORKDIR}/v1.toml" 'health_check_interval_secs = 0' +assert_contains "${WORKDIR}/v1.toml" 'guest_tls_ca = ' +assert_contains "${WORKDIR}/v2.toml" 'version = 2' +assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"' +assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"' +assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0' +# V2 guest TLS is emitted before its driver table; V1 is driver-local. +v1_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v1.toml" | cut -d: -f1)" +v1_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v1.toml" | cut -d: -f1)" +v2_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v2.toml" | cut -d: -f1)" +v2_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v2.toml" | cut -d: -f1)" +[ "${v1_tls_line}" -gt "${v1_driver_line}" ] || fail 'v1 TLS must be driver-local' +[ "${v2_tls_line}" -lt "${v2_driver_line}" ] || fail 'v2 TLS must be gateway-owned' +if OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 e2e_podman_config_schema_version >/dev/null 2>&1; then + fail 'invalid schema version unexpectedly accepted' +fi +set +e +env -u OPENSHELL_GATEWAY_ENDPOINT \ + OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 \ + bash "${ROOT}/e2e/with-podman-gateway.sh" true >"${WORKDIR}/wrapper-schema.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/wrapper-schema.out" 'must be 1 or 2' + +HEAD_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +cat >"${WORKDIR}/manifest.toml" <"${WORKDIR}/bin/fake-wrapper" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" >>"$OPENSHELL_PARITY_TEST_CALLS" +exec "$@" +EOF +cat >"${WORKDIR}/bin/fake-conformance" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = "${OPENSHELL_PARITY_VARIANT:-}" ] || [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = both ]; then + exit 17 +fi +printf '{"untrusted":"raw output is intentionally not normalized"}\n' +EOF +for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli; do + cat >"${WORKDIR}/bin/${artifact}" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +done +chmod +x "${WORKDIR}/bin/"* + +run_harness() { + OPENSHELL_PARITY_CAPABILITY_MANIFEST="${WORKDIR}/manifest.toml" \ + OPENSHELL_PARITY_BASELINE_WORKTREE="${ROOT}" \ + OPENSHELL_PARITY_PODMAN_WRAPPER="${WORKDIR}/bin/fake-wrapper" \ + OPENSHELL_PARITY_BASELINE_GATEWAY_BIN="${WORKDIR}/bin/baseline-gateway" \ + OPENSHELL_PARITY_BASELINE_CLI_BIN="${WORKDIR}/bin/baseline-cli" \ + OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN="${WORKDIR}/bin/candidate-gateway" \ + OPENSHELL_PARITY_CANDIDATE_CLI_BIN="${WORKDIR}/bin/candidate-cli" \ + OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \ + OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ + bash "${ROOT}/e2e/parity/run.sh" --driver podman +} + +run_harness +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance" +[ "$(sed -n '1s/|.*//p' "${WORKDIR}/calls")" = baseline ] || fail 'baseline was not invoked first' +[ "$(sed -n '2s/|.*//p' "${WORKDIR}/calls")" = candidate ] || fail 'candidate was not invoked second' +assert_contains "${WORKDIR}/results/baseline.json" "\"source_sha\":\"${HEAD_SHA}\"" +assert_contains "${WORKDIR}/results/baseline.json" '"schema_version":1' +assert_contains "${WORKDIR}/results/candidate.json" '"schema_version":2' +assert_contains "${WORKDIR}/results/candidate.json" "\"source_sha\":\"${HEAD_SHA}\"" +assert_contains "${WORKDIR}/results/candidate.json" '"success":true' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":true' +assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' +assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' + +set +e +OPENSHELL_PARITY_FAIL_VARIANT=both run_harness >"${WORKDIR}/failure.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/baseline.json" '"success":false' +assert_contains "${WORKDIR}/results/candidate.json" '"success":false' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' +[ "$(wc -l <"${WORKDIR}/calls")" -eq 4 ] || fail 'candidate did not run after baseline failure' + +set +e +bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/driver.out" 'only --driver podman is supported' + +set +e +bash "${ROOT}/e2e/parity/run.sh" --driver >"${WORKDIR}/option.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/option.out" '--driver requires a value' + +echo 'e2e parity deterministic tests passed.' diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh new file mode 100755 index 0000000000..410a5fec48 --- /dev/null +++ b/e2e/support/podman-gateway-config.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Schema-aware Podman gateway configuration generation shared by the local e2e +# wrapper and schema parity harness. This file expects gateway-common.sh to +# have been sourced first. + +e2e_podman_config_schema_version() { + local version="${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION:-2}" + + case "${version}" in + 1|2) printf '%s\n' "${version}" ;; + *) + echo "ERROR: OPENSHELL_E2E_CONFIG_SCHEMA_VERSION must be 1 or 2 (got ${version})." >&2 + return 2 + ;; + esac +} + +e2e_podman_toml_string() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '"%s"' "${value}" +} + +# Write the minimally configured Podman e2e gateway TOML. The schema-v1 +# branch deliberately uses the frozen-main contract: list driver selection, +# driver-local guest TLS, the old "missing" pull-policy spelling, and zero to +# disable Podman health checks. Schema v2 uses gateway-owned guest TLS and the +# current positive health-check setting from the RPM template. +e2e_write_podman_gateway_config() { + local output=$1 + local schema_version=$2 + local root=$3 + local pki_dir=$4 + local jwt_dir=$5 + local gateway_id=$6 + local external_driver=$7 + local driver_socket=$8 + local network_name=$9 + local gateway_port=${10} + local sandbox_image=${11} + local stop_timeout_secs=${12} + local supervisor_image=${13} + local provider_spiffe_socket=${14} + local podman_socket=${15} + local oidc_mode=${16} + local oidc_issuer=${17} + local configured_with_tls + + case "${schema_version}" in + 1) + cp "${root}/deploy/rpm/gateway.toml.default.v1" "${output}" + { + e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" + if [ "${oidc_mode}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${oidc_issuer}" ]; then + e2e_write_gateway_oidc_config "${oidc_issuer}" + fi + fi + printf '\n[openshell.drivers.podman]\n' + if [ "${external_driver}" = "1" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" + else + printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" + printf 'gateway_port = %s\n' "${gateway_port}" + printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" + printf 'image_pull_policy = "missing"\n' + # In schema v1, zero explicitly disables Podman health checks. + printf 'health_check_interval_secs = 0\n' + printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" + printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" + printf 'guest_tls_ca = %s\n' "$(e2e_podman_toml_string "${pki_dir}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + if [ -n "${provider_spiffe_socket}" ]; then + printf 'provider_spiffe_workload_api_socket = %s\n' "$(e2e_podman_toml_string "${provider_spiffe_socket}")" + fi + if [ -n "${podman_socket}" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" + fi + fi + } >>"${output}" + ;; + 2) + cp "${root}/deploy/rpm/gateway.toml.default" "${output}" + # The v2 template opens the Podman table. Insert gateway-owned TLS + # before it rather than reopening [openshell.gateway] later. + configured_with_tls="${output}.tls" + while IFS= read -r line; do + if [ "${line}" = "[openshell.drivers.podman]" ]; then + printf 'guest_tls_ca = %s\n' "$(e2e_podman_toml_string "${pki_dir}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.key")" + fi + printf '%s\n' "${line}" + done <"${output}" >"${configured_with_tls}" + mv "${configured_with_tls}" "${output}" + { + if [ "${external_driver}" = "1" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" + else + printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" + printf 'gateway_port = %s\n' "${gateway_port}" + printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" + printf 'image_pull_policy = "if_not_present"\n' + printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" + printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" + printf 'enable_bind_mounts = true\n' + if [ -n "${provider_spiffe_socket}" ]; then + printf 'provider_spiffe_workload_api_socket = %s\n' "$(e2e_podman_toml_string "${provider_spiffe_socket}")" + fi + if [ -n "${podman_socket}" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" + fi + fi + e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" + if [ "${oidc_mode}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${oidc_issuer}" ]; then + e2e_write_gateway_oidc_config "${oidc_issuer}" + fi + fi + } >>"${output}" + ;; + *) + echo "ERROR: unsupported Podman config schema version: ${schema_version}" >&2 + return 2 + ;; + esac +} diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index efc829cb62..1d2176bc97 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -28,6 +28,8 @@ fi ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=e2e/support/gateway-common.sh source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck source=e2e/support/podman-gateway-config.sh +source "${ROOT}/e2e/support/podman-gateway-config.sh" require_container_engine_lane() { local lane=$1 @@ -370,6 +372,9 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then exit $? fi +# Validate the generated configuration dialect before creating runtime resources. +CONFIG_SCHEMA_VERSION="$(e2e_podman_config_schema_version)" + # Preflight for managed Podman gateway mode. if ! command -v podman >/dev/null 2>&1; then echo "ERROR: podman CLI is required to run Podman-backed e2e tests" >&2 @@ -437,79 +442,25 @@ export OPENSHELL_E2E_SANDBOX_NAMESPACE="${E2E_NAMESPACE}" echo "Starting openshell-gateway on port ${HOST_PORT} (namespace: ${E2E_NAMESPACE})..." e2e_generate_gateway_jwt "${JWT_DIR}" -# Driver-specific options moved from CLI flags into a TOML config table -# (commit 560550d2). Synthesize a minimal config here and pass --config. -# Quote a value as a TOML basic string: see with-docker-gateway.sh for -# the same helper (kept duplicated to avoid sourcing across e2e scripts). -toml_string() { - local value="$1" - value="${value//\\/\\\\}" - value="${value//\"/\\\"}" - printf '"%s"' "${value}" -} - GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" - -# Start from the RPM default template so this e2e test exercises the same TOML -# config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_driver = "podman". On Podman Machine, -# the driver reserves IPv4 loopback for its callback-only listener, so the -# primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. -# -# We append the driver-specific table and override the port via CLI flag -# (CLI > TOML in the merge precedence) so the test can use an ephemeral port. -cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" -# The TLS listener credentials are supplied by CLI below. Schema v2 keeps the -# supervisor client bundle gateway-owned, so add it to [openshell.gateway] -# before the RPM template opens the Podman driver table. -GATEWAY_CONFIG_WITH_TLS="${GATEWAY_CONFIG}.tls" -while IFS= read -r line; do - if [ "${line}" = "[openshell.drivers.podman]" ]; then - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - fi - printf '%s\n' "${line}" -done <"${GATEWAY_CONFIG}" >"${GATEWAY_CONFIG_WITH_TLS}" -mv "${GATEWAY_CONFIG_WITH_TLS}" "${GATEWAY_CONFIG}" -{ - # The RPM template ends in [openshell.drivers.podman]. Append driver-owned - # overrides before opening any nested gateway tables below. - if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then - printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" - else - # The Podman driver scopes isolation by network rather than namespace. - printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" - printf 'gateway_port = %s\n' "${HOST_PORT}" - printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = "if_not_present"\n' - # The RPM template already opts into the 10-second Podman health check. - # Keep CI teardown bounded while the production Podman driver default stays - # conservative for real user workloads. - printf 'stop_timeout_secs = %s\n' "${PODMAN_STOP_TIMEOUT_SECS}" - printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - printf 'enable_bind_mounts = true\n' - if [ -n "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" ]; then - printf 'provider_spiffe_workload_api_socket = %s\n' "$(toml_string "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" - fi - # The in-process Podman driver reads `socket_path` from TOML only — the - # OPENSHELL_PODMAN_SOCKET env var is honoured by the standalone driver - # binary, not the in-process driver used here. Pin the socket to the one - # the harness discovered (e.g. via `podman machine inspect` on macOS) so - # we don't fall back to the driver's stale macOS default. - if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then - printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" - fi - fi - - e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" - if [ "${OIDC_MODE}" != "1" ]; then - e2e_write_gateway_mtls_auth_config - if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" - fi - fi -} >> "${GATEWAY_CONFIG}" +e2e_write_podman_gateway_config \ + "${GATEWAY_CONFIG}" \ + "${CONFIG_SCHEMA_VERSION}" \ + "${ROOT}" \ + "${PKI_DIR}" \ + "${JWT_DIR}" \ + "openshell-e2e-podman-${HOST_PORT}" \ + "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" \ + "${DRIVER_SOCKET}" \ + "${PODMAN_NETWORK_NAME}" \ + "${HOST_PORT}" \ + "${SANDBOX_IMAGE}" \ + "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SUPERVISOR_IMAGE}" \ + "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" \ + "${OPENSHELL_PODMAN_SOCKET:-}" \ + "${OIDC_MODE}" \ + "${OPENSHELL_OIDC_ISSUER:-}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ diff --git a/tasks/parity.toml b/tasks/parity.toml new file mode 100644 index 0000000000..30a4baa33f --- /dev/null +++ b/tasks/parity.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +["test:e2e-parity"] +description = "Run deterministic schema-v1/schema-v2 parity harness tests" +run = "bash e2e/parity/test.sh" +hide = true + +["e2e:parity:podman"] +description = "Compare frozen schema-v1 and current schema-v2 conformance against Podman (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman" diff --git a/tasks/test.toml b/tasks/test.toml index 8825c6ed1f..aaa3d3ef71 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -14,6 +14,7 @@ depends = [ "test:build-env", "test:gateway-pull-policy", "test:gateway-config", + "test:e2e-parity", "test:packaging-assets", "test:codex-security-release-range", "test:docs-website", From 1f8e937e89ca114c967e9431cd262b7041c9d1d3 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 12:20:55 -0400 Subject: [PATCH 13/42] test(e2e): establish compute lifecycle parity baseline Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-live-results.toml | 65 ++++++++ e2e/parity/run.sh | 12 +- e2e/parity/test.sh | 17 +- .../gateway_schema_v2_live_results_test.py | 145 ++++++++++++++++++ 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 e2e/configs/gateway/schema-v2-live-results.toml create mode 100644 python/openshell/gateway_schema_v2_live_results_test.py diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml new file mode 100644 index 0000000000..f11c758c10 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +# Step 5 establishes the portable status/create/Ready/list/exec/delete/list-empty +# contract. A platform-blocked result is not parity evidence; each blocked row +# names the lane that must replace it before the final release gate can pass. + +[[result]] +id = "portable-lifecycle-podman" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Podman compute driver" +driver = "podman" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "a3860084d019ed2ac979e3eaa1ddf085a96b773c" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Paired conformance smoke connected with authenticated mTLS on both variants.", + "Both variants passed status, create, Ready inspection, paginated list visibility, exact-output exec, delete, and eventual empty-list checks.", + "Normalized comparison recorded baseline_success=true, candidate_success=true, and parity=true; equal failures are rejected by the harness.", +] + +[[result]] +id = "portable-lifecycle-docker" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Docker compute driver" +driver = "docker" +status = "platform_blocked" +owner = "OpenShell Linux Docker CI lane" +lane = "linux-x86_64-docker" +blocker = "The validation host has no Docker CLI or Docker daemon socket. Step 7 and Step 13 must execute and assign this lane." + +[[result]] +id = "portable-lifecycle-kubernetes" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Kubernetes compute driver" +driver = "kubernetes" +status = "platform_blocked" +owner = "OpenShell Kubernetes CI lane" +lane = "ephemeral-kind-or-managed-kubernetes" +blocker = "The active kubectl context is an external OpenShift cluster that this validation must not modify. The installed kind cluster belongs to another project; Step 8 and Step 13 must use a dedicated OpenShell cluster." + +[[result]] +id = "portable-lifecycle-vm" +step = 5 +capability = "Portable sandbox lifecycle on the standalone VM compute driver" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM CI lane" +lane = "linux-x86_64-kvm-libkrun" +blocker = "The validation host has no prepared VM runtime bundle or built openshell-driver-vm executable. Step 9 and Step 13 must execute and assign this lane." + +[[result]] +id = "portable-lifecycle-mxc" +step = 5 +capability = "Portable sandbox lifecycle on the Windows MXC compute driver" +driver = "mxc" +status = "platform_blocked" +owner = "OpenShell Windows MXC CI lane" +lane = "windows-x64-and-windows-arm64-mxc" +blocker = "The validation host is Linux and cannot execute the Windows MXC runtime. Step 13 must assign native Windows validation." diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index c61b411492..8dec1ef6d7 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -14,6 +14,7 @@ DRIVER="" BASELINE_WORKTREE="${OPENSHELL_PARITY_BASELINE_WORKTREE:-}" RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" +PODMAN_BIN="${OPENSHELL_PARITY_PODMAN_BIN:-podman}" TEMP_WORKTREE="" RUN_DIR="" @@ -75,7 +76,15 @@ cleanup() { git -C "${ROOT}" worktree remove --force "${TEMP_WORKTREE}" >/dev/null 2>&1 || true fi if [ -n "${RUN_DIR}" ]; then - rm -rf "${RUN_DIR}" || true + # Rootless Podman overlay files can be owned by subordinate UIDs. Remove an + # isolated container store from Podman's user namespace before falling back + # to ordinary cleanup for runs that never reached the container runtime. + if { [ -d "${RUN_DIR}/baseline/data/containers/storage" ] \ + || [ -d "${RUN_DIR}/candidate/data/containers/storage" ]; } \ + && command -v "${PODMAN_BIN}" >/dev/null 2>&1; then + "${PODMAN_BIN}" unshare rm -rf -- "${RUN_DIR}" >/dev/null 2>&1 || true + fi + rm -rf "${RUN_DIR}" >/dev/null 2>&1 || true fi exit "${status}" } @@ -182,6 +191,7 @@ run_variant() { OPENSHELL_GATEWAY_BIN="${gateway}" \ OPENSHELL_BIN="${cli}" \ OPENSHELL_CONFORMANCE_BIN="${conformance}" \ + MISE_TRUSTED_CONFIG_PATHS="${MISE_TRUSTED_CONFIG_PATHS:-${ROOT}}" \ XDG_CONFIG_HOME="${variant_home}/config" \ XDG_STATE_HOME="${variant_home}/state" \ XDG_CACHE_HOME="${variant_home}/cache" \ diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 4b9026b61f..d5f2c416a8 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -63,7 +63,16 @@ mkdir -p "${WORKDIR}/bin" cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF' #!/usr/bin/env bash set -euo pipefail -printf '%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" >>"$OPENSHELL_PARITY_TEST_CALLS" +printf '%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" >>"$OPENSHELL_PARITY_TEST_CALLS" +mkdir -p "$XDG_DATA_HOME/containers/storage" +exec "$@" +EOF +cat >"${WORKDIR}/bin/fake-podman" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$OPENSHELL_PARITY_TEST_PODMAN_CALLS" +[ "$1" = unshare ] || exit 19 +shift exec "$@" EOF cat >"${WORKDIR}/bin/fake-conformance" <<'EOF' @@ -86,6 +95,7 @@ run_harness() { OPENSHELL_PARITY_CAPABILITY_MANIFEST="${WORKDIR}/manifest.toml" \ OPENSHELL_PARITY_BASELINE_WORKTREE="${ROOT}" \ OPENSHELL_PARITY_PODMAN_WRAPPER="${WORKDIR}/bin/fake-wrapper" \ + OPENSHELL_PARITY_PODMAN_BIN="${WORKDIR}/bin/fake-podman" \ OPENSHELL_PARITY_BASELINE_GATEWAY_BIN="${WORKDIR}/bin/baseline-gateway" \ OPENSHELL_PARITY_BASELINE_CLI_BIN="${WORKDIR}/bin/baseline-cli" \ OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ @@ -94,12 +104,15 @@ run_harness() { OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \ OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ + OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \ + MISE_TRUSTED_CONFIG_PATHS= \ bash "${ROOT}/e2e/parity/run.sh" --driver podman } run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance" +assert_contains "${WORKDIR}/calls" "|${ROOT}" [ "$(sed -n '1s/|.*//p' "${WORKDIR}/calls")" = baseline ] || fail 'baseline was not invoked first' [ "$(sed -n '2s/|.*//p' "${WORKDIR}/calls")" = candidate ] || fail 'candidate was not invoked second' assert_contains "${WORKDIR}/results/baseline.json" "\"source_sha\":\"${HEAD_SHA}\"" @@ -110,6 +123,8 @@ assert_contains "${WORKDIR}/results/candidate.json" '"success":true' assert_contains "${WORKDIR}/results/comparison.json" '"parity":true' assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' +assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' +assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' set +e OPENSHELL_PARITY_FAIL_VARIANT=both run_harness >"${WORKDIR}/failure.out" 2>&1 diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py new file mode 100644 index 0000000000..8c6b738288 --- /dev/null +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate machine-readable schema-v2 live parity results.""" + +from __future__ import annotations + +import subprocess +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESULTS_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-live-results.toml" +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" + +REQUIRED_HEADER_FIELDS = { + "manifest_version", + "baseline_commit", + "candidate_start_commit", + "result", +} +REQUIRED_STEP_5_IDS = { + "portable-lifecycle-docker", + "portable-lifecycle-kubernetes", + "portable-lifecycle-mxc", + "portable-lifecycle-podman", + "portable-lifecycle-vm", +} +ALLOWED_STATUSES = { + "pass", + "intentional_change", + "regression", + "platform_blocked", +} +BASE_FIELDS = {"id", "step", "capability", "driver", "status", "lane"} +PASS_FIELDS = { + "validated_baseline_commit", + "validated_candidate_commit", + "evidence", +} +BLOCKED_FIELDS = {"owner", "blocker"} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def assert_full_sha(value: object, field: str) -> str: + assert isinstance(value, str), f"{field} must be a string" + assert len(value) == 40 and all(char in "0123456789abcdef" for char in value), ( + f"{field} must be a full lowercase SHA" + ) + return value + + +def test_live_results_manifest_is_well_formed() -> None: + manifest = load_toml(RESULTS_PATH) + + assert set(manifest) == REQUIRED_HEADER_FIELDS + assert manifest["manifest_version"] == 1 + baseline = assert_full_sha(manifest["baseline_commit"], "baseline_commit") + assert_full_sha(manifest["candidate_start_commit"], "candidate_start_commit") + assert baseline == load_toml(CAPABILITY_PATH)["baseline_commit"] + + results = manifest["result"] + assert isinstance(results, list) and results + ids: list[str] = [] + for result in results: + assert set(result) >= BASE_FIELDS + result_id = result["id"] + assert isinstance(result_id, str) and result_id.strip() + ids.append(result_id) + assert result["status"] in ALLOWED_STATUSES + assert isinstance(result["step"], int) and 1 <= result["step"] <= 15 + for field in ("capability", "driver", "lane"): + assert isinstance(result[field], str) and result[field].strip(), ( + f"{result_id}: {field} is required" + ) + + assert len(ids) == len(set(ids)) + + +def test_step_5_covers_every_in_tree_compute_driver() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 5 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_5_IDS + assert {result["driver"] for result in results} == { + "docker", + "kubernetes", + "mxc", + "podman", + "vm", + } + + +def test_pass_results_pin_executed_commits_and_evidence() -> None: + manifest = load_toml(RESULTS_PATH) + for result in manifest["result"]: + if result["status"] != "pass": + continue + + assert set(result) >= PASS_FIELDS, result["id"] + assert result["validated_baseline_commit"] == manifest["baseline_commit"] + candidate = assert_full_sha( + result["validated_candidate_commit"], + f"{result['id']}.validated_candidate_commit", + ) + assert isinstance(result["evidence"], list) and result["evidence"] + assert all( + isinstance(item, str) and item.strip() for item in result["evidence"] + ) + subprocess.run( + ["git", "merge-base", "--is-ancestor", candidate, "HEAD"], + cwd=REPO_ROOT, + check=True, + ) + + +def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: + for result in load_toml(RESULTS_PATH)["result"]: + if result["status"] != "platform_blocked": + continue + + assert set(result) >= BLOCKED_FIELDS, result["id"] + assert isinstance(result["owner"], str) and result["owner"].strip() + assert isinstance(result["blocker"], str) and len(result["blocker"]) >= 80 + + +def test_step_5_records_only_executed_podman_as_pass() -> None: + statuses = { + result["driver"]: result["status"] + for result in load_toml(RESULTS_PATH)["result"] + if result["step"] == 5 + } + + assert statuses["podman"] == "pass" + assert all( + status == "platform_blocked" + for driver, status in statuses.items() + if driver != "podman" + ) From 0ea6041d2187d83a5fbd2a5b43fd948539513a68 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 12:43:26 -0400 Subject: [PATCH 14/42] fix(config): preserve gateway option compatibility Signed-off-by: Jesse Jaggars --- crates/openshell-server/src/cli.rs | 265 +++++++++++++++--- crates/openshell-server/src/config_file.rs | 40 ++- crates/openshell-server/src/lib.rs | 2 + deploy/helm/openshell/values.yaml | 5 +- docs/reference/gateway-config.mdx | 18 +- .../gateway/schema-v2-capability-parity.toml | 6 +- e2e/parity/gateway-options.sh | 201 +++++++++++++ tasks/parity.toml | 4 + 8 files changed, 486 insertions(+), 55 deletions(-) create mode 100755 e2e/parity/gateway-options.sh diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index c1a42d72a5..ce35612fe9 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -239,13 +239,40 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry } } -fn reject_legacy_driver_selector_env() -> Result<()> { - if std::env::var_os("OPENSHELL_DRIVERS").is_some() { +fn resolve_legacy_driver_selector_env(args: &mut RunArgs) -> Result { + let Some(raw) = std::env::var_os("OPENSHELL_DRIVERS") else { + return Ok(false); + }; + let raw = raw.into_string().map_err(|_| { + miette::miette!( + "OPENSHELL_DRIVERS contains an invalid compute driver name; expected ASCII letters, digits, '-' or '_'" + ) + })?; + let values = raw.split(',').map(str::trim).collect::>(); + if values.len() != 1 || values[0].is_empty() { return Err(miette::miette!( - "OPENSHELL_DRIVERS is no longer supported; use OPENSHELL_COMPUTE_DRIVER with exactly one driver name" + "OPENSHELL_DRIVERS must contain exactly one non-empty compute driver name; comma-delimited lists are not supported" )); } - Ok(()) + let legacy = openshell_core::config::normalize_compute_driver_name(values[0]).map_err(|_| { + miette::miette!( + "OPENSHELL_DRIVERS contains an invalid compute driver name; expected ASCII letters, digits, '-' or '_'" + ) + })?; + + if let Some(canonical) = args.compute_driver.as_deref() { + let canonical = openshell_core::config::normalize_compute_driver_name(canonical) + .map_err(|error| miette::miette!("{error}"))?; + if canonical != legacy { + return Err(miette::miette!( + "OPENSHELL_DRIVERS conflicts with the canonical compute-driver selection; remove OPENSHELL_DRIVERS" + )); + } + } else { + args.compute_driver = Some(legacy); + } + + Ok(true) } #[cfg(test)] @@ -258,8 +285,6 @@ fn prepare_server_config_with_drivers( matches: &ArgMatches, compute_drivers: &ComputeDriverRegistry, ) -> Result { - reject_legacy_driver_selector_env()?; - // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -272,7 +297,8 @@ fn prepare_server_config_with_drivers( if let Some(file) = file.as_ref() { merge_file_into_args(args, &file.openshell.gateway, matches); } - normalize_compute_driver_socket_args(args, matches)?; + let legacy_compute_driver_env_seen = resolve_legacy_driver_selector_env(args)?; + normalize_compute_driver_socket_args(args)?; let compute_driver = compute_drivers .select(args.compute_driver.as_deref()) .map_err(|error| miette::miette!("{error}"))?; @@ -508,6 +534,7 @@ fn prepare_server_config_with_drivers( config_file: file, guest_tls, compute_driver, + legacy_compute_driver_env_seen, }) } @@ -540,6 +567,10 @@ async fn run_from_args( gateway_resource, ); + if prepared.legacy_compute_driver_env_seen { + warn!("OPENSHELL_DRIVERS is deprecated; migrate to OPENSHELL_COMPUTE_DRIVER"); + } + let has_client_ca = prepared .config .tls @@ -806,7 +837,7 @@ fn validate_grpc_rate_limit_args(requests: Option, window_seconds: Option Result<()> { +fn normalize_compute_driver_socket_args(args: &mut RunArgs) -> Result<()> { let Some(socket) = args.compute_driver_socket.as_ref() else { return Ok(()); }; @@ -815,7 +846,7 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches "--compute-driver-socket must not be an empty path" )); } - if arg_defaulted(matches, "compute_driver") { + if args.compute_driver.is_none() { return Err(miette::miette!( "--compute-driver-socket requires --compute-driver or OPENSHELL_COMPUTE_DRIVER=" )); @@ -858,7 +889,7 @@ fn resolve_mtls_auth_enabled( #[cfg(test)] mod tests { - use super::{Cli, command, reject_legacy_driver_selector_env}; + use super::{Cli, command}; use crate::TEST_ENV_LOCK as ENV_LOCK; use clap::Parser; use std::net::{IpAddr, Ipv4Addr}; @@ -1140,17 +1171,120 @@ mod tests { } #[test] - fn legacy_compute_driver_environment_is_rejected_even_when_empty() { + fn legacy_compute_driver_environment_accepts_one_normalized_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", " PodMan "); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + + assert!(super::resolve_legacy_driver_selector_env(&mut args).unwrap()); + assert_eq!(args.compute_driver.as_deref(), Some("podman")); + } + + #[test] + fn legacy_compute_driver_environment_flows_through_server_preparation() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_path = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman"); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--disable-tls", + ]); + let registry = test_registry("podman", true, true); + + let prepared = + super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); + + assert_eq!(prepared.compute_driver.name(), "podman"); + assert!(prepared.legacy_compute_driver_env_seen); + } + + #[test] + fn legacy_compute_driver_environment_rejects_empty_plural_and_invalid_values() { let _lock = ENV_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - for value in ["docker", ""] { - let guard = EnvVarGuard::set("OPENSHELL_DRIVERS", value); - let error = super::reject_legacy_driver_selector_env() - .expect_err("legacy environment selector must be rejected"); - assert!(error.to_string().contains("OPENSHELL_COMPUTE_DRIVER")); - drop(guard); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + + for value in ["", " ", ",", "podman,", ",podman", "podman,docker"] { + let legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", value); + let (mut args, _) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("empty and plural legacy selectors must be rejected"); + assert!(error.to_string().contains("exactly one non-empty")); + drop(legacy); } + + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman/path"); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("invalid legacy selector must be rejected"); + assert!(error.to_string().contains("invalid compute driver name")); + assert!(!error.to_string().contains("podman/path")); + } + + #[test] + fn legacy_compute_driver_environment_allows_equal_canonical_and_rejects_conflict() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "PODMAN"); + + let (mut equal, _) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + ]); + assert!(super::resolve_legacy_driver_selector_env(&mut equal).unwrap()); + assert_eq!(equal.compute_driver.as_deref(), Some("podman")); + + let (mut conflict, _) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "docker", + ]); + let error = super::resolve_legacy_driver_selector_env(&mut conflict) + .expect_err("different canonical and legacy selectors must conflict"); + assert!(error.to_string().contains("conflicts")); + assert!(!error.to_string().contains("podman")); + assert!(!error.to_string().contains("docker")); + } + + #[test] + fn legacy_compute_driver_environment_supports_remote_driver_socket() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _socket = EnvVarGuard::set( + "OPENSHELL_COMPUTE_DRIVER_SOCKET", + "/run/openshell/kyma.sock", + ); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + + assert!(super::resolve_legacy_driver_selector_env(&mut args).unwrap()); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); + assert_eq!( + args.compute_driver_socket.as_deref(), + Some(std::path::Path::new("/run/openshell/kyma.sock")) + ); } #[test] @@ -1298,20 +1432,6 @@ mod tests { assert!(error.to_string().contains("--drivers")); } - #[test] - fn rejects_legacy_drivers_environment_variable() { - let _lock = ENV_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - for value in ["docker", ""] { - let _guard = EnvVarGuard::set("OPENSHELL_DRIVERS", value); - let error = reject_legacy_driver_selector_env() - .expect_err("legacy OPENSHELL_DRIVERS must be rejected when present"); - assert!(error.to_string().contains("OPENSHELL_DRIVERS")); - assert!(error.to_string().contains("OPENSHELL_COMPUTE_DRIVER")); - } - } - #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK @@ -1409,6 +1529,46 @@ mod tests { assert_eq!(local_tls.client_cert, tls.path().join("client/tls.crt")); } + #[test] + fn tls_client_certificate_requirement_is_derived_from_ca_and_oidc() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_path = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let registry = test_registry("shared", false, false); + + for (oidc_issuer, expected) in [(None, true), (Some("https://idp.example.com"), false)] { + let mut startup_args = vec![ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "shared", + "--tls-cert", + "/tls/server.crt", + "--tls-key", + "/tls/server.key", + "--tls-client-ca", + "/tls/ca.crt", + ]; + if let Some(issuer) = oidc_issuer { + startup_args.extend(["--oidc-issuer", issuer]); + } + let (mut args, matches) = parse_with_args(&startup_args); + let prepared = + super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); + + assert_eq!( + prepared.config.tls.as_ref().unwrap().require_client_auth, + expected, + "oidc issuer: {oidc_issuer:?}" + ); + } + } + #[test] fn mtls_auth_auto_defaults_for_local_tls_driver() { let _lock = ENV_LOCK @@ -1658,6 +1818,28 @@ compute_driver = "podman" assert_eq!(env_args.compute_driver.as_deref(), Some("vm")); } + #[test] + fn legacy_compute_driver_environment_conflicts_with_file_selection() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "docker"); + let file = config_file_from_toml( + r#" +[openshell.gateway] +compute_driver = "podman" +"#, + ); + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("different file and legacy selectors must conflict"); + assert!(error.to_string().contains("conflicts")); + } + #[test] fn file_oidc_block_populates_oidc_args() { let _lock = ENV_LOCK @@ -1826,7 +2008,7 @@ ssh_session_ttl_secs = 1234 let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", @@ -1835,7 +2017,7 @@ ssh_session_ttl_secs = 1234 "--compute-driver-socket", "/run/openshell/kyma.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!( args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/run/openshell/kyma.sock")) @@ -1851,14 +2033,14 @@ ssh_session_ttl_secs = 1234 let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", "--compute-driver-socket", "/run/openshell/kyma.sock", ]); - let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); + let err = super::normalize_compute_driver_socket_args(&mut args).unwrap_err(); assert!( err.to_string().contains("requires --compute-driver "), @@ -1874,7 +2056,7 @@ ssh_session_ttl_secs = 1234 let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", @@ -1883,7 +2065,7 @@ ssh_session_ttl_secs = 1234 "--compute-driver-socket", "/run/openshell/extension.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!(args.compute_driver.as_deref(), Some("docker")); } @@ -1895,7 +2077,7 @@ ssh_session_ttl_secs = 1234 let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", @@ -1904,7 +2086,7 @@ ssh_session_ttl_secs = 1234 "--compute-driver-socket", "/run/openshell/vm.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!(args.compute_driver.as_deref(), Some("vm")); } @@ -1919,9 +2101,8 @@ ssh_session_ttl_secs = 1234 ); let _g2 = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "kyma"); - let (mut args, matches) = - parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!( args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/var/run/openshell/kyma.sock")) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 47dede5c89..3efd6cf7a5 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -28,7 +28,7 @@ use base64::Engine as _; use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, TlsConfig, + GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, }; use serde::{Deserialize, Serialize}; @@ -148,7 +148,7 @@ pub struct GatewayFileSection { // ── Nested tables ──────────────────────────────────────────────────── #[serde(default)] - pub tls: Option, + pub tls: Option, #[serde(default)] pub oidc: Option, #[serde(default)] @@ -173,6 +173,27 @@ pub struct GatewayFileSection { pub database_url: Option, } +/// Gateway listener TLS fields accepted in `gateway.toml`. +/// +/// Client-certificate handshake policy is derived from the presence of a +/// client CA and OIDC configuration. It is intentionally not operator-settable +/// in TOML because changing it can either weaken CA-only gateways or require a +/// second credential from OIDC clients. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GatewayTlsFileConfig { + pub cert_path: PathBuf, + pub key_path: PathBuf, + #[serde(default)] + pub client_ca_path: Option, + #[serde(default)] + pub external_cert_path: Option, + #[serde(default)] + pub external_key_path: Option, + #[serde(default)] + pub external_server_names: Vec, +} + /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -924,6 +945,21 @@ nonsense = true assert!(matches!(err, ConfigFileError::Parse { .. })); } + #[test] + fn rejects_operator_selected_tls_client_auth_policy() { + let toml = r#" +[openshell.gateway.tls] +cert_path = "/tls/server.crt" +key_path = "/tls/server.key" +client_ca_path = "/tls/ca.crt" +require_client_auth = false +"#; + let tmp = write_tmp(toml); + let error = load(tmp.path()).expect_err("derived TLS client-auth policy must be rejected"); + assert!(matches!(error, ConfigFileError::Parse { .. })); + assert!(error.to_string().contains("require_client_auth")); + } + #[test] fn rejects_removed_driver_fields_at_gateway_scope() { for field in [ diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 18c86efaed..2a2bc1a4a6 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -249,6 +249,7 @@ pub(crate) struct ServerStartupConfig { pub config_file: Option, pub guest_tls: Option, pub compute_driver: ComputeDriverSelection, + pub legacy_compute_driver_env_seen: bool, } /// Server state shared across handlers. @@ -449,6 +450,7 @@ pub(crate) async fn run_server( config_file, guest_tls, compute_driver, + legacy_compute_driver_env_seen: _, } = startup; let (shutdown_tx, shutdown_rx) = watch::channel(false); diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 93144497ca..8f5bfa9959 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -255,8 +255,9 @@ server: # Override only when sandboxes must reach the gateway via a different # hostname (e.g. an external ingress or a host alias). grpcEndpoint: "" - # TLS configuration for the server. The server always terminates mTLS - # directly and requires client certificates. + # The gateway terminates TLS directly. Its client CA authenticates sandbox + # callbacks; OIDC-enabled listeners permit bearer-only user clients while + # still validating any client certificate they present. # -- Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get # hostAliases entries mapping host.docker.internal and host.openshell.internal # to this IP, allowing them to reach services running on the Docker host. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 3b3cff69b1..75d5d2022d 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -72,7 +72,11 @@ future version. To migrate an existing file: 1. Set `[openshell] version = 2`. 2. Replace `compute_drivers = [""]` with the scalar `compute_driver = ""`. Replace `--drivers` and `OPENSHELL_DRIVERS` - with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. + with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. Package upgrades + temporarily accept one non-empty `OPENSHELL_DRIVERS` value as a deprecated + environment-only alias when it agrees with any canonical selection. + Comma-delimited, invalid, or conflicting legacy values fail startup. The + removed `--driver` and `--drivers` flags remain unsupported. 3. Move every compute-driver option into `[openshell.drivers.]`. Schema version 2 does not inherit driver defaults from `[openshell.gateway]`. Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway @@ -98,9 +102,10 @@ future version. To migrate an existing file: state retains its recorded or recoverable identity, including legacy 10001; the driver does not assign 10001 to an overlay without supporting state. -Unknown fields and non-table `[openshell.drivers.]` values fail startup. -This strict validation prevents misspelled or misplaced security-sensitive -settings from being silently ignored. +Every `[openshell.drivers.]` entry must be a TOML table. The gateway +validates driver-specific fields when it selects and constructs that driver; +it does not deserialize unselected driver tables. Unknown or misplaced fields +in the selected table fail startup instead of being silently ignored. ## Full Example @@ -183,7 +188,6 @@ timeout = "500ms" cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" client_ca_path = "/etc/openshell/certs/client-ca.pem" -require_client_auth = false # Optional: SNI-based dual certificate for external (e.g. ACME) TLS. # external_cert_path = "/etc/openshell/certs/external.pem" # external_key_path = "/etc/openshell/certs/external-key.pem" @@ -255,7 +259,9 @@ namespace = "openshell" allow_reference_namespace = false ``` -Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to map a verified client certificate to a CLI user identity. This application-layer identity switch does not control the TLS handshake. When `client_ca_path` is set without OIDC, the listener requires a valid client certificate. When OIDC is configured, bearer-only clients may connect; the listener still validates any client certificate they present against the configured CA. Kubernetes deployments must leave `mtls_auth.enabled` unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. + +The client-certificate handshake policy is derived and has no `require_client_auth` TOML field. This preserves bearer-only OIDC clients and prevents a file setting from silently weakening CA-only gateways. `[openshell.gateway.tls]` supports optional SNI-based dual-certificate mode for deployments that need separate internal and external server certificates. Set `external_cert_path` and `external_key_path` to point at the external (e.g. ACME/publicly-trusted) certificate and key. List the hostnames that should be served with the external certificate in `external_server_names`. Connections whose TLS SNI hostname matches one of those names receive the external certificate; all other connections (including those with no SNI) receive the primary internal certificate from `cert_path`/`key_path`. Both fields must be set together — providing only one is a configuration error. On Kubernetes with the Helm chart, the external certificate is managed automatically when `certManager.serverIssuerRef.name` is set; the chart populates these fields from the cert-manager-issued external server certificate. diff --git a/e2e/configs/gateway/schema-v2-capability-parity.toml b/e2e/configs/gateway/schema-v2-capability-parity.toml index 56afce90a0..650669e9a7 100644 --- a/e2e/configs/gateway/schema-v2-capability-parity.toml +++ b/e2e/configs/gateway/schema-v2-capability-parity.toml @@ -93,8 +93,8 @@ status = "not_run" id = "gateway-listener-tls-and-sni" topics = ["auth_tls_jwt", "listeners"] origin_main_access_paths = ["--tls-cert / OPENSHELL_TLS_CERT", "--tls-key / OPENSHELL_TLS_KEY", "--tls-client-ca / OPENSHELL_TLS_CLIENT_CA", "[openshell.gateway.tls]"] -schema_v2_access_paths = ["[openshell.gateway.tls].{cert_path,key_path,client_ca_path,require_client_auth,external_cert_path,external_key_path,external_server_names}", "same CLI and environment variables for primary bundle"] -behavioral_oracle = "The listener presents the primary or configured SNI certificate, validates client certificates when required, and rejects incomplete TLS bundles." +schema_v2_access_paths = ["[openshell.gateway.tls].{cert_path,key_path,client_ca_path,external_cert_path,external_key_path,external_server_names}", "same CLI and environment variables for primary bundle"] +behavioral_oracle = "The listener presents the primary or configured SNI certificate, derives client-certificate requirements from client CA and OIDC presence, rejects the unsupported require_client_auth file field, and rejects incomplete TLS bundles." required_environment = "test CA, primary and external certificates, TLS client" test_lane = "e2e-docker" status = "not_run" @@ -344,7 +344,7 @@ id = "external-compute-driver-socket" topics = ["external_drivers"] origin_main_access_paths = ["--drivers / --driver / OPENSHELL_DRIVERS plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.] remote socket table"] schema_v2_access_paths = ["--compute-driver / OPENSHELL_COMPUTE_DRIVER plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.].socket_path"] -behavioral_oracle = "A selected non-reserved external driver connects through its configured Unix socket; legacy plural selector flags and environment variables are rejected." +behavioral_oracle = "A selected non-reserved external driver connects through its configured Unix socket; legacy plural selector flags are rejected, while one OPENSHELL_DRIVERS environment value remains a deprecated upgrade alias when it does not conflict with canonical selection." required_environment = "external compute-driver gRPC fixture over Unix socket" test_lane = "extension-driver" status = "not_run" diff --git a/e2e/parity/gateway-options.sh b/e2e/parity/gateway-options.sh new file mode 100755 index 0000000000..f83b4dd5b6 --- /dev/null +++ b/e2e/parity/gateway-options.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Paired process-level checks for gateway-wide options that do not require a +# sandbox. The caller supplies immutable baseline and candidate gateway builds. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BASELINE_GATEWAY="${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" +CANDIDATE_GATEWAY="${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" +RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-${ROOT}/target/parity/gateway-options}" +BASELINE_SHA="74960ebfaeec4673885089ed995fad902459749f" +CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-gateway-options.XXXXXX")" +PIDS=() + +cleanup() { + local status=$? pid + for pid in "${PIDS[@]}"; do + kill -INT "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + done + rm -rf "${WORKDIR}" + exit "${status}" +} +trap cleanup EXIT + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +for binary in "${BASELINE_GATEWAY}" "${CANDIDATE_GATEWAY}"; do + [ -x "${binary}" ] || fail "gateway binary is not executable: ${binary:-}" +done +command -v curl >/dev/null 2>&1 || fail "curl is required" +command -v podman >/dev/null 2>&1 || fail "podman is required" +podman info >/dev/null 2>&1 || fail "podman service is not reachable" + +PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock}" +[ -S "${PODMAN_SOCKET}" ] || fail "Podman API socket is unavailable: ${PODMAN_SOCKET}" +mkdir -p "${RESULTS_DIR}" + +pick_port() { + python3 - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +} + +write_config() { + local output=$1 schema=$2 variant=$3 file_port=$4 health_port=$5 metrics_port=$6 + cat >"${output}" <>"${output}" + else + printf 'compute_driver = "podman"\n' >>"${output}" + fi + cat >>"${output}" <<'EOF' + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.drivers.podman] +EOF +} + +wait_for_url() { + local pid=$1 url=$2 log=$3 elapsed=0 + while [ "${elapsed}" -lt 100 ]; do + if curl --noproxy '*' --max-time 1 -fsS "${url}" >/dev/null 2>&1; then + return 0 + fi + if ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "=== gateway log ===" >&2 + cat "${log}" >&2 || true + return 1 + fi + sleep 0.1 + elapsed=$((elapsed + 1)) + done + echo "timed out waiting for ${url}" >&2 + cat "${log}" >&2 || true + return 1 +} + +stop_gateway() { + local pid=$1 + kill -INT "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + PIDS=() +} + +run_variant() { + local variant=$1 schema=$2 sha=$3 gateway=$4 + local dir="${WORKDIR}/${variant}" + local config="${dir}/gateway.toml" db="${dir}/gateway.db" log="${dir}/gateway.log" + local file_port env_port primary_port health_port metrics_port second_primary + mkdir -p "${dir}/state" + file_port="$(pick_port)" + env_port="$(pick_port)" + primary_port="$(pick_port)" + health_port="$(pick_port)" + metrics_port="$(pick_port)" + second_primary="$(pick_port)" + write_config "${config}" "${schema}" "${variant}" "${file_port}" "${health_port}" "${metrics_port}" + + echo "==> ${variant}: precedence, listeners, and initial SQLite open" + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + OPENSHELL_SERVER_PORT="${env_port}" \ + "${gateway}" \ + --config "${config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --name "${variant}-cli" \ + --port "${primary_port}" \ + --log-level debug >"${log}" 2>&1 & + local pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${health_port}/healthz" "${log}" \ + || fail "${variant} health listener did not start" + curl --noproxy '*' --max-time 2 -fsS "http://127.0.0.1:${metrics_port}/metrics" >/dev/null \ + || fail "${variant} metrics listener is unavailable" + # A plain HTTP request to the gRPC listener returns 404; successful TCP/HTTP + # exchange is sufficient to prove that the selected primary port is bound. + curl --noproxy '*' --max-time 2 -sS "http://127.0.0.1:${primary_port}/" >/dev/null \ + || fail "${variant} primary listener is unavailable" + if curl --noproxy '*' --max-time 1 -sS "http://127.0.0.1:${file_port}/" >/dev/null 2>&1; then + fail "${variant} file port unexpectedly beat the CLI port" + fi + if curl --noproxy '*' --max-time 1 -sS "http://127.0.0.1:${env_port}/" >/dev/null 2>&1; then + fail "${variant} environment port unexpectedly beat the CLI port" + fi + grep -F "127.0.0.1:${primary_port}" "${log}" >/dev/null \ + || fail "${variant} startup log did not identify the effective primary bind" + [ -s "${db}" ] || fail "${variant} SQLite database was not created" + stop_gateway "${pid}" + + echo "==> ${variant}: SQLite reopen and health-port zero override" + : >"${log}" + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + "${gateway}" \ + --config "${config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --port "${second_primary}" \ + --health-port 0 >"${log}" 2>&1 & + pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${metrics_port}/metrics" "${log}" \ + || fail "${variant} did not reopen its SQLite database" + if curl --noproxy '*' --max-time 1 -fsS "http://127.0.0.1:${health_port}/healthz" >/dev/null 2>&1; then + fail "${variant} health listener remained active after --health-port 0" + fi + stop_gateway "${pid}" + + echo "==> ${variant}: database URL in TOML is rejected without value disclosure" + local invalid="${dir}/invalid.toml" secret="parity-secret-${variant}" + awk -v secret="${secret}" ' + /^name =/ { print "database_url = \"postgres://user:" secret "@127.0.0.1/db\"" } + { print } + ' "${config}" >"${invalid}" + if OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + "${gateway}" --config "${invalid}" >"${dir}/invalid.log" 2>&1; then + fail "${variant} accepted database_url in gateway TOML" + fi + grep -F 'database_url' "${dir}/invalid.log" >/dev/null \ + || fail "${variant} database rejection did not identify the field" + if grep -F "${secret}" "${dir}/invalid.log" >/dev/null; then + fail "${variant} database rejection disclosed the configured secret" + fi + + cat >"${RESULTS_DIR}/gateway-options-${variant}.json" <"${RESULTS_DIR}/gateway-options-comparison.json" <<'EOF' +{"profile":"gateway-options","baseline_success":true,"candidate_success":true,"parity":true} +EOF + +echo "Gateway option parity passed." diff --git a/tasks/parity.toml b/tasks/parity.toml index 30a4baa33f..7c040befd6 100644 --- a/tasks/parity.toml +++ b/tasks/parity.toml @@ -9,3 +9,7 @@ hide = true ["e2e:parity:podman"] description = "Compare frozen schema-v1 and current schema-v2 conformance against Podman (opt-in live test)" run = "bash e2e/parity/run.sh --driver podman" + +["e2e:parity:gateway-options"] +description = "Compare process-level gateway option behavior across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/gateway-options.sh" From eb5dfad820b7f85662754647c2e722508b043bf7 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 12:47:52 -0400 Subject: [PATCH 15/42] test(e2e): record gateway option parity Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-live-results.toml | 32 +++++++++++++++++++ e2e/parity/gateway-options.sh | 25 ++++++++++++++- .../gateway_schema_v2_live_results_test.py | 19 +++++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index f11c758c10..f058cbdb5c 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -63,3 +63,35 @@ status = "platform_blocked" owner = "OpenShell Windows MXC CI lane" lane = "windows-x64-and-windows-arm64-mxc" blocker = "The validation host is Linux and cannot execute the Windows MXC runtime. Step 13 must assign native Windows validation." + +[[result]] +id = "gateway-wide-process-options" +step = 6 +capability = "Gateway listeners, configuration precedence, persistence, and legacy singleton selector" +driver = "gateway" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh binaries from both recorded commits started against isolated schema-v1 and schema-v2 files, SQLite databases, ports, state directories, and a rootless Podman socket.", + "Both variants exposed primary, health, and metrics listeners; the CLI primary port beat conflicting environment and file values, and --health-port 0 disabled the file-configured auxiliary listener.", + "Both variants created and reopened the same isolated SQLite database, rejected database_url in TOML, identified the rejected field, and did not disclose its secret-bearing value.", + "Both variants accepted one legacy OPENSHELL_DRIVERS=podman selector when no canonical selector was present; the candidate emitted its deprecation warning without logging the value.", + "Normalized comparison recorded baseline_success=true, candidate_success=true, and parity=true.", +] + +[[result]] +id = "gateway-tls-client-auth-policy" +step = 6 +capability = "Gateway TLS client-certificate handshake policy" +driver = "gateway" +status = "intentional_change" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea" +lane = "local-linux-x86_64-deterministic-tls" +evidence = [ + "Security review confirmed that origin/main accepted require_client_auth in TOML but ignored it and instead derived runtime policy from client CA and OIDC presence.", + "Schema v2 now rejects the unsupported file field while preserving the established derived policy: CA without OIDC requires a client certificate, while CA with OIDC permits bearer-only clients and validates certificates that are presented.", + "Focused preparation tests cover both derived branches, and TLS integration tests pass required-certificate, optional-certificate, valid-CA, wrong-CA, and multiplexed HTTP/gRPC handshakes.", +] diff --git a/e2e/parity/gateway-options.sh b/e2e/parity/gateway-options.sh index f83b4dd5b6..1a0a98a0d4 100755 --- a/e2e/parity/gateway-options.sh +++ b/e2e/parity/gateway-options.sh @@ -171,6 +171,29 @@ run_variant() { fi stop_gateway "${pid}" + echo "==> ${variant}: legacy singleton environment selector" + local legacy_config="${dir}/legacy-selector.toml" + grep -v '^compute_driver' "${config}" >"${legacy_config}" + : >"${log}" + env -u OPENSHELL_COMPUTE_DRIVER \ + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + OPENSHELL_DRIVERS=podman \ + "${gateway}" \ + --config "${legacy_config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --port "${second_primary}" >"${log}" 2>&1 & + pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${health_port}/healthz" "${log}" \ + || fail "${variant} did not accept the legacy singleton selector" + if [ "${variant}" = candidate ]; then + grep -F 'OPENSHELL_DRIVERS is deprecated' "${log}" >/dev/null \ + || fail "candidate did not emit the legacy selector deprecation warning" + fi + stop_gateway "${pid}" + echo "==> ${variant}: database URL in TOML is rejected without value disclosure" local invalid="${dir}/invalid.toml" secret="parity-secret-${variant}" awk -v secret="${secret}" ' @@ -188,7 +211,7 @@ run_variant() { fi cat >"${RESULTS_DIR}/gateway-options-${variant}.json" < None: } -def test_pass_results_pin_executed_commits_and_evidence() -> None: +def test_executed_results_pin_commits_and_evidence() -> None: manifest = load_toml(RESULTS_PATH) for result in manifest["result"]: - if result["status"] != "pass": + if result["status"] not in {"pass", "intentional_change"}: continue assert set(result) >= PASS_FIELDS, result["id"] @@ -120,6 +124,17 @@ def test_pass_results_pin_executed_commits_and_evidence() -> None: ) +def test_step_6_records_gateway_option_and_tls_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 6 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_6_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["gateway-wide-process-options"] == "pass" + assert statuses["gateway-tls-client-auth-policy"] == "intentional_change" + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": From e49fc3a6428193498f031dd05e1bb0d3d1debb3c Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 12:49:43 -0400 Subject: [PATCH 16/42] docs(config): close gateway-wide parity gaps Signed-off-by: Jesse Jaggars --- .../schema-v2-parity-gap-dispositions.toml | 36 +++++++++---------- ..._schema_v2_parity_gap_dispositions_test.py | 22 +++++++++--- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml index 0b4dd110c3..d0a3a65f3c 100644 --- a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml +++ b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml @@ -11,14 +11,14 @@ candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" [[gaps]] id = "legacy-environment-selector-upgrade" -severity = "blocker" +severity = "none" parity_relation = "regression" -disposition = "must_fix_before_parity" +disposition = "resolved" origin_main_behavior = "OPENSHELL_DRIVERS accepts one driver name and is loaded by RPM, Debian, and Homebrew gateway services through gateway.env." -candidate_behavior = "Any presence of OPENSHELL_DRIVERS aborts startup before the canonical selector is resolved." -impact = "An otherwise valid in-place package upgrade can make the gateway unavailable without changing the operator-owned environment file." -resolution = "Accept one non-empty OPENSHELL_DRIVERS value as a deprecated environment-only alias when OPENSHELL_COMPUTE_DRIVER is absent; reject multiple values and conflicting canonical and legacy values, and emit a migration warning without logging secrets. Keep removed CLI flags rejected." -validation = "Start the candidate through package-style environment loading with one legacy driver, a conflicting canonical driver, multiple legacy drivers, and the canonical replacement; assert selection, diagnostics, and readiness." +candidate_behavior = "Schema v2 accepts one non-empty OPENSHELL_DRIVERS value as a deprecated environment-only alias, rejects ambiguity and conflicts, and keeps removed CLI flags rejected." +impact = "The package-upgrade regression is resolved while preserving singular selector semantics and actionable migration guidance." +resolution = "The gateway now accepts one non-empty OPENSHELL_DRIVERS value, preserves canonical selector precedence, rejects multiple values and conflicting canonical and legacy values, and emits one deprecation warning after tracing initialization without logging the selector." +validation = "Paired frozen-baseline and candidate processes reached readiness through OPENSHELL_DRIVERS=podman; focused tests cover empty, plural, malformed, non-UTF-8, equal canonical, conflicting canonical, and remote-socket cases." owner_step = 6 [[gaps]] @@ -35,26 +35,26 @@ owner_step = 12 [[gaps]] id = "tls-require-client-auth-ignored" -severity = "major" +severity = "none" parity_relation = "preexisting_inaccessible_option" -disposition = "must_fix_or_remove_claim" +disposition = "resolved" origin_main_behavior = "The TOML schema accepts and documents require_client_auth, but runtime derives the value from client CA presence and OIDC instead of using the configured boolean." -candidate_behavior = "Schema v2 retains the field and the same derived runtime behavior, so an explicit value remains silently ignored." -impact = "A security-sensitive option appears configurable but cannot change listener authentication behavior." -resolution = "Represent file-level require_client_auth as an optional value, honor an explicit value with documented precedence and OIDC interaction, or remove the field from the file schema and documentation. Require security review before changing runtime authentication semantics." -validation = "With a client CA, test explicit true, explicit false, omission, and OIDC combinations using TLS clients with and without certificates." +candidate_behavior = "Schema v2 rejects require_client_auth and documents the derived client CA and OIDC policy instead of silently accepting an ineffective security option." +impact = "The misleading security-sensitive configuration claim is removed without weakening CA-only gateways or breaking bearer-only OIDC clients." +resolution = "Security review selected removal over runtime wiring: the file-only TLS schema excludes require_client_auth while the internal runtime field continues enforcing the established client CA and OIDC policy." +validation = "File parsing rejects require_client_auth; preparation tests cover CA-only and CA-plus-OIDC derivation; TLS integration tests cover required, optional, valid-CA, wrong-CA, and no-certificate handshakes." owner_step = 6 [[gaps]] id = "unselected-driver-validation-claim" -severity = "minor" +severity = "none" parity_relation = "documentation_gap" -disposition = "documentation_fix_required" +disposition = "resolved" origin_main_behavior = "Only the selected driver table receives driver-specific deserialization; unknown fields inside an unselected table are ignored." -candidate_behavior = "Schema v2 validates every driver entry is a table but still deserializes only the selected driver, while documentation broadly claims unknown driver fields fail startup." -impact = "Operators may believe a dormant driver configuration was fully validated when only its TOML table shape was checked." -resolution = "Narrow documentation to gateway fields, all driver-table shapes, and selected-driver fields unless registry-level validation for every recognized table is intentionally added." -validation = "Start Podman with an unknown Docker field present, then select Docker with the same field; assert the former is accepted and the latter fails." +candidate_behavior = "Schema v2 validates every driver entry is a table and lazily validates driver-specific fields only when that driver is selected, as the reference now states." +impact = "The documentation now distinguishes table-shape validation from selected-driver deserialization, so it no longer overstates startup guarantees." +resolution = "The gateway configuration reference was narrowed to state that driver-specific fields are validated when their driver is selected." +validation = "Configuration tests establish that every driver entry must be a TOML table and that selected driver tables reject unknown fields; the published reference describes the lazy boundary." owner_step = 6 [[gaps]] diff --git a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py index 1060414208..d02c0ab82b 100644 --- a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py +++ b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py @@ -57,6 +57,7 @@ "must_fix_before_release_gate", "must_fix_or_remove_claim", "no_action", + "resolved", } @@ -121,7 +122,7 @@ def test_non_findings_do_not_require_product_changes() -> None: assert gap["disposition"] == "no_action" -def test_security_sensitive_tls_gap_requires_fix_or_removed_claim() -> None: +def test_security_sensitive_tls_gap_records_reviewed_resolution() -> None: tls_gap = next( gap for gap in load_ledger()["gaps"] @@ -129,8 +130,21 @@ def test_security_sensitive_tls_gap_requires_fix_or_removed_claim() -> None: ) assert tls_gap["parity_relation"] == "preexisting_inaccessible_option" - assert tls_gap["disposition"] == "must_fix_or_remove_claim" - assert "security review" in tls_gap["resolution"] + assert tls_gap["disposition"] == "resolved" + assert "Security review" in tls_gap["resolution"] + assert "rejects require_client_auth" in tls_gap["candidate_behavior"] + + +def test_step_6_gap_dispositions_are_resolved() -> None: + step_6_gaps = [gap for gap in load_ledger()["gaps"] if gap["owner_step"] == 6] + + assert {gap["id"] for gap in step_6_gaps} == { + "legacy-environment-selector-upgrade", + "tls-require-client-auth-ignored", + "unselected-driver-validation-claim", + } + assert all(gap["severity"] == "none" for gap in step_6_gaps) + assert all(gap["disposition"] == "resolved" for gap in step_6_gaps) def test_legacy_environment_resolution_preserves_singular_semantics() -> None: @@ -141,5 +155,5 @@ def test_legacy_environment_resolution_preserves_singular_semantics() -> None: ) assert "one non-empty OPENSHELL_DRIVERS value" in legacy_gap["resolution"] - assert "reject multiple values" in legacy_gap["resolution"] + assert "rejects multiple values" in legacy_gap["resolution"] assert "conflicting canonical and legacy values" in legacy_gap["resolution"] From 4af9e5b0f23812acf4b302fcd4620e258971a3d9 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 15:31:01 -0400 Subject: [PATCH 17/42] fix(podman): apply configured pids limit Signed-off-by: Jesse Jaggars --- .../openshell-driver-podman/src/container.rs | 24 ++++++++++++++----- .../schema-v2-intentional-changes.toml | 10 ++++++++ ...eway_schema_v2_intentional_changes_test.py | 2 ++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 4918c9210a..541cb2f756 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -340,8 +340,15 @@ struct SecretMount { struct ResourceLimits { cpu: CpuLimits, memory: MemoryLimits, - #[serde(rename = "PidsLimit", skip_serializing_if = "Option::is_none")] - pids_limit: Option, + // Podman's libpod API consumes the OCI LinuxResources shape. A Docker-style + // scalar PidsLimit is silently ignored and leaves the runtime default. + #[serde(skip_serializing_if = "Option::is_none")] + pids: Option, +} + +#[derive(Serialize)] +struct PidsLimits { + limit: i64, } #[derive(Serialize)] @@ -658,7 +665,9 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: config.sandbox_pids_limit.map(std::num::NonZeroI64::get), + pids: config + .sandbox_pids_limit + .map(|limit| PidsLimits { limit: limit.get() }), } } @@ -1555,7 +1564,7 @@ mod tests { } #[test] - fn container_spec_applies_cpu_and_memory_limits() { + fn container_spec_applies_resource_limits() { use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, }; @@ -1584,7 +1593,10 @@ mod tests { spec["resource_limits"]["memory"]["limit"].as_u64(), Some(2 * 1024 * 1024 * 1024) ); - assert_eq!(spec["resource_limits"]["PidsLimit"].as_i64(), Some(2048)); + assert_eq!( + spec["resource_limits"]["pids"]["limit"].as_i64(), + Some(2048) + ); } #[test] @@ -1594,7 +1606,7 @@ mod tests { config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); - assert!(spec["resource_limits"].get("PidsLimit").is_none()); + assert!(spec["resource_limits"].get("pids").is_none()); } #[test] diff --git a/e2e/configs/gateway/schema-v2-intentional-changes.toml b/e2e/configs/gateway/schema-v2-intentional-changes.toml index b6d92f29ce..c69a0d59d2 100644 --- a/e2e/configs/gateway/schema-v2-intentional-changes.toml +++ b/e2e/configs/gateway/schema-v2-intentional-changes.toml @@ -109,6 +109,16 @@ rationale = "A typed optional non-zero limit removes ambiguous zero handling and parity_disposition = "intentional_change" validation_capability_ids = ["docker-security-and-provider-configuration", "podman-runtime-security-and-health"] +[[intentional_changes]] +id = "podman-pid-limit-restored" +category = "bug_fix" +origin_main_contract = "Podman accepts a positive sandbox_pids_limit, but serializes it with Docker's PidsLimit shape; libpod ignores that field and applies its default limit of 2048." +schema_v2_contract = "Podman serializes a positive sandbox_pids_limit as OCI resource_limits.pids.limit, and the configured value is applied to the container." +migration = "No configuration migration is required; existing positive values begin taking effect after upgrading." +rationale = "Schema-v2 live validation exposed that the frozen baseline accepted this security control without enforcing it at runtime. Preserving that defect would make configuration parity unsafe." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + [[intentional_changes]] id = "podman-health-zero-sentinel-removed" category = "sentinel_removal" diff --git a/python/openshell/gateway_schema_v2_intentional_changes_test.py b/python/openshell/gateway_schema_v2_intentional_changes_test.py index e604aedc1c..acf1bff3d6 100644 --- a/python/openshell/gateway_schema_v2_intentional_changes_test.py +++ b/python/openshell/gateway_schema_v2_intentional_changes_test.py @@ -40,6 +40,7 @@ "middleware-payload-name-normalized", "package-default-only-auto-migration", "podman-health-zero-sentinel-removed", + "podman-pid-limit-restored", "podman-ssh-socket-rename", "sandbox-pid-zero-sentinel-removed", "schema-version-cutover", @@ -48,6 +49,7 @@ "vm-sandbox-identity-selection", } ALLOWED_CATEGORIES = { + "bug_fix", "cardinality", "default_behavior", "migration_policy", From d03ee9894c9028863ac700214f417fb03bfc27da Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 15:41:39 -0400 Subject: [PATCH 18/42] test(e2e): validate Podman option parity Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-live-results.toml | 36 ++++++ e2e/parity/podman-options.sh | 111 ++++++++++++++++++ e2e/parity/run.sh | 81 +++++++++++-- e2e/parity/test.sh | 60 +++++++++- e2e/support/podman-gateway-config.sh | 41 ++++++- e2e/with-podman-gateway.sh | 7 ++ .../gateway_schema_v2_live_results_test.py | 17 +++ tasks/parity.toml | 4 + 8 files changed, 341 insertions(+), 16 deletions(-) create mode 100755 e2e/parity/podman-options.sh diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index f058cbdb5c..9b3ee9b3d6 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -95,3 +95,39 @@ evidence = [ "Schema v2 now rejects the unsupported file field while preserving the established derived policy: CA without OIDC requires a client certificate, while CA with OIDC permits bearer-only clients and validates certificates that are presented.", "Focused preparation tests cover both derived branches, and TLS integration tests pass required-certificate, optional-certificate, valid-CA, wrong-CA, and multiplexed HTTP/gRPC handshakes.", ] + +[[result]] +id = "docker-driver-option-parity" +step = 7 +capability = "Docker image, callback, pull-policy, resource, security, provider, and mount options" +driver = "docker" +status = "platform_blocked" +owner = "OpenShell Linux Docker CI lane" +lane = "linux-x86_64-docker-with-apparmor-sub-lane" +blocker = "The validation host has neither a Docker CLI nor a Docker daemon, and the Docker driver deliberately rejects the available Libpod socket. A dedicated Docker lane must run paired lifecycle and inspect-based resource, label, mount, callback, and pull-policy checks; candidate-only AppArmor, authenticated proxy, and SPIFFE checks must run on qualifying daemons." + +[[result]] +id = "podman-driver-option-parity" +step = 7 +capability = "Podman image, callback, pull-policy, resource, mount, bootstrap, SSH, and health options" +driver = "podman" +status = "intentional_change" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e09070d4ba0b30f0ac278fbb9938c1520ecff696" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh binaries from both recorded commits ran the same candidate-owned oracle against isolated schema-v1 and schema-v2 gateway processes, state, PKI, sockets, networks, and container stores.", + "Both variants selected the same immutable sandbox image, mapped their schema-specific pull-policy spelling to Podman missing, applied 750m CPU and 384Mi memory limits, preserved managed labels, mounted read-only bind and tmpfs configuration, injected sandbox-token and guest-TLS bootstrap files, used the renamed SSH socket, became healthy at the configured seven-second interval, and completed callback exec.", + "The normalized results were identical except for pids_limit: the frozen baseline accepted 31 but sent Docker's ignored PidsLimit field and received Podman's 2048 default, while the candidate sent OCI resource_limits.pids.limit and applied 31.", + "comparison.json recorded baseline_success=true, candidate_success=true, parity=false, classification=intentional_change, intentional_change_id=podman-pid-limit-restored, and accepted=true.", +] + +[[result]] +id = "podman-qualified-security-option-parity" +step = 7 +capability = "Podman AppArmor, authenticated proxy, SPIFFE, and rootful user-namespace options" +driver = "podman" +status = "platform_blocked" +owner = "OpenShell Podman security matrix" +lane = "linux-podman-apparmor-proxy-spiffe-rootful-userns" +blocker = "The available daemon is rootless and reports AppArmor disabled, and this local run has no authenticated proxy or SPIFFE Workload API fixture. A qualifying rootful matrix must validate those environment-dependent options live; deterministic driver tests remain coverage, not parity evidence." diff --git a/e2e/parity/podman-options.sh b/e2e/parity/podman-options.sh new file mode 100755 index 0000000000..f1894b5875 --- /dev/null +++ b/e2e/parity/podman-options.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Shared candidate-owned oracle for both schema variants. It intentionally +# asserts only stable externally observable Podman semantics. + +CLI="${OPENSHELL_BIN:?OPENSHELL_BIN is required}" +RESULT="${OPENSHELL_PARITY_ORACLE_RESULT:?OPENSHELL_PARITY_ORACLE_RESULT is required}" +VARIANT="${OPENSHELL_PARITY_VARIANT:?OPENSHELL_PARITY_VARIANT is required}" +IMAGE="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" +GATEWAY_LOG="${OPENSHELL_E2E_GATEWAY_LOG:?OPENSHELL_E2E_GATEWAY_LOG is required}" +NAME="po-${VARIANT:0:1}-${RANDOM}" +WORKDIR="${TMPDIR:-/tmp}/openshell-parity-options-${NAME}" +mkdir -p "${WORKDIR}" +CREATED=0 +case "${VARIANT}" in + baseline) EXPECTED_PIDS_LIMIT=2048 ;; + candidate) EXPECTED_PIDS_LIMIT=31 ;; + *) echo "ERROR: podman-options oracle: unknown parity variant ${VARIANT}" >&2; exit 2 ;; +esac + +fail() { echo "ERROR: podman-options oracle: $*" >&2; exit 1; } +json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } +podman_cmd() { + if [ "${OPENSHELL_E2E_CONTAINER_ENGINE_UNSET_XDG_CONFIG_HOME:-0}" = 1 ]; then + env -u XDG_CONFIG_HOME podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + elif [ -n "${OPENSHELL_E2E_CONTAINER_ENGINE_XDG_CONFIG_HOME:-}" ]; then + XDG_CONFIG_HOME="${OPENSHELL_E2E_CONTAINER_ENGINE_XDG_CONFIG_HOME}" podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + else + podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + fi +} +cleanup() { + status=$? + if [ "${CREATED}" = 1 ]; then "${CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true; fi + rm -rf "${WORKDIR}" + exit "${status}" +} +trap cleanup EXIT + +mkdir -p "${WORKDIR}/bind-source" +printf '%s\n' parity-bind-mount >"${WORKDIR}/bind-source/probe" +bind_source="$(json_escape "${WORKDIR}/bind-source")" +DRIVER_CONFIG="{\"podman\":{\"mounts\":[{\"type\":\"bind\",\"source\":\"${bind_source}\",\"target\":\"/tmp/parity-bind\",\"read_only\":true,\"selinux_label\":\"private\"},{\"type\":\"tmpfs\",\"target\":\"/tmp/parity-cache\",\"options\":[\"nosuid\",\"nodev\"],\"size_bytes\":1048576,\"mode\":448}]}}" +"${CLI}" sandbox create --name "${NAME}" --cpu 750m --memory 384Mi \ + --driver-config-json "${DRIVER_CONFIG}" --detach +CREATED=1 +podman_cmd ps -aq --filter label=openshell.managed=true --filter "label=openshell.ai/sandbox-name=${NAME}" > "${WORKDIR}/ids" +env wc -l "${WORKDIR}/ids" | env grep -E "^[[:space:]]*1[[:space:]]" >/dev/null || fail "expected exactly one managed container" + +# Image IDs, names, and inspect attributes are checked but never emitted. +while IFS= read -r id; do + podman_cmd image inspect --format "{{.Id}}" "${IMAGE}" | env sed "s/^sha256://" > "${WORKDIR}/expected-image" + podman_cmd inspect --format "{{.Image}}" "${id}" | env sed "s/^sha256://" > "${WORKDIR}/actual-image" + env cmp -s "${WORKDIR}/expected-image" "${WORKDIR}/actual-image" || fail "selected sandbox image ID differs" + podman_cmd inspect --format "{{index .Config.Labels \"openshell.managed\"}}" "${id}" | env grep -Fx true >/dev/null || fail "managed label missing" + podman_cmd inspect --format "{{index .Config.Labels \"openshell.ai/sandbox-name\"}}" "${id}" | env grep -Fx "${NAME}" >/dev/null || fail "sandbox name label missing" + for label in openshell.ai/sandbox-id openshell.ai/sandbox-workspace; do + podman_cmd inspect --format "{{index .Config.Labels \"${label}\"}}" "${id}" | env grep -Ev "^(|)$" >/dev/null || fail "${label} missing" + done + actual_pids_limit="$(podman_cmd inspect --format "{{.HostConfig.PidsLimit}}" "${id}")" + [ "${actual_pids_limit}" = "${EXPECTED_PIDS_LIMIT}" ] || fail "pids limit is ${actual_pids_limit}, expected ${EXPECTED_PIDS_LIMIT}" + podman_cmd inspect --format "{{.HostConfig.CpuQuota}}" "${id}" | env grep -Fx 75000 >/dev/null || fail "CPU quota is not 750m" + podman_cmd inspect --format "{{.HostConfig.CpuPeriod}}" "${id}" | env grep -Fx 100000 >/dev/null || fail "CPU period is not 100000" + podman_cmd inspect --format "{{.HostConfig.Memory}}" "${id}" | env grep -Fx 402653184 >/dev/null || fail "memory limit is not 384Mi" + podman_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/tmp/parity-bind"}}{{.RW}}{{end}}{{end}}' "${id}" \ + | env grep -Fx false >/dev/null || fail "bind mount is not read-only" + podman_cmd inspect --format "{{.Config.Entrypoint}}" "${id}" | env grep -F /opt/openshell/bin/openshell-sandbox >/dev/null || fail "supervisor entrypoint missing" + podman_cmd inspect --format "{{index .Config.Cmd 0}} {{index .Config.Cmd 1}}" "${id}" | env grep -Fx -- "--workdir /sandbox" >/dev/null || fail "supervisor workdir differs" + podman_cmd inspect --format "{{range .Config.Env}}{{println .}}{{end}}" "${id}" | env grep -Fx OPENSHELL_SSH_SOCKET_PATH=/run/openshell/parity-ssh.sock >/dev/null || fail "SSH environment differs" + podman_cmd inspect --format "{{range .Config.Env}}{{println .}}{{end}}" "${id}" | env grep -E "^OPENSHELL_ENDPOINT=https://host\.containers\.internal:" >/dev/null || fail "callback endpoint differs" +done < "${WORKDIR}/ids" +# Both schema spellings must map to Podman's pull-if-missing request. Inspect +# the driver emission so regressions to always or never do not pass merely +# because the wrapper preloaded the image. +sed $'s/\033\[[0-9;]*m//g' "${GATEWAY_LOG}" \ + | env grep -F 'Ensuring sandbox image' \ + | env grep -F 'policy=missing' >/dev/null \ + || fail "image pull policy did not map to Podman missing" + +# Podman 5.8 reports Healthcheck.Interval in nanoseconds; wait for the +# eventual state instead of accepting a merely running container. +healthy=0 +for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90; do + while IFS= read -r id; do + podman_cmd inspect --format "{{.Config.Healthcheck.Interval}}" "${id}" | env grep -E "^(7000000000|7s)$" >/dev/null || fail "health interval is not 7 seconds" + if podman_cmd inspect --format "{{.State.Health.Status}}" "${id}" | env grep -Fx healthy >/dev/null; then healthy=1; fi + done < "${WORKDIR}/ids" + [ "${healthy}" = 1 ] && break + sleep 1 +done +[ "${healthy}" = 1 ] || fail "container did not become healthy" +container_id="$(cat "${WORKDIR}/ids")" +podman_cmd exec "${container_id}" sh -c 'test "$(cat /tmp/parity-bind/probe)" = parity-bind-mount' \ + || fail "read-only bind mount is unavailable" +podman_cmd exec "${container_id}" test -d /tmp/parity-cache \ + || fail "tmpfs mount is unavailable" +podman_cmd exec "${container_id}" test -s /etc/openshell/auth/sandbox.jwt \ + || fail "sandbox token mount is unavailable" +for tls_file in ca.crt tls.crt tls.key; do + podman_cmd exec "${container_id}" test -s "/etc/openshell/tls/client/${tls_file}" \ + || fail "guest TLS mount ${tls_file} is unavailable" +done +"${CLI}" sandbox exec --name "${NAME}" --no-tty --no-login-shell -- true + +# This is the normalized result: no container IDs, timestamps, IPs, or ports. +escaped_image="$(json_escape "${IMAGE}")" +printf "%s\n" "{\"scenario\":\"podman-options\",\"sandbox_image\":\"${escaped_image}\",\"image_pull_policy\":\"if_not_present\",\"managed_labels\":true,\"supervisor_entrypoint\":\"/opt/openshell/bin/openshell-sandbox\",\"supervisor_workdir\":\"/sandbox\",\"callback_endpoint_scheme\":\"https\",\"callback_endpoint_host\":\"host.containers.internal\",\"ssh_socket_path\":\"/run/openshell/parity-ssh.sock\",\"cpu_millis\":750,\"memory_bytes\":402653184,\"pids_limit\":${actual_pids_limit},\"bind_mount\":\"read_only\",\"tmpfs_mount\":true,\"sandbox_token_mount\":true,\"guest_tls_mounts\":true,\"health_check_interval_secs\":7,\"health\":\"healthy\",\"callback_exec\":true}" > "${RESULT}" diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 8dec1ef6d7..9c1fed7499 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -11,16 +11,19 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" MANIFEST="${OPENSHELL_PARITY_CAPABILITY_MANIFEST:-${ROOT}/e2e/configs/gateway/schema-v2-capability-parity.toml}" DRIVER="" +SCENARIO="smoke" +COMMAND_CLASS="conformance_smoke" BASELINE_WORKTREE="${OPENSHELL_PARITY_BASELINE_WORKTREE:-}" RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" +PODMAN_OPTIONS_ORACLE="${OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE:-${ROOT}/e2e/parity/podman-options.sh}" PODMAN_BIN="${OPENSHELL_PARITY_PODMAN_BIN:-podman}" TEMP_WORKTREE="" RUN_DIR="" usage() { cat >&2 <&2; exit 2; } + SCENARIO=$2 + shift 2 + ;; --baseline-worktree) [ "$#" -ge 2 ] || { echo "ERROR: --baseline-worktree requires a path." >&2; exit 2; } BASELINE_WORKTREE=$2 @@ -52,6 +60,12 @@ while [ "$#" -gt 0 ]; do esac done +case "${SCENARIO}" in + smoke) COMMAND_CLASS="conformance_smoke" ;; + podman-options) COMMAND_CLASS="podman_options" ;; + *) echo "ERROR: unsupported parity scenario: ${SCENARIO}." >&2; exit 2 ;; +esac + if [ "${DRIVER}" != "podman" ]; then echo "ERROR: only --driver podman is supported by the schema parity harness (got ${DRIVER:-})." >&2 echo " Docker, Kubernetes, and VM backends are reserved for later parity waves." >&2 @@ -162,21 +176,53 @@ CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE build_variant candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE require_executable "Podman parity wrapper" "${WRAPPER}" +if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; then + echo "ERROR: Podman options oracle does not exist: ${PODMAN_OPTIONS_ORACLE}" >&2 + exit 2 +fi write_result() { local variant=$1 source_sha=$2 schema=$3 status=$4 + local normalized_result="" + if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" <"${RUN_DIR}/baseline.semantic" + sed -E 's/"pids_limit":[0-9]+/"pids_limit":IGNORED/' "${RESULTS_DIR}/candidate.normalized.json" >"${RUN_DIR}/candidate.semantic" + if grep -F '"pids_limit":2048' "${RESULTS_DIR}/baseline.normalized.json" >/dev/null \ + && grep -F '"pids_limit":31' "${RESULTS_DIR}/candidate.normalized.json" >/dev/null \ + && cmp -s "${RUN_DIR}/baseline.semantic" "${RUN_DIR}/candidate.semantic"; then + COMPARISON_ACCEPTED=true + COMPARISON_CLASSIFICATION="intentional_change" + intentional_change_id='"podman-pid-limit-restored"' + fi + fi fi cat >"${RESULTS_DIR}/comparison.json" < schema parity ${variant} (schema v${schema}, ${DRIVER})" + echo "==> schema parity ${variant} (schema v${schema}, ${DRIVER}, ${SCENARIO})" + local option_profile="" + local -a command + if [ "${SCENARIO}" = "podman-options" ]; then + option_profile="podman-options" + command=(bash "${PODMAN_OPTIONS_ORACLE}") + else + command=("${conformance}" run --openshell-bin "${cli}" --output json) + fi if env \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ + OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ + OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ + OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE="${RESULTS_DIR}/${variant}.gateway.toml" \ OPENSHELL_GATEWAY_BIN="${gateway}" \ OPENSHELL_BIN="${cli}" \ OPENSHELL_CONFORMANCE_BIN="${conformance}" \ @@ -196,7 +253,7 @@ run_variant() { XDG_STATE_HOME="${variant_home}/state" \ XDG_CACHE_HOME="${variant_home}/cache" \ XDG_DATA_HOME="${variant_home}/data" \ - "${WRAPPER}" "${conformance}" run --openshell-bin "${cli}" --output json \ + "${WRAPPER}" "${command[@]}" \ 2>&1 | tee "${RESULTS_DIR}/${variant}.log"; then result_status=true else @@ -217,8 +274,12 @@ baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) write_comparison "${baseline_success}" "${candidate_success}" -if [ "${baseline_exit}" -ne 0 ] || [ "${candidate_exit}" -ne 0 ]; then - echo "ERROR: schema parity requires both baseline and candidate conformance smoke runs to succeed." >&2 +if [ "${COMPARISON_ACCEPTED}" != true ]; then + echo "ERROR: schema parity comparison classified ${SCENARIO} as a regression." >&2 exit 1 fi -echo "Schema parity passed: baseline schema v1 and candidate schema v2 succeeded." +if [ "${COMPARISON_CLASSIFICATION}" = intentional_change ]; then + echo "Schema parity accepted an intentional change: podman-pid-limit-restored (${SCENARIO})." +else + echo "Schema parity passed: baseline schema v1 and candidate schema v2 succeeded (${SCENARIO})." +fi diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index d5f2c416a8..8932a14c1d 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -35,6 +35,17 @@ assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"' assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"' assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0' # V2 guest TLS is emitted before its driver table; V1 is driver-local. +OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v1-options.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" +OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v2-options.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" +for config in "${WORKDIR}/v1-options.toml" "${WORKDIR}/v2-options.toml"; do + assert_contains "${config}" 'sandbox_pids_limit = 31' + assert_contains "${config}" 'health_check_interval_secs = 7' + assert_not_contains "${config}" 'app_armor_profile = ' +done +assert_contains "${WORKDIR}/v1-options.toml" 'sandbox_ssh_socket_path = "/run/openshell/parity-ssh.sock"' +assert_contains "${WORKDIR}/v2-options.toml" 'ssh_socket_path = "/run/openshell/parity-ssh.sock"' +if OPENSHELL_E2E_PODMAN_OPTION_PROFILE=unknown e2e_podman_option_profile >/dev/null 2>&1; then fail 'unknown option profile unexpectedly accepted'; fi + v1_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v1.toml" | cut -d: -f1)" v1_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v1.toml" | cut -d: -f1)" v2_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v2.toml" | cut -d: -f1)" @@ -63,8 +74,16 @@ mkdir -p "${WORKDIR}/bin" cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF' #!/usr/bin/env bash set -euo pipefail -printf '%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" >>"$OPENSHELL_PARITY_TEST_CALLS" +printf '%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" +if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then + case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac + stable=true + if [ "${OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT:-0}" = 1 ] && [ "${OPENSHELL_PARITY_VARIANT}" = candidate ]; then stable=false; fi + if [ "${OPENSHELL_PARITY_TEST_SKIP_RESULT:-}" != "${OPENSHELL_PARITY_VARIANT}" ]; then + printf '%s\n' "{\"scenario\":\"podman-options\",\"stable\":${stable},\"pids_limit\":${pids}}" > "${OPENSHELL_PARITY_ORACLE_RESULT}" + fi +fi exec "$@" EOF cat >"${WORKDIR}/bin/fake-podman" <<'EOF' @@ -96,6 +115,7 @@ run_harness() { OPENSHELL_PARITY_BASELINE_WORKTREE="${ROOT}" \ OPENSHELL_PARITY_PODMAN_WRAPPER="${WORKDIR}/bin/fake-wrapper" \ OPENSHELL_PARITY_PODMAN_BIN="${WORKDIR}/bin/fake-podman" \ + OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE="${WORKDIR}/bin/fake-conformance" \ OPENSHELL_PARITY_BASELINE_GATEWAY_BIN="${WORKDIR}/bin/baseline-gateway" \ OPENSHELL_PARITY_BASELINE_CLI_BIN="${WORKDIR}/bin/baseline-cli" \ OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ @@ -106,7 +126,7 @@ run_harness() { OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \ MISE_TRUSTED_CONFIG_PATHS= \ - bash "${ROOT}/e2e/parity/run.sh" --driver podman + bash "${ROOT}/e2e/parity/run.sh" --driver podman "$@" } run_harness @@ -126,6 +146,40 @@ assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally n assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' +run_harness --scenario podman-options +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"podman-options"' +assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"podman_options"' +assert_contains "${WORKDIR}/results/baseline.json" '"normalized_result":"baseline.normalized.json"' +assert_contains "${WORKDIR}/results/baseline.normalized.json" '"stable":true' +assert_contains "${WORKDIR}/results/baseline.normalized.json" '"pids_limit":2048' +assert_contains "${WORKDIR}/results/candidate.normalized.json" '"pids_limit":31' +assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' +assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' +assert_contains "${WORKDIR}/results/comparison.json" '"scenario":"podman-options"' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"intentional_change"' +assert_contains "${WORKDIR}/results/comparison.json" '"intentional_change_id":"podman-pid-limit-restored"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":true' + +set +e +OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT=1 run_harness --scenario podman-options >"${WORKDIR}/drift.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false' + +rm -f "${WORKDIR}/results/candidate.normalized.json" +set +e +OPENSHELL_PARITY_TEST_SKIP_RESULT=candidate run_harness --scenario podman-options >"${WORKDIR}/missing-result.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false' + set +e OPENSHELL_PARITY_FAIL_VARIANT=both run_harness >"${WORKDIR}/failure.out" 2>&1 status=$? @@ -134,7 +188,7 @@ assert_status "${status}" 1 assert_contains "${WORKDIR}/results/baseline.json" '"success":false' assert_contains "${WORKDIR}/results/candidate.json" '"success":false' assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' -[ "$(wc -l <"${WORKDIR}/calls")" -eq 4 ] || fail 'candidate did not run after baseline failure' +[ "$(wc -l <"${WORKDIR}/calls")" -eq 10 ] || fail 'candidate did not run after baseline failure' set +e bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1 diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index 410a5fec48..b4d013e533 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -25,6 +25,22 @@ e2e_podman_toml_string() { printf '"%s"' "${value}" } +# Return the explicitly selected behavioral profile. Keep the default empty so +# ordinary smoke coverage continues to exercise the minimal configuration. +e2e_podman_option_profile() { + case "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" in + "") printf '%s\n' "" ;; + podman-options) printf '%s\n' "podman-options" ;; + *) + echo "ERROR: unsupported OPENSHELL_E2E_PODMAN_OPTION_PROFILE: ${OPENSHELL_E2E_PODMAN_OPTION_PROFILE}" >&2 + return 2 + ;; + esac +} + +# Frozen baseline 74960ebfaeec4673885089ed995fad902459749f does not accept +# Podman app_armor_profile, so it is deliberately not part of this paired profile. + # Write the minimally configured Podman e2e gateway TOML. The schema-v1 # branch deliberately uses the frozen-main contract: list driver selection, # driver-local guest TLS, the old "missing" pull-policy spelling, and zero to @@ -48,7 +64,12 @@ e2e_write_podman_gateway_config() { local podman_socket=${15} local oidc_mode=${16} local oidc_issuer=${17} - local configured_with_tls + local configured_with_tls option_profile + + case "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" in + ""|podman-options) option_profile="${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" ;; + *) echo "ERROR: unsupported OPENSHELL_E2E_PODMAN_OPTION_PROFILE: ${OPENSHELL_E2E_PODMAN_OPTION_PROFILE}" >&2; return 2 ;; + esac case "${schema_version}" in 1) @@ -69,8 +90,15 @@ e2e_write_podman_gateway_config() { printf 'gateway_port = %s\n' "${gateway_port}" printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" printf 'image_pull_policy = "missing"\n' - # In schema v1, zero explicitly disables Podman health checks. - printf 'health_check_interval_secs = 0\n' + if [ "${option_profile}" = "podman-options" ]; then + printf 'sandbox_pids_limit = 31\n' + printf 'health_check_interval_secs = 7\n' + + printf 'sandbox_ssh_socket_path = "/run/openshell/parity-ssh.sock"\n' + else + # In schema v1, zero explicitly disables Podman health checks. + printf 'health_check_interval_secs = 0\n' + fi printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" printf 'guest_tls_ca = %s\n' "$(e2e_podman_toml_string "${pki_dir}/ca.crt")" @@ -88,6 +116,9 @@ e2e_write_podman_gateway_config() { ;; 2) cp "${root}/deploy/rpm/gateway.toml.default" "${output}" + if [ "${option_profile}" = "podman-options" ]; then + sed -i 's/^health_check_interval_secs = .*/health_check_interval_secs = 7/' "${output}" + fi # The v2 template opens the Podman table. Insert gateway-owned TLS # before it rather than reopening [openshell.gateway] later. configured_with_tls="${output}.tls" @@ -108,6 +139,10 @@ e2e_write_podman_gateway_config() { printf 'gateway_port = %s\n' "${gateway_port}" printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" printf 'image_pull_policy = "if_not_present"\n' + if [ "${option_profile}" = "podman-options" ]; then + printf 'sandbox_pids_limit = 31\n' + printf 'ssh_socket_path = "/run/openshell/parity-ssh.sock"\n' + fi printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" printf 'enable_bind_mounts = true\n' diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 1d2176bc97..1f2aaaa730 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -102,6 +102,7 @@ GATEWAY_BIN="" CLI_BIN="" GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" +export OPENSHELL_E2E_GATEWAY_LOG="${GATEWAY_LOG}" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" DRIVER_BIN="" @@ -375,6 +376,9 @@ fi # Validate the generated configuration dialect before creating runtime resources. CONFIG_SCHEMA_VERSION="$(e2e_podman_config_schema_version)" +# Validate the opt-in profile before building images or allocating runtime resources. +e2e_podman_option_profile >/dev/null + # Preflight for managed Podman gateway mode. if ! command -v podman >/dev/null 2>&1; then echo "ERROR: podman CLI is required to run Podman-backed e2e tests" >&2 @@ -461,6 +465,9 @@ e2e_write_podman_gateway_config \ "${OPENSHELL_PODMAN_SOCKET:-}" \ "${OIDC_MODE}" \ "${OPENSHELL_OIDC_ISSUER:-}" +if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then + cp "${GATEWAY_CONFIG}" "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE}" +fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index 00013a783c..d7beb5b3c7 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -31,6 +31,11 @@ "gateway-tls-client-auth-policy", "gateway-wide-process-options", } +REQUIRED_STEP_7_IDS = { + "docker-driver-option-parity", + "podman-driver-option-parity", + "podman-qualified-security-option-parity", +} ALLOWED_STATUSES = { "pass", "intentional_change", @@ -135,6 +140,18 @@ def test_step_6_records_gateway_option_and_tls_dispositions() -> None: assert statuses["gateway-tls-client-auth-policy"] == "intentional_change" +def test_step_7_records_driver_option_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 7 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_7_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["podman-driver-option-parity"] == "intentional_change" + assert statuses["docker-driver-option-parity"] == "platform_blocked" + assert statuses["podman-qualified-security-option-parity"] == "platform_blocked" + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": diff --git a/tasks/parity.toml b/tasks/parity.toml index 7c040befd6..1079bce63a 100644 --- a/tasks/parity.toml +++ b/tasks/parity.toml @@ -10,6 +10,10 @@ hide = true description = "Compare frozen schema-v1 and current schema-v2 conformance against Podman (opt-in live test)" run = "bash e2e/parity/run.sh --driver podman" +["e2e:parity:podman-options"] +description = "Compare paired Podman sandbox option semantics across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman --scenario podman-options" + ["e2e:parity:gateway-options"] description = "Compare process-level gateway option behavior across schema v1 and v2 (opt-in live test)" run = "bash e2e/parity/gateway-options.sh" From 44ef38793f3321d848c3ec7972799800f62a330e Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 18:50:35 -0400 Subject: [PATCH 19/42] test(e2e): add Kubernetes option parity harness Signed-off-by: Jesse Jaggars --- e2e/parity/kubernetes-options-test.sh | 54 ++++ e2e/parity/kubernetes-options.sh | 432 ++++++++++++++++++++++++++ tasks/parity.toml | 9 + 3 files changed, 495 insertions(+) create mode 100644 e2e/parity/kubernetes-options-test.sh create mode 100644 e2e/parity/kubernetes-options.sh diff --git a/e2e/parity/kubernetes-options-test.sh b/e2e/parity/kubernetes-options-test.sh new file mode 100644 index 0000000000..4f14687e7a --- /dev/null +++ b/e2e/parity/kubernetes-options-test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT="${ROOT}/e2e/parity/kubernetes-options.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config baseline >"${TMP}/baseline.toml" +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config candidate >"${TMP}/candidate.toml" +python3 -I - "${TMP}/baseline.toml" "${TMP}/candidate.toml" <<'PY' +import sys, tomllib +def check(condition, message): + if not condition: + raise RuntimeError(message) +baseline=tomllib.load(open(sys.argv[1],'rb'))['openshell'] +candidate=tomllib.load(open(sys.argv[2],'rb'))['openshell'] +check(baseline['version']==1 and candidate['version']==2,'schema versions differ') +check(baseline['gateway']['compute_drivers']==['kubernetes'],'baseline selector differs') +check(candidate['gateway']['compute_driver']=='kubernetes','candidate selector differs') +shared={'default_image','supervisor_image','client_tls_secret_name','service_account_name','host_gateway_ip','enable_user_namespaces','sa_token_ttl_secs'} +check(shared <= baseline['gateway'].keys(),'baseline inherited fields missing') +check(shared.isdisjoint(candidate['gateway'].keys()),'candidate leaked driver fields into gateway table') +check(shared <= candidate['drivers']['kubernetes'].keys(),'candidate driver fields missing') +b=dict(baseline['drivers']['kubernetes']); b.update({key:baseline['gateway'][key] for key in shared}) +c=dict(candidate['drivers']['kubernetes']) +for projection in (b,c): + projection['gateway_id']='' + projection['grpc_endpoint']='http://host.openshell.internal:' + for field in ('image_pull_policy','supervisor_image_pull_policy'): + projection[field]={'IfNotPresent':'if_not_present'}.get(projection[field],projection[field]) +check(b==c,'schema-independent Kubernetes option projections differ') +PY + +: >"${TMP}/kubeconfig" +set +e +OPENSHELL_PARITY_BASELINE_ROOT="${TMP}/not-a-worktree" \ +OPENSHELL_PARITY_BASELINE_GATEWAY=/bin/true \ +OPENSHELL_PARITY_CANDIDATE_GATEWAY=/bin/true \ +OPENSHELL_PARITY_CLI=/bin/true \ +OPENSHELL_PARITY_KUBECONFIG="${TMP}/kubeconfig" \ +OPENSHELL_PARITY_KUBE_CONTEXT=default/external-production-cluster \ +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 \ +bash "${SCRIPT}" >"${TMP}/unsafe.out" 2>&1 +status=$? +set -e +[ "${status}" -ne 0 ] +grep -F 'refusing non-parity context' "${TMP}/unsafe.out" >/dev/null + +echo "Kubernetes option parity deterministic tests passed." diff --git a/e2e/parity/kubernetes-options.sh b/e2e/parity/kubernetes-options.sh new file mode 100644 index 0000000000..658e751c3e --- /dev/null +++ b/e2e/parity/kubernetes-options.sh @@ -0,0 +1,432 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compare the frozen schema-v1 and current schema-v2 Kubernetes driver against +# one explicitly supplied, disposable kind cluster. Gateway processes run on +# the host so the same cluster and candidate-owned oracle exercise both schemas. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BASELINE_SHA="${OPENSHELL_PARITY_BASELINE_SHA:-74960ebfaeec4673885089ed995fad902459749f}" +CANDIDATE_SHA="${OPENSHELL_PARITY_CANDIDATE_SHA:-$(git -C "${ROOT}" rev-parse HEAD)}" +BASELINE_ROOT="${OPENSHELL_PARITY_BASELINE_ROOT:-}" +BASELINE_GATEWAY="${OPENSHELL_PARITY_BASELINE_GATEWAY:-}" +CANDIDATE_GATEWAY="${OPENSHELL_PARITY_CANDIDATE_GATEWAY:-}" +CLI="${OPENSHELL_PARITY_CLI:-}" +ARTIFACT_MANIFEST="${OPENSHELL_PARITY_ARTIFACT_MANIFEST:-}" +KUBECONFIG_PATH="${OPENSHELL_PARITY_KUBECONFIG:-}" +KUBE_CONTEXT="${OPENSHELL_PARITY_KUBE_CONTEXT:-}" +HOST_GATEWAY_IP="${OPENSHELL_PARITY_HOST_GATEWAY_IP:-}" +RUN_ID="${OPENSHELL_PARITY_RUN_ID:-$(date +%s)-$$}" +OUT="${OPENSHELL_PARITY_OUTPUT_DIR:-${ROOT}/target/parity/step8-kubernetes-${CANDIDATE_SHA:0:8}}" +SANDBOX_IMAGE="${OPENSHELL_PARITY_KUBERNETES_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +SUPERVISOR_IMAGE="${OPENSHELL_PARITY_KUBERNETES_SUPERVISOR_IMAGE:-ghcr.io/nvidia/openshell/supervisor:latest}" +RUNTIME_CLASS="openshell-parity-runc-${RUN_ID}" + +fail() { + echo "ERROR: Kubernetes option parity: $*" >&2 + exit 1 +} + +kctl() { + kubectl --kubeconfig "${KUBECONFIG_PATH}" --context "${KUBE_CONTEXT}" "$@" +} + +pick_port() { + python3 -I - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("0.0.0.0", 0)) + print(sock.getsockname()[1]) +PY +} + +write_config() { + local variant=$1 + local path=$2 + local namespace=$3 + local port=$4 + local run_dir=$5 + local gateway_id="step8-${variant}-${RUN_ID}" + local pull_policy + + if [ "${variant}" = baseline ]; then + pull_policy=IfNotPresent + cat >"${path}" <"${path}" <}" ;; esac +[[ "${BASELINE_SHA}" =~ ^[0-9a-f]{40}$ ]] || fail "baseline SHA must be a full lowercase SHA-1" +[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]] || fail "candidate SHA must be a full lowercase SHA-1" +[[ "${RUN_ID}" =~ ^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$ ]] || fail "run ID must be a lowercase DNS label of at most 32 characters" +[[ "${SANDBOX_IMAGE}" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@+-]{0,254}$ ]] || fail "sandbox image contains unsafe characters" +[[ "${SUPERVISOR_IMAGE}" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@+-]{0,254}$ ]] || fail "supervisor image contains unsafe characters" +python3 -I - "${HOST_GATEWAY_IP}" <<'PY' +import ipaddress, sys +value=ipaddress.ip_address(sys.argv[1]) +if value.version != 4: + raise SystemExit("host gateway IP must be IPv4") +PY +[ "$(git -C "${BASELINE_ROOT}" rev-parse HEAD)" = "${BASELINE_SHA}" ] || fail "baseline worktree is not ${BASELINE_SHA}" +[ "$(git -C "${ROOT}" rev-parse HEAD)" = "${CANDIDATE_SHA}" ] || fail "candidate worktree is not ${CANDIDATE_SHA}" +[ "$(kubectl --kubeconfig "${KUBECONFIG_PATH}" config current-context)" = "${KUBE_CONTEXT}" ] || fail "private kubeconfig current context differs from requested parity context" +[ "$(kctl -n kube-system get configmap openshell-parity-guard -o jsonpath='{.data.context}')" = "${KUBE_CONTEXT}" ] || fail "cluster lacks the matching provisioning-time parity guard" +[ "$(kctl -n kube-system get configmap openshell-parity-guard -o jsonpath='{.data.purpose}')" = schema-v2-capability-parity ] || fail "cluster parity guard has the wrong purpose" +kctl get nodes -o name | grep -q '^node/openshell-parity-' || fail "requested context is not the dedicated OpenShell parity cluster" +[ "$(kctl get crd sandboxes.agents.x-k8s.io -o jsonpath='{.status.conditions[?(@.type=="Established")].status}')" = True ] || fail "Agent Sandbox CRD is not established" +[ -f "${ARTIFACT_MANIFEST}" ] || fail "OPENSHELL_PARITY_ARTIFACT_MANIFEST is required" +python3 -I - "${ARTIFACT_MANIFEST}" "${BASELINE_SHA}" "${CANDIDATE_SHA}" "${BASELINE_GATEWAY}" "${CANDIDATE_GATEWAY}" "${CLI}" <<'PY' +import hashlib, pathlib, sys, tomllib +manifest=tomllib.load(open(sys.argv[1],'rb')) +expected={'baseline_commit':sys.argv[2],'candidate_commit':sys.argv[3]} +for key, value in expected.items(): + if manifest.get(key) != value: + raise SystemExit(f'artifact manifest {key} does not match') +for key, path in zip(('baseline_gateway_sha256','candidate_gateway_sha256','candidate_cli_sha256'),sys.argv[4:]): + digest=hashlib.sha256(pathlib.Path(path).read_bytes()).hexdigest() + if manifest.get(key) != digest: + raise SystemExit(f'artifact manifest {key} does not match supplied binary') +PY + +PARITY_ROOT="$(realpath -m "${ROOT}/target/parity")" +OUT="$(realpath -m "${OUT}")" +case "${OUT}" in "${PARITY_ROOT}"/step8-kubernetes-*) ;; *) fail "output must be a step8-kubernetes-* directory below ${PARITY_ROOT}" ;; esac +[ ! -L "${OUT}" ] || fail "output directory must not be a symlink" +rm -rf --one-file-system "${OUT}" +umask 077 +mkdir -p "${OUT}/raw" +cp "${ARTIFACT_MANIFEST}" "${OUT}/artifact-manifest.toml" +printf '%s\n' "${BASELINE_SHA}" >"${OUT}/baseline.sha" +printf '%s\n' "${CANDIDATE_SHA}" >"${OUT}/candidate.sha" +printf '%s\n' "${KUBE_CONTEXT}" >"${OUT}/context" + +runtime_class_created=false +cleanup_cluster_fixture() { + local status=$? + local cleanup_status=0 + set +e + if ${runtime_class_created}; then + kctl delete runtimeclass "${RUNTIME_CLASS}" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 + cleanup_status=$? + fi + set -e + if [ "${status}" -eq 0 ] && [ "${cleanup_status}" -ne 0 ]; then + echo "ERROR: failed to confirm RuntimeClass cleanup" >&2 + exit 1 + fi + exit "${status}" +} +trap cleanup_cluster_fixture EXIT +cat </dev/null +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: ${RUNTIME_CLASS} +handler: runc +EOF +runtime_class_created=true + +run_variant() ( + set -euo pipefail + local variant=$1 + local gateway=$2 + local namespace="openshell-parity-${variant}-${RUN_ID}" + local sandbox="k8s-${variant:0:1}-${RUN_ID: -6}" + local resource="default--${sandbox}" + local run_dir="${OUT}/raw/${variant}" + local config="${run_dir}/gateway.toml" + local port + local gateway_pid= + local registered_endpoint + local namespace_created=false + mkdir -p "${run_dir}/jwt" "${run_dir}/xdg-config/openshell/gateways/parity" "${run_dir}/xdg-state" "${run_dir}/xdg-data" + + cleanup_variant() { + local status=$? + local cleanup_status=0 + set +e + if [ -n "${gateway_pid}" ]; then + kill "${gateway_pid}" >/dev/null 2>&1 || true + wait "${gateway_pid}" >/dev/null 2>&1 || true + fi + rm -f "${run_dir}/jwt/signing.pem" "${run_dir}/client.key" + if ${namespace_created}; then + kctl delete namespace "${namespace}" --ignore-not-found --wait=true --timeout=120s >"${run_dir}/namespace-delete.log" 2>&1 + cleanup_status=$? + fi + set -e + if [ "${status}" -eq 0 ] && [ "${cleanup_status}" -ne 0 ]; then + echo "ERROR: ${variant} namespace cleanup was not confirmed" >&2 + exit 1 + fi + exit "${status}" + } + trap cleanup_variant EXIT + + openssl genpkey -algorithm ED25519 -out "${run_dir}/jwt/signing.pem" >/dev/null 2>&1 + openssl pkey -in "${run_dir}/jwt/signing.pem" -pubout -out "${run_dir}/jwt/public.pem" >/dev/null 2>&1 + printf 'step8-%s\n' "${variant}" >"${run_dir}/jwt/kid" + openssl req -x509 -newkey rsa:2048 -nodes -subj "/CN=step8-parity-client" \ + -keyout "${run_dir}/client.key" -out "${run_dir}/client.crt" -days 1 >/dev/null 2>&1 + + kctl create namespace "${namespace}" >"${run_dir}/namespace-create.log" + namespace_created=true + kctl -n "${namespace}" create serviceaccount parity-sandbox >"${run_dir}/service-account.log" + kctl -n "${namespace}" create secret generic parity-pull-secret \ + --type=kubernetes.io/dockerconfigjson --from-literal=.dockerconfigjson='{"auths":{}}' >"${run_dir}/pull-secret.log" + kctl -n "${namespace}" create secret generic parity-client-tls \ + --from-file=ca.crt="${run_dir}/client.crt" \ + --from-file=tls.crt="${run_dir}/client.crt" \ + --from-file=tls.key="${run_dir}/client.key" >"${run_dir}/client-tls-secret.log" + + port="$(pick_port)" + write_config "${variant}" "${config}" "${namespace}" "${port}" "${run_dir}" + KUBECONFIG="${KUBECONFIG_PATH}" \ + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + OPENSHELL_DB_URL="sqlite:${run_dir}/gateway.db" \ + "${gateway}" --config "${config}" >"${run_dir}/gateway.log" 2>&1 & + gateway_pid=$! + listener_ready=false + for _ in $(seq 1 60); do + if ! kill -0 "${gateway_pid}" >/dev/null 2>&1; then + fail "${variant} gateway exited before binding; see ${run_dir}/gateway.log" + fi + if python3 -I - "${port}" <<'PY' +import socket, sys +try: + with socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=.2): + pass +except OSError: + raise SystemExit(1) +PY + then + listener_ready=true + break + fi + sleep 0.5 + done + ${listener_ready} || fail "${variant} gateway did not bind within 30 seconds" + + registered_endpoint="http://127.0.0.1:${port}" + cat >"${run_dir}/xdg-config/openshell/gateways/parity/metadata.json" <"${run_dir}/xdg-config/openshell/active_gateway" + + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 360 "${CLI}" sandbox create --name "${sandbox}" --cpu 250m --memory 128Mi --detach \ + >"${run_dir}/create.log" 2>&1 + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 60 "${CLI}" sandbox exec --name "${sandbox}" --no-tty -- \ + sh -c 'printf step8-kubernetes-exec' >"${run_dir}/exec.log" 2>&1 + grep -q 'step8-kubernetes-exec' "${run_dir}/exec.log" || fail "${variant} callback exec marker missing" + + kctl -n "${namespace}" get sandbox "${resource}" -o json >"${run_dir}/sandbox.json" + kctl -n "${namespace}" get pod "${resource}" -o json >"${run_dir}/pod.json" + kctl -n "${namespace}" get pvc "workspace-${resource}" -o json >"${run_dir}/pvc.json" + + python3 -I - "${run_dir}" "${SANDBOX_IMAGE}" "${SUPERVISOR_IMAGE}" "${HOST_GATEWAY_IP}" "${RUNTIME_CLASS}" <<'PY' +import json, pathlib, sys +def check(condition, message): + if not condition: + raise RuntimeError(message) +run=pathlib.Path(sys.argv[1]); sandbox_image, supervisor_image, host_ip, runtime_class=sys.argv[2:] +pod=json.loads((run/'pod.json').read_text()); pvc=json.loads((run/'pvc.json').read_text()); sb=json.loads((run/'sandbox.json').read_text()) +spec=pod['spec']; agent=next(c for c in spec['containers'] if c['name']=='agent'); env={x['name']:x.get('value','') for x in agent.get('env',[])} +inits={c['name']:c for c in spec.get('initContainers',[])}; install=inits['openshell-supervisor-install'] +vols={v['name']:v for v in spec.get('volumes',[])}; mounts={m['name']:m for m in agent.get('volumeMounts',[])} +hosts={(h, a['ip']) for a in spec.get('hostAliases',[]) for h in a.get('hostnames',[])} +check(pod['status']['phase']=='Running','Pod is not Running') +conditions={c['type']:c['status'] for c in sb.get('status',{}).get('conditions',[])} +check(conditions.get('Ready')=='True','Sandbox Ready condition is not true') +check(agent['image']==sandbox_image and agent['imagePullPolicy']=='IfNotPresent','sandbox image or pull policy differs') +check(install['image']==supervisor_image and install['imagePullPolicy']=='IfNotPresent','supervisor image or pull policy differs') +check([x['name'] for x in spec.get('imagePullSecrets',[])]==['parity-pull-secret'],'image pull Secret differs') +check(spec['serviceAccountName']=='parity-sandbox','ServiceAccount differs') +check(env['OPENSHELL_ENDPOINT'].startswith('http://host.openshell.internal:'),'callback endpoint differs') +check(env['OPENSHELL_SSH_SOCKET_PATH']=='/run/openshell/parity-kubernetes-ssh.sock','SSH socket differs') +check(env['OPENSHELL_SANDBOX_UID']=='1000' and env['OPENSHELL_SANDBOX_GID']=='1000','sandbox identity differs') +check(('host.openshell.internal',host_ip) in hosts and ('host.docker.internal',host_ip) in hosts,'host aliases differ') +check(spec['runtimeClassName']==runtime_class and spec.get('hostUsers',True) is not False,'RuntimeClass or user namespace posture differs') +check(agent['securityContext']['appArmorProfile']['type']=='Unconfined','AppArmor profile differs') +check(agent['resources']['requests']=={'cpu':'250m','memory':'128Mi'},'resource requests differ') +check(agent['resources']['limits']=={'cpu':'250m','memory':'128Mi'},'resource limits differ') +check(vols['openshell-sa-token']['projected']['sources'][0]['serviceAccountToken']['expirationSeconds']==600,'ServiceAccount token TTL differs') +check(vols['openshell-client-tls']['secret']['secretName']=='parity-client-tls','client TLS Secret differs') +check(mounts['openshell-client-tls']['readOnly'] is True,'client TLS mount is not read-only') +check(pvc['status']['phase']=='Bound' and pvc['spec']['storageClassName']=='standard','PVC phase or StorageClass differs') +check(pvc['spec']['resources']['requests']['storage']=='64Mi','PVC storage request differs') +labels=sb['metadata']['labels'] +for key in ('openshell.ai/sandbox-id','openshell.ai/sandbox-name','openshell.ai/sandbox-workspace','openshell.ai/gateway-id','openshell.ai/managed-by'): + check(labels.get(key),f'managed label {key} missing') +observed_sideload='init-container' if 'openshell-supervisor-install' in inits else 'unknown' +observed_topology='combined' if [c['name'] for c in spec['containers']]==['agent'] else 'other' +observed_workspace_mode='shared' if pvc['metadata']['namespace']==pod['metadata']['namespace'] and pvc['metadata']['name'].startswith('workspace-default--') else 'other' +check(observed_sideload=='init-container','supervisor sideload method differs') +check(observed_topology=='combined','supervisor topology differs') +check(observed_workspace_mode=='shared','workspace placement differs') +normalized={ + 'scenario':'kubernetes-core-options','pod_phase':'Running','sandbox_ready':True, + 'sandbox_image':agent['image'],'sandbox_image_pull_policy':agent['imagePullPolicy'], + 'image_pull_secrets':['parity-pull-secret'],'service_account':'parity-sandbox', + 'supervisor_image':install['image'],'supervisor_image_pull_policy':install['imagePullPolicy'], + 'supervisor_sideload_method':observed_sideload,'topology':observed_topology, + 'callback_endpoint_host':'host.openshell.internal','callback_exec':True, + 'ssh_socket_path':env['OPENSHELL_SSH_SOCKET_PATH'],'client_tls_secret':'parity-client-tls', + 'host_gateway_ip':host_ip,'sa_token_ttl_secs':600,'runtime_class_handler':'runc', + 'enable_user_namespaces':False,'app_armor_profile':'Unconfined','sandbox_uid':1000,'sandbox_gid':1000, + 'workspace_mode':observed_workspace_mode,'workspace_storage':'64Mi','workspace_storage_class':'standard','pvc_phase':'Bound', + 'cpu':'250m','memory':'128Mi','managed_labels':True, +} +(run.parent.parent/f'{run.name}.normalized.json').write_text(json.dumps(normalized,sort_keys=True,separators=(',',':'))+'\n') +PY + + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 60 "${CLI}" sandbox delete "${sandbox}" >"${run_dir}/delete.log" 2>&1 + for _ in $(seq 1 60); do + if ! kctl -n "${namespace}" get sandbox "${resource}" >/dev/null 2>&1 \ + && ! kctl -n "${namespace}" get pod "${resource}" >/dev/null 2>&1 \ + && ! kctl -n "${namespace}" get pvc "workspace-${resource}" >/dev/null 2>&1; then + break + fi + sleep 1 + done + ! kctl -n "${namespace}" get sandbox "${resource}" >/dev/null 2>&1 || fail "${variant} Sandbox remained after delete" + ! kctl -n "${namespace}" get pod "${resource}" >/dev/null 2>&1 || fail "${variant} Pod remained after delete" + ! kctl -n "${namespace}" get pvc "workspace-${resource}" >/dev/null 2>&1 || fail "${variant} PVC remained after delete" +) + +set +e +run_variant baseline "${BASELINE_GATEWAY}" +baseline_status=$? +run_variant candidate "${CANDIDATE_GATEWAY}" +candidate_status=$? +set -e + +baseline_success=false; candidate_success=false +[ "${baseline_status}" -eq 0 ] && baseline_success=true +[ "${candidate_status}" -eq 0 ] && candidate_success=true +parity=false; classification=regression; accepted=false +if ${baseline_success} && ${candidate_success} && [ -f "${OUT}/baseline.normalized.json" ] && [ -f "${OUT}/candidate.normalized.json" ]; then + if cmp -s "${OUT}/baseline.normalized.json" "${OUT}/candidate.normalized.json"; then + parity=true; classification=pass; accepted=true + fi +fi +cat >"${OUT}/comparison.json" < Date: Thu, 3 Sep 2026 19:03:09 -0400 Subject: [PATCH 20/42] test(e2e): record Kubernetes option parity Signed-off-by: Jesse Jaggars --- .../schema-v2-kubernetes-core-comparison.json | 9 ++ ...ema-v2-kubernetes-option-dispositions.toml | 152 ++++++++++++++++++ .../gateway/schema-v2-live-results.toml | 26 +++ ..._v2_kubernetes_option_dispositions_test.py | 139 ++++++++++++++++ .../gateway_schema_v2_live_results_test.py | 15 ++ 5 files changed, 341 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json create mode 100644 e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml create mode 100644 python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py diff --git a/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json b/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json new file mode 100644 index 0000000000..344343adf1 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json @@ -0,0 +1,9 @@ +{ + "accepted": true, + "baseline_commit": "74960ebfaeec4673885089ed995fad902459749f", + "baseline_success": true, + "candidate_commit": "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd", + "candidate_success": true, + "classification": "pass", + "parity": true +} diff --git a/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml b/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml new file mode 100644 index 0000000000..75e2f35b03 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Step 8 records field-level Kubernetes evidence separately from the broader +# capability manifest. A core pass means both frozen schema v1 and schema v2 +# successfully produced and executed the same observed Kubernetes resources. +# A platform-blocked field or value still requires its assigned qualifying lane. +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd" +core_comparison = "e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json" + +[[coverage]] +id = "shared-combined-core" +status = "pass" +lane = "kind-v1.36.1-rootless-podman-host-gateway" +fields = [ + "namespace", + "default_image", + "image_pull_policy", + "image_pull_secrets", + "service_account_name", + "supervisor_image", + "supervisor_image_pull_policy", + "grpc_endpoint", + "ssh_socket_path", + "client_tls_secret_name", + "host_gateway_ip", + "sa_token_ttl_secs", + "workspace_mode", + "gateway_id", + "workspace_default_storage_size", + "workspace_storage_class", + "default_runtime_class_name", + "supervisor_sideload_method", + "topology", + "app_armor_profile", + "sandbox_uid", + "sandbox_gid", +] +evidence = [ + "Both variants reached Ready through ServiceAccount-token exchange and gateway-minted JWT authentication, then completed callback exec.", + "The API oracle matched images, Kubernetes pull-policy spelling, pull Secret, ServiceAccount, callback and SSH environment, client TLS mount, host aliases, 600-second token projection, runc RuntimeClass, Unconfined AppArmor request, UID/GID environment, CPU/memory, managed metadata, Bound 64Mi standard PVC, and deletion.", + "The normalized baseline and candidate results are byte-identical; comparison.json records both successes, parity=true, classification=pass, and accepted=true.", +] + +[[coverage]] +id = "operator-namespace-discovery" +status = "platform_blocked" +owner = "OpenShell Kubernetes workspace-mode CI" +lane = "managed-and-operator-workspace-matrix" +fields = ["operator_namespace_label", "operator_namespace_file"] +requirement = "Run paired managed and operator workspace lifecycles with isolated namespace RBAC, label discovery, allowlist hot reload, positive placement, and negative rejection probes." + +[[coverage]] +id = "managed-ssh-ingress" +status = "platform_blocked" +owner = "OpenShell Kubernetes network-policy CI" +lane = "managed-workspace-network-policy" +fields = [ + "managed_ssh_ingress.enabled", + "managed_ssh_ingress.gateway_namespace", + "managed_ssh_ingress.gateway_pod_selector", +] +requirement = "Run paired managed-workspace creation and inspect the generated NetworkPolicy while proving the configured gateway selector can connect and a nonmatching pod cannot." + +[[coverage]] +id = "sidecar-topology" +status = "platform_blocked" +owner = "OpenShell Kubernetes sidecar CI" +lane = "kubernetes-sidecar-network-enforcement" +fields = ["sidecar.proxy_uid", "sidecar.process_binary_aware_network_policy"] +requirement = "Run paired sidecar topology with relaxed and process-aware policy profiles and verify pod security context, proxy UID, network init, and enforcement behavior." + +[[coverage]] +id = "authenticated-upstream-proxy" +status = "platform_blocked" +owner = "OpenShell Kubernetes proxy CI" +lane = "kubernetes-authenticated-connect-proxy" +fields = [ + "https_proxy", + "no_proxy", + "proxy_auth_secret_name", + "proxy_auth_secret_key", + "proxy_auth_allow_insecure", + "proxy_connect_by_hostname", +] +requirement = "Provide an authenticated HTTP CONNECT proxy and Secret, then run paired allow, deny, no-proxy, credential, insecure-opt-in, and hostname-resolution probes." + +[[coverage]] +id = "spiffe-workload-api" +status = "platform_blocked" +owner = "OpenShell SPIRE Kubernetes CI" +lane = "kubernetes-spire-csi" +fields = ["provider_spiffe_workload_api_socket_path"] +requirement = "Provide SPIRE and its CSI driver, then run paired gateway and sandbox JWT-SVID acquisition from the allowed Workload API socket path and reject a disallowed path." + +[[coverage]] +id = "user-namespace-isolation" +status = "platform_blocked" +owner = "OpenShell Kubernetes userns CI" +lane = "kubernetes-userns-supported-runtime" +fields = ["enable_user_namespaces"] +requirement = "Run paired hostUsers=false lifecycle and host-ID mapping probes on a node whose kernel, kubelet, and runtime support Kubernetes user namespaces." + +[[qualified_value]] +id = "managed-and-operator-workspace-values" +field = "workspace_mode" +status = "platform_blocked" +owner = "OpenShell Kubernetes workspace-mode CI" +lane = "managed-and-operator-workspace-matrix" +requirement = "The local core pass exercised shared mode; managed namespace creation and operator placement/rejection remain assigned live checks." + +[[qualified_value]] +id = "image-volume-sideload" +field = "supervisor_sideload_method" +status = "platform_blocked" +owner = "OpenShell Kubernetes version matrix" +lane = "kubernetes-image-volume" +requirement = "The local core pass exercised init-container; run paired image-volume lifecycle on a cluster with the ImageVolume feature enabled." + +[[qualified_value]] +id = "sidecar-topology-value" +field = "topology" +status = "platform_blocked" +owner = "OpenShell Kubernetes sidecar CI" +lane = "kubernetes-sidecar-network-enforcement" +requirement = "The local core pass exercised combined topology; sidecar requires its enforcement fixture and positive/negative network probes." + +[[qualified_value]] +id = "apparmor-enforcement" +field = "app_armor_profile" +status = "platform_blocked" +owner = "OpenShell Kubernetes AppArmor CI" +lane = "kubernetes-apparmor-runtime-default-localhost" +requirement = "The local core pass proved Unconfined API projection; an AppArmor-enabled node with an installed localhost profile must prove RuntimeDefault and Localhost enforcement." + +[[qualified_value]] +id = "runtime-class-isolation" +field = "default_runtime_class_name" +status = "platform_blocked" +owner = "OpenShell Kubernetes confidential-runtime CI" +lane = "kubernetes-kata-runtimeclass" +requirement = "The local core pass proved runc RuntimeClass placement; a configured Kata or equivalent runtime must prove isolation-specific lifecycle behavior." + +[[qualified_value]] +id = "gpu-resource-combinations" +field = "create-request.resources.gpu" +status = "platform_blocked" +owner = "OpenShell Kubernetes GPU CI" +lane = "kubernetes-nvidia-device-plugin" +requirement = "Provide NVIDIA GPU nodes and the device plugin, then run paired GPU count/resource-name combinations and verify scheduling, limits, execution, and cleanup." diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index 9b3ee9b3d6..84fff90c05 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -131,3 +131,29 @@ status = "platform_blocked" owner = "OpenShell Podman security matrix" lane = "linux-podman-apparmor-proxy-spiffe-rootful-userns" blocker = "The available daemon is rootless and reports AppArmor disabled, and this local run has no authenticated proxy or SPIFFE Workload API fixture. A qualifying rootful matrix must validate those environment-dependent options live; deterministic driver tests remain coverage, not parity evidence." + +[[result]] +id = "kubernetes-core-option-parity" +step = 8 +capability = "Kubernetes shared workspace, images, placement, bootstrap, security projection, resources, and combined supervisor options" +driver = "kubernetes" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd" +lane = "kind-v1.36.1-rootless-podman-host-gateway" +evidence = [ + "Fresh candidate gateway and CLI binaries were built from the recorded candidate commit and bound by SHA-256, with the frozen baseline binary, in the retained artifact manifest.", + "Both host gateway variants used isolated schema-v1/schema-v2 files, SQLite stores, JWT keys, ports, namespaces, ServiceAccounts, Secrets, and resource names against a guarded disposable kind cluster.", + "Both variants reached Ready through authenticated callback, completed exact-marker exec, and produced matching Sandbox, Pod, and PVC semantics for images and pull policies, pull Secret, ServiceAccount, TLS and token projections, host aliases, SSH endpoint, runc placement, AppArmor request, UID/GID, CPU/memory, shared storage, managed metadata, and deletion.", + "comparison.json recorded baseline_success=true, candidate_success=true, parity=true, classification=pass, and accepted=true; private keys and all per-run Kubernetes fixtures were removed.", +] + +[[result]] +id = "kubernetes-qualified-option-parity" +step = 8 +capability = "Kubernetes managed/operator workspaces, sidecar enforcement, proxy, SPIFFE, user namespaces, AppArmor enforcement, confidential runtimes, and GPUs" +driver = "kubernetes" +status = "platform_blocked" +owner = "OpenShell Kubernetes qualified option matrix" +lane = "kubernetes-managed-operator-sidecar-proxy-spire-userns-apparmor-kata-gpu" +blocker = "The disposable core lane executed shared, combined, init-container, runc, and Unconfined values but did not execute the infrastructure-qualified alternatives. Assigned lanes must run paired managed/operator namespace discovery, sidecar network enforcement, authenticated proxy, SPIRE CSI, hostUsers=false, AppArmor RuntimeDefault/Localhost, image-volume, Kata RuntimeClass, and NVIDIA device-plugin GPU probes." diff --git a/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py b/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py new file mode 100644 index 0000000000..deb413b0da --- /dev/null +++ b/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate field-level Step 8 Kubernetes parity dispositions.""" + +import json +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +LEDGER = ( + ROOT + / "e2e" + / "configs" + / "gateway" + / "schema-v2-kubernetes-option-dispositions.toml" +) +CAPABILITIES = ROOT / "e2e" / "configs" / "gateway" / "schema-v2-capability-parity.toml" +LIVE_RESULTS = ROOT / "e2e" / "configs" / "gateway" / "schema-v2-live-results.toml" +KUBERNETES_CAPABILITY_IDS = { + "kubernetes-core-placement-and-images", + "kubernetes-workspace-isolation", + "kubernetes-supervisor-topology", + "kubernetes-egress-spiffe-and-security", +} +FIELD_GROUP = re.compile( + r"\[openshell\.(?Pgateway|drivers\.kubernetes(?:\.(?P[a-z_]+))?)\]" + r"\.\{(?P[^}]+)\}" +) + + +def load_ledger() -> dict: + with LEDGER.open("rb") as handle: + return tomllib.load(handle) + + +def manifest_kubernetes_fields() -> set[str]: + with CAPABILITIES.open("rb") as handle: + capabilities = tomllib.load(handle)["capabilities"] + fields: set[str] = set() + for capability in capabilities: + if capability["id"] not in KUBERNETES_CAPABILITY_IDS: + continue + for access_path in [ + *capability["origin_main_access_paths"], + *capability["schema_v2_access_paths"], + ]: + for match in FIELD_GROUP.finditer(access_path): + table = match.group("table") + if table == "gateway" and "inherited by Kubernetes" not in access_path: + continue + prefix = ( + f"{match.group('subtable')}." if match.group("subtable") else "" + ) + fields.update( + f"{prefix}{field.strip()}" + for field in match.group("fields").split(",") + ) + return fields + + +def test_kubernetes_ledger_covers_every_manifest_driver_field_once() -> None: + ledger = load_ledger() + coverage = ledger["coverage"] + observed = [field for entry in coverage for field in entry["fields"]] + + assert set(observed) == manifest_kubernetes_fields() + assert len(observed) == len(set(observed)) + + +def test_kubernetes_core_pass_is_paired_and_qualified_checks_stay_blocked() -> None: + ledger = load_ledger() + core = next(entry for entry in ledger["coverage"] if entry["status"] == "pass") + + assert core["id"] == "shared-combined-core" + assert len(core["evidence"]) >= 3 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert len(ledger["validated_candidate_commit"]) == 40 + + comparison_path = ROOT / ledger["core_comparison"] + comparison = json.loads(comparison_path.read_text()) + assert comparison == { + "accepted": True, + "baseline_commit": ledger["baseline_commit"], + "baseline_success": True, + "candidate_commit": ledger["validated_candidate_commit"], + "candidate_success": True, + "classification": "pass", + "parity": True, + } + with LIVE_RESULTS.open("rb") as handle: + live_results = tomllib.load(handle)["result"] + live_core = next( + result + for result in live_results + if result["id"] == "kubernetes-core-option-parity" + ) + assert live_core["status"] == comparison["classification"] + assert live_core["validated_baseline_commit"] == comparison["baseline_commit"] + assert live_core["validated_candidate_commit"] == comparison["candidate_commit"] + + blocked = [ + entry + for entry in [*ledger["coverage"], *ledger["qualified_value"]] + if entry["status"] == "platform_blocked" + ] + assert blocked + for entry in blocked: + assert entry["owner"] + assert entry["lane"] + assert entry["requirement"] + + +def test_environment_qualified_security_checks_are_not_claimed_by_core() -> None: + ledger = load_ledger() + blocked_fields = { + field + for entry in ledger["coverage"] + if entry["status"] == "platform_blocked" + for field in entry["fields"] + } + qualified = {entry["id"] for entry in ledger["qualified_value"]} + + assert { + "enable_user_namespaces", + "https_proxy", + "proxy_auth_secret_name", + "provider_spiffe_workload_api_socket_path", + "sidecar.proxy_uid", + } <= blocked_fields + assert { + "apparmor-enforcement", + "gpu-resource-combinations", + "image-volume-sideload", + "managed-and-operator-workspace-values", + "runtime-class-isolation", + "sidecar-topology-value", + } <= qualified diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index d7beb5b3c7..12f4e46384 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -36,6 +36,10 @@ "podman-driver-option-parity", "podman-qualified-security-option-parity", } +REQUIRED_STEP_8_IDS = { + "kubernetes-core-option-parity", + "kubernetes-qualified-option-parity", +} ALLOWED_STATUSES = { "pass", "intentional_change", @@ -152,6 +156,17 @@ def test_step_7_records_driver_option_dispositions() -> None: assert statuses["podman-qualified-security-option-parity"] == "platform_blocked" +def test_step_8_records_kubernetes_option_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 8 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_8_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["kubernetes-core-option-parity"] == "pass" + assert statuses["kubernetes-qualified-option-parity"] == "platform_blocked" + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": From 51f69d8003412993390964816d30baad01c21398 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 19:19:13 -0400 Subject: [PATCH 21/42] test(e2e): disposition VM parity lanes Signed-off-by: Jesse Jaggars --- .../gateway/schema-v2-capability-parity.toml | 6 +++--- .../schema-v2-intentional-changes.toml | 6 +++--- .../gateway/schema-v2-live-results.toml | 20 +++++++++++++++++++ .../gateway_schema_v2_live_results_test.py | 14 +++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/e2e/configs/gateway/schema-v2-capability-parity.toml b/e2e/configs/gateway/schema-v2-capability-parity.toml index 650669e9a7..3927f81df1 100644 --- a/e2e/configs/gateway/schema-v2-capability-parity.toml +++ b/e2e/configs/gateway/schema-v2-capability-parity.toml @@ -312,9 +312,9 @@ status = "not_run" [[capabilities]] id = "vm-launch-and-resource-configuration" topics = ["vm"] -origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{openshell_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}"] -schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key}"] -behavioral_oracle = "The gateway finds and launches the VM driver from driver_dir, forwards the renamed callback endpoint, and launches a guest with the selected state, images, resources, and identity." +origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "standalone openshell-driver-vm --openshell-endpoint / OPENSHELL_GRPC_ENDPOINT"] +schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key}", "standalone openshell-driver-vm --grpc-endpoint / OPENSHELL_GRPC_ENDPOINT"] +behavioral_oracle = "The gateway finds and launches the VM driver from driver_dir, forwards the TOML grpc_endpoint through the schema-appropriate standalone-driver flag, and launches a guest with the selected state, images, resources, and identity." required_environment = "Linux libkrun/KVM host, VM driver binary, and OCI image" test_lane = "e2e-vm" status = "not_run" diff --git a/e2e/configs/gateway/schema-v2-intentional-changes.toml b/e2e/configs/gateway/schema-v2-intentional-changes.toml index c69a0d59d2..67b265f3e8 100644 --- a/e2e/configs/gateway/schema-v2-intentional-changes.toml +++ b/e2e/configs/gateway/schema-v2-intentional-changes.toml @@ -72,9 +72,9 @@ validation_capability_ids = ["podman-runtime-security-and-health"] [[intentional_changes]] id = "vm-grpc-endpoint-rename" category = "rename" -origin_main_contract = "The VM driver callback override is named openshell_endpoint." -schema_v2_contract = "The VM driver callback override is named grpc_endpoint." -migration = "Rename [openshell.drivers.vm].openshell_endpoint and the standalone flag to grpc_endpoint and --grpc-endpoint." +origin_main_contract = "The VM gateway TOML callback override is grpc_endpoint, while the spawned standalone driver uses the internal field openshell_endpoint and flag --openshell-endpoint; OPENSHELL_GRPC_ENDPOINT is the environment override." +schema_v2_contract = "The VM gateway TOML callback override remains grpc_endpoint, and the spawned standalone driver now uses the same internal field name plus --grpc-endpoint; OPENSHELL_GRPC_ENDPOINT is unchanged." +migration = "No gateway TOML key changes. Operators invoking openshell-driver-vm directly must replace --openshell-endpoint with --grpc-endpoint." rationale = "All compute drivers use the same name for the gateway gRPC callback endpoint." parity_disposition = "intentional_change" validation_capability_ids = ["vm-launch-and-resource-configuration", "external-compute-driver-socket"] diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index 84fff90c05..5af2a2f2ff 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -157,3 +157,23 @@ status = "platform_blocked" owner = "OpenShell Kubernetes qualified option matrix" lane = "kubernetes-managed-operator-sidecar-proxy-spire-userns-apparmor-kata-gpu" blocker = "The disposable core lane executed shared, combined, init-container, runc, and Unconfined values but did not execute the infrastructure-qualified alternatives. Assigned lanes must run paired managed/operator namespace discovery, sidecar network enforcement, authenticated proxy, SPIRE CSI, hostUsers=false, AppArmor RuntimeDefault/Localhost, image-volume, Kata RuntimeClass, and NVIDIA device-plugin GPU probes." + +[[result]] +id = "vm-launch-and-resource-configuration" +step = 9 +capability = "VM launch, callback, persistent state, images, resources, and guest identity" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM CI lane" +lane = "linux-x86_64-kvm-libkrun" +blocker = "This host has no prepared VM runtime bundle and no frozen-baseline or candidate openshell-driver-vm executable linked to a runnable libkrun environment. Both source trees passed their deterministic VM library suites, but only a paired KVM/libkrun lane can prove Ready, exec, restart recovery, overlay persistence, resource sizing, callback equivalence, and deletion." + +[[result]] +id = "vm-guest-security-and-spiffe" +step = 9 +capability = "VM guest token containment, TLS, owner state, proxy credentials, and SPIFFE opt-in" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM security CI lane" +lane = "linux-x86_64-kvm-libkrun-proxy-spiffe" +blocker = "The guest JWT mode and visibility, private owner-marker and sandbox-state persistence, callback recovery, TLS key permissions, proxy credential containment, and guest-reachable SPIFFE TCP behavior require a booted VM. The assigned lane must add authenticated-proxy and Workload API TCP fixtures to the paired libkrun/KVM run." diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index 12f4e46384..68fe89fa51 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -40,6 +40,10 @@ "kubernetes-core-option-parity", "kubernetes-qualified-option-parity", } +REQUIRED_STEP_9_IDS = { + "vm-guest-security-and-spiffe", + "vm-launch-and-resource-configuration", +} ALLOWED_STATUSES = { "pass", "intentional_change", @@ -167,6 +171,16 @@ def test_step_8_records_kubernetes_option_dispositions() -> None: assert statuses["kubernetes-qualified-option-parity"] == "platform_blocked" +def test_step_9_records_vm_runtime_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 9 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_9_IDS + assert all(result["status"] == "platform_blocked" for result in results) + assert all(result["driver"] == "vm" for result in results) + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": From 31322df7125a6284e6575d949da45ae7f921dfca Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 19:38:58 -0400 Subject: [PATCH 22/42] test(e2e): add external driver parity lane Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 65 ++++++++++++++++++++++++++++++++++++++-------- e2e/parity/test.sh | 35 ++++++++++++++++++++++--- tasks/parity.toml | 4 +++ 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 9c1fed7499..80d21d1b33 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -23,12 +23,13 @@ RUN_DIR="" usage() { cat >&2 <&2; exit 2 ;; esac @@ -142,9 +144,12 @@ require_executable() { build_variant() { local variant=$1 source_root=$2 target_dir=$3 gateway_override=$4 cli_override=$5 conformance_override=$6 local gateway_var=$7 cli_var=$8 conformance_var=$9 - local gateway cli conformance jobs=() + local gateway cli conformance jobs=() gateway_features=() if [ -n "${CARGO_BUILD_JOBS:-}" ]; then jobs=(-j "${CARGO_BUILD_JOBS}"); fi + if [ "${SCENARIO}" = external-driver ]; then + gateway_features=(--no-default-features --features telemetry) + fi target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac gateway="${gateway_override:-${target_dir}/debug/openshell-gateway}" @@ -153,7 +158,7 @@ build_variant() { if [ -z "${gateway_override}" ]; then echo "Building ${variant} gateway in ${target_dir}..." - (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-gateway --bin openshell-gateway) + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-gateway --bin openshell-gateway "${gateway_features[@]}") fi if [ -z "${cli_override}" ]; then echo "Building ${variant} CLI in ${target_dir}..." @@ -175,6 +180,37 @@ BASELINE_GATEWAY="" BASELINE_CLI="" BASELINE_CONFORMANCE="" CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE build_variant candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE + +build_external_driver() { + local variant=$1 source_root=$2 target_dir=$3 override=$4 output_var=$5 + local binary + if [ "${SCENARIO}" != external-driver ]; then + printf -v "${output_var}" '%s' "" + return + fi + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + binary="${override:-${target_dir}/debug/openshell-driver-podman}" + if [ -z "${override}" ]; then + echo "Building ${variant} external Podman driver in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build -p openshell-driver-podman --bin openshell-driver-podman) + fi + require_executable "${variant} external Podman driver" "${binary}" + printf -v "${output_var}" '%s' "${binary}" +} + +BASELINE_EXTERNAL_DRIVER="" CANDIDATE_EXTERNAL_DRIVER="" +build_external_driver baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" BASELINE_EXTERNAL_DRIVER +build_external_driver candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" CANDIDATE_EXTERNAL_DRIVER +if [ "${SCENARIO}" = external-driver ]; then + baseline_external_realpath="$(realpath "${BASELINE_EXTERNAL_DRIVER}")" + candidate_external_realpath="$(realpath "${CANDIDATE_EXTERNAL_DRIVER}")" + if [ "${baseline_external_realpath}" = "${candidate_external_realpath}" ] \ + || [ "${BASELINE_EXTERNAL_DRIVER}" -ef "${CANDIDATE_EXTERNAL_DRIVER}" ]; then + echo "ERROR: external-driver parity requires distinct baseline and candidate driver artifacts." >&2 + exit 2 + fi +fi require_executable "Podman parity wrapper" "${WRAPPER}" if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; then echo "ERROR: Podman options oracle does not exist: ${PODMAN_OPTIONS_ORACLE}" >&2 @@ -182,11 +218,15 @@ if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; fi write_result() { - local variant=$1 source_sha=$2 schema=$3 status=$4 - local normalized_result="" + local variant=$1 source_sha=$2 schema=$3 status=$4 external_driver=$5 + local normalized_result="" external_driver_digest="" gateway_profile="in-tree" + if [ -n "${external_driver}" ]; then + external_driver_digest=",\"external_driver_sha256\":\"$(sha256sum "${external_driver}" | cut -d' ' -f1)\"" + gateway_profile="driver-free" + fi if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" < schema parity ${variant} (schema v${schema}, ${DRIVER}, ${SCENARIO})" @@ -239,9 +279,12 @@ run_variant() { else command=("${conformance}" run --openshell-bin "${cli}" --output json) fi - if env \ + if env -u OPENSHELL_GATEWAY_ENDPOINT -u OPENSHELL_GATEWAY_CONFIG \ + -u OPENSHELL_COMPUTE_DRIVER -u OPENSHELL_COMPUTE_DRIVER_SOCKET -u OPENSHELL_DRIVERS \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ + OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ + OPENSHELL_EXTERNAL_DRIVER_BIN="${external_driver}" \ OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE="${RESULTS_DIR}/${variant}.gateway.toml" \ @@ -259,16 +302,16 @@ run_variant() { else result_status=false fi - write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" + write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${external_driver}" [ "${result_status}" = true ] } baseline_exit=0 candidate_exit=0 -run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" || baseline_exit=$? +run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" || baseline_exit=$? # Do not short-circuit: a candidate result is useful even when the frozen # baseline failed, and two equal failures must never constitute parity. -run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" || candidate_exit=$? +run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" || candidate_exit=$? baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 8932a14c1d..d3c2ada346 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -74,7 +74,10 @@ mkdir -p "${WORKDIR}/bin" cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF' #!/usr/bin/env bash set -euo pipefail -printf '%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" +for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS; do + [ -z "${!variable:-}" ] || exit 23 +done +printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac @@ -102,7 +105,7 @@ if [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = "${OPENSHELL_PARITY_VARIANT:-}" ] || fi printf '{"untrusted":"raw output is intentionally not normalized"}\n' EOF -for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli; do +for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver; do cat >"${WORKDIR}/bin/${artifact}" <<'EOF' #!/usr/bin/env bash exit 0 @@ -122,6 +125,8 @@ run_harness() { OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN="${WORKDIR}/bin/candidate-gateway" \ OPENSHELL_PARITY_CANDIDATE_CLI_BIN="${WORKDIR}/bin/candidate-cli" \ OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN="${WORKDIR}/bin/baseline-driver" \ + OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN="${OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE:-${WORKDIR}/bin/candidate-driver}" \ OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \ OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \ @@ -129,7 +134,12 @@ run_harness() { bash "${ROOT}/e2e/parity/run.sh" --driver podman "$@" } -run_harness +OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:9 \ +OPENSHELL_GATEWAY_CONFIG=/tmp/untrusted.toml \ +OPENSHELL_COMPUTE_DRIVER=wrong \ +OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \ +OPENSHELL_DRIVERS=wrong \ + run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance" assert_contains "${WORKDIR}/calls" "|${ROOT}" @@ -146,6 +156,23 @@ assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally n assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' +run_harness --scenario external-driver +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/bin/baseline-driver" +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/bin/candidate-driver" +assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"external-driver"' +assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"external_driver_conformance_smoke"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_profile":"driver-free"' +assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_sha256"' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"pass"' + +set +e +OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/baseline-driver" \ + run_harness --scenario external-driver >"${WORKDIR}/same-driver.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/same-driver.out" 'requires distinct baseline and candidate driver artifacts' + run_harness --scenario podman-options assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" @@ -188,7 +215,7 @@ assert_status "${status}" 1 assert_contains "${WORKDIR}/results/baseline.json" '"success":false' assert_contains "${WORKDIR}/results/candidate.json" '"success":false' assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' -[ "$(wc -l <"${WORKDIR}/calls")" -eq 10 ] || fail 'candidate did not run after baseline failure' +[ "$(wc -l <"${WORKDIR}/calls")" -eq 12 ] || fail 'candidate did not run after baseline failure' set +e bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1 diff --git a/tasks/parity.toml b/tasks/parity.toml index d6cd0c4363..a44012ac33 100644 --- a/tasks/parity.toml +++ b/tasks/parity.toml @@ -14,6 +14,10 @@ run = "bash e2e/parity/run.sh --driver podman" description = "Compare paired Podman sandbox option semantics across schema v1 and v2 (opt-in live test)" run = "bash e2e/parity/run.sh --driver podman --scenario podman-options" +["e2e:parity:podman-external-driver"] +description = "Compare paired external Podman driver lifecycle semantics across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman --scenario external-driver" + ["e2e:parity:gateway-options"] description = "Compare process-level gateway option behavior across schema v1 and v2 (opt-in live test)" run = "bash e2e/parity/gateway-options.sh" From a7de50e131193781d7755a15beb99e3a934683d0 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 20:05:39 -0400 Subject: [PATCH 23/42] fix(e2e): preserve external driver pull policy Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 10 +++++++--- e2e/parity/test.sh | 8 ++++++++ e2e/support/podman-gateway-config.sh | 8 ++++++++ e2e/with-podman-gateway.sh | 3 ++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 80d21d1b33..77b73559c1 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -218,15 +218,19 @@ if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; fi write_result() { - local variant=$1 source_sha=$2 schema=$3 status=$4 external_driver=$5 + local variant=$1 source_sha=$2 schema=$3 status=$4 gateway=$5 cli=$6 conformance=$7 external_driver=$8 local normalized_result="" external_driver_digest="" gateway_profile="in-tree" + local gateway_digest cli_digest conformance_digest + gateway_digest="$(sha256sum "${gateway}" | cut -d' ' -f1)" + cli_digest="$(sha256sum "${cli}" | cut -d' ' -f1)" + conformance_digest="$(sha256sum "${conformance}" | cut -d' ' -f1)" if [ -n "${external_driver}" ]; then external_driver_digest=",\"external_driver_sha256\":\"$(sha256sum "${external_driver}" | cut -d' ' -f1)\"" gateway_profile="driver-free" fi if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" <&2; return 2 ;; + esac +} + e2e_podman_toml_string() { local value="$1" value="${value//\\/\\\\}" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 1f2aaaa730..699a05d538 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -375,6 +375,7 @@ fi # Validate the generated configuration dialect before creating runtime resources. CONFIG_SCHEMA_VERSION="$(e2e_podman_config_schema_version)" +EXTERNAL_DRIVER_PULL_POLICY="$(e2e_podman_external_driver_pull_policy "${CONFIG_SCHEMA_VERSION}")" # Validate the opt-in profile before building images or allocating runtime resources. e2e_podman_option_profile >/dev/null @@ -473,7 +474,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="if_not_present" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ From 251cf4486aad6b7d43d72547f55e8a5980a18bc6 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 20:52:38 -0400 Subject: [PATCH 24/42] test(e2e): attest parity artifacts and launches Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 43 ++++++++++++++++++++++++++++++++++++-- e2e/parity/test.sh | 19 +++++++++++++++++ e2e/with-podman-gateway.sh | 12 +++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 77b73559c1..449cb2ab84 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -181,6 +181,19 @@ CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE build_variant candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE +BASELINE_GATEWAY_ORIGIN=built_by_harness +BASELINE_CLI_ORIGIN=built_by_harness +BASELINE_CONFORMANCE_ORIGIN=built_by_harness +CANDIDATE_GATEWAY_ORIGIN=built_by_harness +CANDIDATE_CLI_ORIGIN=built_by_harness +CANDIDATE_CONFORMANCE_ORIGIN=built_by_harness +[ -z "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" ] || BASELINE_GATEWAY_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" ] || BASELINE_CLI_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" ] || BASELINE_CONFORMANCE_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" ] || CANDIDATE_GATEWAY_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" ] || CANDIDATE_CLI_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" ] || CANDIDATE_CONFORMANCE_ORIGIN=supplied_override + build_external_driver() { local variant=$1 source_root=$2 target_dir=$3 override=$4 output_var=$5 local binary @@ -200,6 +213,14 @@ build_external_driver() { } BASELINE_EXTERNAL_DRIVER="" CANDIDATE_EXTERNAL_DRIVER="" +BASELINE_EXTERNAL_DRIVER_ORIGIN=not_applicable +CANDIDATE_EXTERNAL_DRIVER_ORIGIN=not_applicable +if [ "${SCENARIO}" = external-driver ]; then + BASELINE_EXTERNAL_DRIVER_ORIGIN=built_by_harness + CANDIDATE_EXTERNAL_DRIVER_ORIGIN=built_by_harness + [ -z "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" ] || BASELINE_EXTERNAL_DRIVER_ORIGIN=supplied_override + [ -z "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" ] || CANDIDATE_EXTERNAL_DRIVER_ORIGIN=supplied_override +fi build_external_driver baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" BASELINE_EXTERNAL_DRIVER build_external_driver candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" CANDIDATE_EXTERNAL_DRIVER if [ "${SCENARIO}" = external-driver ]; then @@ -220,17 +241,30 @@ fi write_result() { local variant=$1 source_sha=$2 schema=$3 status=$4 gateway=$5 cli=$6 conformance=$7 external_driver=$8 local normalized_result="" external_driver_digest="" gateway_profile="in-tree" - local gateway_digest cli_digest conformance_digest + local gateway_digest cli_digest conformance_digest gateway_features=default + local gateway_origin cli_origin conformance_origin external_driver_origin + if [ "${variant}" = baseline ]; then + gateway_origin=${BASELINE_GATEWAY_ORIGIN} + cli_origin=${BASELINE_CLI_ORIGIN} + conformance_origin=${BASELINE_CONFORMANCE_ORIGIN} + external_driver_origin=${BASELINE_EXTERNAL_DRIVER_ORIGIN} + else + gateway_origin=${CANDIDATE_GATEWAY_ORIGIN} + cli_origin=${CANDIDATE_CLI_ORIGIN} + conformance_origin=${CANDIDATE_CONFORMANCE_ORIGIN} + external_driver_origin=${CANDIDATE_EXTERNAL_DRIVER_ORIGIN} + fi gateway_digest="$(sha256sum "${gateway}" | cut -d' ' -f1)" cli_digest="$(sha256sum "${cli}" | cut -d' ' -f1)" conformance_digest="$(sha256sum "${conformance}" | cut -d' ' -f1)" if [ -n "${external_driver}" ]; then external_driver_digest=",\"external_driver_sha256\":\"$(sha256sum "${external_driver}" | cut -d' ' -f1)\"" gateway_profile="driver-free" + gateway_features="--no-default-features --features telemetry" fi if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" <&2 + result_status=false + fi write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${gateway}" "${cli}" "${conformance}" "${external_driver}" [ "${result_status}" = true ] } diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 23a6be6910..b4fd8c0e85 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -79,6 +79,19 @@ for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_CO done printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" +case "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" in + 1) pull_policy=missing ;; + 2) pull_policy=if_not_present ;; +esac +transport=in_tree +external=false +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then + transport=remote_uds + external=true +fi +printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s"}\n' \ + "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" "${external}" "${transport}" "${pull_policy}" \ + >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac stable=true @@ -167,6 +180,12 @@ assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/bin/candidate-driver" assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"external-driver"' assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"external_driver_conformance_smoke"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_profile":"driver-free"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_cargo_features":"--no-default-features --features telemetry"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' +assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"conformance_sha256"' diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 699a05d538..65232f333d 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -469,6 +469,18 @@ e2e_write_podman_gateway_config \ if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then cp "${GATEWAY_CONFIG}" "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE}" fi +if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then + driver_transport=in_tree + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + driver_transport=remote_uds + fi + printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s"}\n' \ + "${CONFIG_SCHEMA_VERSION}" \ + "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ + "${driver_transport}" \ + "${EXTERNAL_DRIVER_PULL_POLICY}" \ + >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" +fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ From 51fe98a03db1edeb21a9e4931087666adfd29516 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 3 Sep 2026 21:32:50 -0400 Subject: [PATCH 25/42] test(e2e): require clean parity build sources Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 449cb2ab84..8e667c2603 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -14,6 +14,7 @@ DRIVER="" SCENARIO="smoke" COMMAND_CLASS="conformance_smoke" BASELINE_WORKTREE="${OPENSHELL_PARITY_BASELINE_WORKTREE:-}" +CANDIDATE_WORKTREE="${OPENSHELL_PARITY_CANDIDATE_WORKTREE:-${ROOT}}" RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" PODMAN_OPTIONS_ORACLE="${OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE:-${ROOT}/e2e/parity/podman-options.sh}" @@ -31,6 +32,7 @@ Overrides: OPENSHELL_PARITY_CANDIDATE_{GATEWAY,CLI,CONFORMANCE}_BIN OPENSHELL_PARITY_{BASELINE,CANDIDATE}_EXTERNAL_DRIVER_BIN OPENSHELL_PARITY_{BASELINE,CANDIDATE}_CARGO_TARGET_DIR + OPENSHELL_PARITY_CANDIDATE_WORKTREE (clean checkout at the current HEAD) EOF } @@ -84,7 +86,16 @@ if ! [[ "${BASELINE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then exit 2 fi BASELINE_SHA="${BASELINE_SHA,,}" -CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +EXPECTED_CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +if [ ! -d "${CANDIDATE_WORKTREE}" ]; then + echo "ERROR: candidate worktree does not exist: ${CANDIDATE_WORKTREE}" >&2 + exit 2 +fi +CANDIDATE_SHA="$(git -C "${CANDIDATE_WORKTREE}" rev-parse HEAD 2>/dev/null || true)" +if [ "${CANDIDATE_SHA}" != "${EXPECTED_CANDIDATE_SHA}" ]; then + echo "ERROR: candidate worktree must be at current commit ${EXPECTED_CANDIDATE_SHA}: ${CANDIDATE_WORKTREE}" >&2 + exit 2 +fi cleanup() { local status=$? @@ -128,6 +139,16 @@ else BASELINE_WORKTREE="${TEMP_WORKTREE}" fi +require_clean_source() { + local variant=$1 source_root=$2 dirty + dirty="$(git -C "${source_root}" status --porcelain=v1 --untracked-files=all)" + if [ -n "${dirty}" ]; then + echo "ERROR: ${variant} source worktree must be clean before building parity artifacts: ${source_root}" >&2 + printf '%s\n' "${dirty}" >&2 + exit 2 + fi +} + RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-run.XXXXXX")" RESULTS_DIR="${RESULTS_DIR:-${ROOT}/target/parity/results}" mkdir -p "${RESULTS_DIR}" @@ -157,14 +178,17 @@ build_variant() { conformance="${conformance_override:-${target_dir}/debug/openshell-conformance}" if [ -z "${gateway_override}" ]; then + require_clean_source "${variant}" "${source_root}" echo "Building ${variant} gateway in ${target_dir}..." (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-gateway --bin openshell-gateway "${gateway_features[@]}") fi if [ -z "${cli_override}" ]; then + require_clean_source "${variant}" "${source_root}" echo "Building ${variant} CLI in ${target_dir}..." (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-cli) fi if [ -z "${conformance_override}" ]; then + require_clean_source "${variant}" "${source_root}" echo "Building ${variant} conformance CLI in ${target_dir}..." (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-conformance-cli) fi @@ -179,7 +203,7 @@ build_variant() { BASELINE_GATEWAY="" BASELINE_CLI="" BASELINE_CONFORMANCE="" CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE -build_variant candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE +build_variant candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE BASELINE_GATEWAY_ORIGIN=built_by_harness BASELINE_CLI_ORIGIN=built_by_harness @@ -205,6 +229,7 @@ build_external_driver() { case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac binary="${override:-${target_dir}/debug/openshell-driver-podman}" if [ -z "${override}" ]; then + require_clean_source "${variant}" "${source_root}" echo "Building ${variant} external Podman driver in ${target_dir}..." (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build -p openshell-driver-podman --bin openshell-driver-podman) fi @@ -222,7 +247,7 @@ if [ "${SCENARIO}" = external-driver ]; then [ -z "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" ] || CANDIDATE_EXTERNAL_DRIVER_ORIGIN=supplied_override fi build_external_driver baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" BASELINE_EXTERNAL_DRIVER -build_external_driver candidate "${ROOT}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" CANDIDATE_EXTERNAL_DRIVER +build_external_driver candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" CANDIDATE_EXTERNAL_DRIVER if [ "${SCENARIO}" = external-driver ]; then baseline_external_realpath="$(realpath "${BASELINE_EXTERNAL_DRIVER}")" candidate_external_realpath="$(realpath "${CANDIDATE_EXTERNAL_DRIVER}")" From 8711fbff9c2a634d3dfb0346b5861412f53c4719 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 06:01:59 -0400 Subject: [PATCH 26/42] test(e2e): bind parity runtime artifacts Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 117 ++++++++++++++++++++++++++++++++----- e2e/parity/test.sh | 40 ++++++++++--- e2e/with-podman-gateway.sh | 57 ++++++++++++++++-- 3 files changed, 186 insertions(+), 28 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 8e667c2603..4099eb1b4f 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -31,6 +31,7 @@ Overrides: OPENSHELL_PARITY_BASELINE_{GATEWAY,CLI,CONFORMANCE}_BIN OPENSHELL_PARITY_CANDIDATE_{GATEWAY,CLI,CONFORMANCE}_BIN OPENSHELL_PARITY_{BASELINE,CANDIDATE}_EXTERNAL_DRIVER_BIN + OPENSHELL_PARITY_{BASELINE,CANDIDATE}_SUPERVISOR_BIN OPENSHELL_PARITY_{BASELINE,CANDIDATE}_CARGO_TARGET_DIR OPENSHELL_PARITY_CANDIDATE_WORKTREE (clean checkout at the current HEAD) EOF @@ -152,6 +153,7 @@ require_clean_source() { RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-run.XXXXXX")" RESULTS_DIR="${RESULTS_DIR:-${ROOT}/target/parity/results}" mkdir -p "${RESULTS_DIR}" +RESULTS_DIR="$(cd "${RESULTS_DIR}" && pwd)" require_executable() { local label=$1 @@ -257,6 +259,71 @@ if [ "${SCENARIO}" = external-driver ]; then exit 2 fi fi + +supervisor_target_triple() { + case "$(uname -sm)" in + "Linux x86_64") printf '%s\n' x86_64-unknown-linux-musl ;; + "Linux aarch64"|"Linux arm64") printf '%s\n' aarch64-unknown-linux-musl ;; + *) echo "ERROR: Podman parity supervisor builds require Linux x86_64 or arm64." >&2; return 2 ;; + esac +} + +build_supervisor() { + local variant=$1 source_root=$2 target_dir=$3 override=$4 output_var=$5 + local binary target jobs=() + target="$(supervisor_target_triple)" + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + binary="${override:-${target_dir}/${target}/release/openshell-sandbox}" + if [ -z "${override}" ]; then + require_clean_source "${variant}" "${source_root}" + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then jobs=(-j "${CARGO_BUILD_JOBS}"); fi + echo "Building ${variant} supervisor in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" --release --target "${target}" -p openshell-sandbox --bin openshell-sandbox) + "${source_root}/tasks/scripts/verify-static-binary.sh" "${binary}" + fi + require_executable "${variant} supervisor" "${binary}" + printf -v "${output_var}" '%s' "${binary}" +} + +BASELINE_SUPERVISOR="" CANDIDATE_SUPERVISOR="" +BASELINE_SUPERVISOR_ORIGIN=built_by_harness +CANDIDATE_SUPERVISOR_ORIGIN=built_by_harness +[ -z "${OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN:-}" ] || BASELINE_SUPERVISOR_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN:-}" ] || CANDIDATE_SUPERVISOR_ORIGIN=supplied_override +build_supervisor baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN:-}" BASELINE_SUPERVISOR +build_supervisor candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN:-}" CANDIDATE_SUPERVISOR + +stage_artifact() { + local variant=$1 role=$2 source=$3 mode=$4 path_var=$5 digest_var=$6 + local destination="${RESULTS_DIR}/artifacts/${variant}/${role}" + mkdir -p "$(dirname "${destination}")" + install -m "${mode}" "${source}" "${destination}" + printf -v "${path_var}" '%s' "${destination}" + printf -v "${digest_var}" '%s' "$(sha256sum "${destination}" | cut -d' ' -f1)" +} + +stage_executable() { + stage_artifact "$1" "$2" "$3" 0555 "$4" "$5" +} + +BASELINE_GATEWAY_DIGEST="" BASELINE_CLI_DIGEST="" BASELINE_CONFORMANCE_DIGEST="" BASELINE_EXTERNAL_DRIVER_DIGEST="" BASELINE_SUPERVISOR_DIGEST="" BASELINE_SUPERVISOR_DOCKERFILE="" BASELINE_SUPERVISOR_DOCKERFILE_DIGEST="" +CANDIDATE_GATEWAY_DIGEST="" CANDIDATE_CLI_DIGEST="" CANDIDATE_CONFORMANCE_DIGEST="" CANDIDATE_EXTERNAL_DRIVER_DIGEST="" CANDIDATE_SUPERVISOR_DIGEST="" CANDIDATE_SUPERVISOR_DOCKERFILE="" CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST="" +stage_executable baseline gateway "${BASELINE_GATEWAY}" BASELINE_GATEWAY BASELINE_GATEWAY_DIGEST +stage_executable baseline cli "${BASELINE_CLI}" BASELINE_CLI BASELINE_CLI_DIGEST +stage_executable baseline conformance "${BASELINE_CONFORMANCE}" BASELINE_CONFORMANCE BASELINE_CONFORMANCE_DIGEST +stage_executable baseline supervisor "${BASELINE_SUPERVISOR}" BASELINE_SUPERVISOR BASELINE_SUPERVISOR_DIGEST +stage_artifact baseline supervisor.Dockerfile "${BASELINE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 BASELINE_SUPERVISOR_DOCKERFILE BASELINE_SUPERVISOR_DOCKERFILE_DIGEST +stage_executable candidate gateway "${CANDIDATE_GATEWAY}" CANDIDATE_GATEWAY CANDIDATE_GATEWAY_DIGEST +stage_executable candidate cli "${CANDIDATE_CLI}" CANDIDATE_CLI CANDIDATE_CLI_DIGEST +stage_executable candidate conformance "${CANDIDATE_CONFORMANCE}" CANDIDATE_CONFORMANCE CANDIDATE_CONFORMANCE_DIGEST +stage_executable candidate supervisor "${CANDIDATE_SUPERVISOR}" CANDIDATE_SUPERVISOR CANDIDATE_SUPERVISOR_DIGEST +stage_artifact candidate supervisor.Dockerfile "${CANDIDATE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 CANDIDATE_SUPERVISOR_DOCKERFILE CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST +if [ "${SCENARIO}" = external-driver ]; then + stage_executable baseline external-driver "${BASELINE_EXTERNAL_DRIVER}" BASELINE_EXTERNAL_DRIVER BASELINE_EXTERNAL_DRIVER_DIGEST + stage_executable candidate external-driver "${CANDIDATE_EXTERNAL_DRIVER}" CANDIDATE_EXTERNAL_DRIVER CANDIDATE_EXTERNAL_DRIVER_DIGEST +fi + require_executable "Podman parity wrapper" "${WRAPPER}" if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; then echo "ERROR: Podman options oracle does not exist: ${PODMAN_OPTIONS_ORACLE}" >&2 @@ -264,32 +331,32 @@ if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; fi write_result() { - local variant=$1 source_sha=$2 schema=$3 status=$4 gateway=$5 cli=$6 conformance=$7 external_driver=$8 + local variant=$1 source_sha=$2 schema=$3 status=$4 + local gateway_digest=$5 cli_digest=$6 conformance_digest=$7 external_driver_digest_value=$8 supervisor_digest=$9 supervisor_dockerfile_digest=${10} local normalized_result="" external_driver_digest="" gateway_profile="in-tree" - local gateway_digest cli_digest conformance_digest gateway_features=default - local gateway_origin cli_origin conformance_origin external_driver_origin + local gateway_features=default + local gateway_origin cli_origin conformance_origin external_driver_origin supervisor_origin if [ "${variant}" = baseline ]; then gateway_origin=${BASELINE_GATEWAY_ORIGIN} cli_origin=${BASELINE_CLI_ORIGIN} conformance_origin=${BASELINE_CONFORMANCE_ORIGIN} external_driver_origin=${BASELINE_EXTERNAL_DRIVER_ORIGIN} + supervisor_origin=${BASELINE_SUPERVISOR_ORIGIN} else gateway_origin=${CANDIDATE_GATEWAY_ORIGIN} cli_origin=${CANDIDATE_CLI_ORIGIN} conformance_origin=${CANDIDATE_CONFORMANCE_ORIGIN} external_driver_origin=${CANDIDATE_EXTERNAL_DRIVER_ORIGIN} + supervisor_origin=${CANDIDATE_SUPERVISOR_ORIGIN} fi - gateway_digest="$(sha256sum "${gateway}" | cut -d' ' -f1)" - cli_digest="$(sha256sum "${cli}" | cut -d' ' -f1)" - conformance_digest="$(sha256sum "${conformance}" | cut -d' ' -f1)" - if [ -n "${external_driver}" ]; then - external_driver_digest=",\"external_driver_sha256\":\"$(sha256sum "${external_driver}" | cut -d' ' -f1)\"" + if [ -n "${external_driver_digest_value}" ]; then + external_driver_digest=",\"external_driver_sha256\":\"${external_driver_digest_value}\"" gateway_profile="driver-free" gateway_features="--no-default-features --features telemetry" fi if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" <&2 + return 1 + fi +} + run_variant() { - local variant=$1 source_sha=$2 schema=$3 gateway=$4 cli=$5 conformance=$6 external_driver=$7 result_status - local variant_home="${RUN_DIR}/${variant}" + local variant=$1 source_sha=$2 schema=$3 gateway=$4 cli=$5 conformance=$6 external_driver=$7 supervisor=$8 supervisor_dockerfile=$9 + local gateway_digest=${10} cli_digest=${11} conformance_digest=${12} external_driver_digest=${13} supervisor_digest=${14} supervisor_dockerfile_digest=${15} + local result_status variant_home="${RUN_DIR}/${variant}" + local supervisor_image="openshell/supervisor:parity-${variant}-${source_sha:0:12}" mkdir -p "${variant_home}/config" "${variant_home}/state" "${variant_home}/cache" "${variant_home}/data" echo "==> schema parity ${variant} (schema v${schema}, ${DRIVER}, ${SCENARIO})" local option_profile="" @@ -348,6 +426,9 @@ run_variant() { OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ OPENSHELL_EXTERNAL_DRIVER_BIN="${external_driver}" \ + OPENSHELL_E2E_SUPERVISOR_BIN="${supervisor}" \ + OPENSHELL_E2E_SUPERVISOR_DOCKERFILE="${supervisor_dockerfile}" \ + OPENSHELL_SUPERVISOR_IMAGE="${supervisor_image}" \ OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE="${RESULTS_DIR}/${variant}.gateway.toml" \ @@ -370,16 +451,24 @@ run_variant() { echo "ERROR: ${variant} launcher did not emit a launch manifest." >&2 result_status=false fi - write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${gateway}" "${cli}" "${conformance}" "${external_driver}" + verify_artifact_digest "${variant} gateway" "${gateway}" "${gateway_digest}" || result_status=false + verify_artifact_digest "${variant} CLI" "${cli}" "${cli_digest}" || result_status=false + verify_artifact_digest "${variant} conformance CLI" "${conformance}" "${conformance_digest}" || result_status=false + verify_artifact_digest "${variant} supervisor" "${supervisor}" "${supervisor_digest}" || result_status=false + verify_artifact_digest "${variant} supervisor Dockerfile" "${supervisor_dockerfile}" "${supervisor_dockerfile_digest}" || result_status=false + if [ -n "${external_driver}" ]; then + verify_artifact_digest "${variant} external driver" "${external_driver}" "${external_driver_digest}" || result_status=false + fi + write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${gateway_digest}" "${cli_digest}" "${conformance_digest}" "${external_driver_digest}" "${supervisor_digest}" "${supervisor_dockerfile_digest}" [ "${result_status}" = true ] } baseline_exit=0 candidate_exit=0 -run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" || baseline_exit=$? +run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" "${BASELINE_SUPERVISOR}" "${BASELINE_SUPERVISOR_DOCKERFILE}" "${BASELINE_GATEWAY_DIGEST}" "${BASELINE_CLI_DIGEST}" "${BASELINE_CONFORMANCE_DIGEST}" "${BASELINE_EXTERNAL_DRIVER_DIGEST}" "${BASELINE_SUPERVISOR_DIGEST}" "${BASELINE_SUPERVISOR_DOCKERFILE_DIGEST}" || baseline_exit=$? # Do not short-circuit: a candidate result is useful even when the frozen # baseline failed, and two equal failures must never constitute parity. -run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" || candidate_exit=$? +run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" "${CANDIDATE_SUPERVISOR}" "${CANDIDATE_SUPERVISOR_DOCKERFILE}" "${CANDIDATE_GATEWAY_DIGEST}" "${CANDIDATE_CLI_DIGEST}" "${CANDIDATE_CONFORMANCE_DIGEST}" "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" "${CANDIDATE_SUPERVISOR_DIGEST}" "${CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST}" || candidate_exit=$? baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index b4fd8c0e85..3cb84bdcc0 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -77,7 +77,7 @@ set -euo pipefail for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS; do [ -z "${!variable:-}" ] || exit 23 done -printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" +printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" case "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" in 1) pull_policy=missing ;; @@ -89,9 +89,16 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then transport=remote_uds external=true fi -printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s"}\n' \ +printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%064d","supervisor_runtime_image":"localhost/openshell/supervisor@sha256:%064d"}\n' \ "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" "${external}" "${transport}" "${pull_policy}" \ + "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" +if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then + replacement="${OPENSHELL_GATEWAY_BIN}.replacement" + printf '#!/usr/bin/env bash\nexit 0\n# mutated\n' >"${replacement}" + chmod 0555 "${replacement}" + mv "${replacement}" "${OPENSHELL_GATEWAY_BIN}" +fi if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac stable=true @@ -118,7 +125,7 @@ if [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = "${OPENSHELL_PARITY_VARIANT:-}" ] || fi printf '{"untrusted":"raw output is intentionally not normalized"}\n' EOF -for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver; do +for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver baseline-supervisor candidate-supervisor; do cat >"${WORKDIR}/bin/${artifact}" <<'EOF' #!/usr/bin/env bash exit 0 @@ -145,6 +152,8 @@ run_harness() { OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN="${WORKDIR}/bin/baseline-driver" \ OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN="${OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE:-${WORKDIR}/bin/candidate-driver}" \ + OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN="${WORKDIR}/bin/baseline-supervisor" \ + OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN="${WORKDIR}/bin/candidate-supervisor" \ OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \ OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \ @@ -158,8 +167,8 @@ OPENSHELL_COMPUTE_DRIVER=wrong \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \ OPENSHELL_DRIVERS=wrong \ run_harness -assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance" -assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance" +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" assert_contains "${WORKDIR}/calls" "|${ROOT}" [ "$(sed -n '1s/|.*//p' "${WORKDIR}/calls")" = baseline ] || fail 'baseline was not invoked first' [ "$(sed -n '2s/|.*//p' "${WORKDIR}/calls")" = candidate ] || fail 'candidate was not invoked second' @@ -175,8 +184,8 @@ assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' run_harness --scenario external-driver -assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/bin/baseline-driver" -assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/bin/candidate-driver" +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/baseline/external-driver|${WORKDIR}/results/artifacts/baseline/supervisor" +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/candidate/external-driver|${WORKDIR}/results/artifacts/candidate/supervisor" assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"external-driver"' assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"external_driver_conformance_smoke"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_profile":"driver-free"' @@ -185,10 +194,14 @@ assert_contains "${WORKDIR}/results/baseline.json" '"gateway_origin":"supplied_o assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"supplied_override"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor@sha256:' assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"conformance_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_dockerfile_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_sha256"' assert_contains "${WORKDIR}/results/comparison.json" '"classification":"pass"' @@ -201,8 +214,8 @@ assert_status "${status}" 2 assert_contains "${WORKDIR}/same-driver.out" 'requires distinct baseline and candidate driver artifacts' run_harness --scenario podman-options -assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/bin/baseline-gateway|${WORKDIR}/bin/baseline-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" -assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/bin/candidate-gateway|${WORKDIR}/bin/candidate-cli|${WORKDIR}/bin/fake-conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance|${ROOT}|podman-options" assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"podman-options"' assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"podman_options"' assert_contains "${WORKDIR}/results/baseline.json" '"normalized_result":"baseline.normalized.json"' @@ -244,6 +257,15 @@ assert_contains "${WORKDIR}/results/candidate.json" '"success":false' assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' [ "$(wc -l <"${WORKDIR}/calls")" -eq 12 ] || fail 'candidate did not run after baseline failure' +set +e +OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT=candidate run_harness >"${WORKDIR}/mutation.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/mutation.out" 'candidate gateway changed after it was staged for execution' +assert_contains "${WORKDIR}/results/candidate.json" '"success":false' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' + set +e bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1 status=$? diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 65232f333d..1dcea1fbf9 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -316,6 +316,39 @@ resolve_podman_supervisor_image() { ensure_podman_supervisor_image() { local image=$1 + if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then + local dockerfile=${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor} + local context="${WORKDIR}/supervisor-image" arch + case "$(uname -m)" in + x86_64|amd64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + *) echo "ERROR: unsupported supervisor image architecture: $(uname -m)" >&2; exit 2 ;; + esac + if [ ! -x "${OPENSHELL_E2E_SUPERVISOR_BIN}" ]; then + echo "ERROR: supplied supervisor binary is not executable: ${OPENSHELL_E2E_SUPERVISOR_BIN}" >&2 + exit 2 + fi + if [ ! -f "${dockerfile}" ]; then + echo "ERROR: supervisor Dockerfile not found: ${dockerfile}" >&2 + exit 2 + fi + mkdir -p "${context}/deploy/docker/.build/prebuilt-binaries/${arch}" + install -m 0555 "${OPENSHELL_E2E_SUPERVISOR_BIN}" \ + "${context}/deploy/docker/.build/prebuilt-binaries/${arch}/openshell-sandbox" + cp "${dockerfile}" "${context}/deploy/docker/Dockerfile.supervisor" + echo "Building Podman supervisor image ${image} from supplied binary..." + ( + cd "${context}" + podman_cmd build \ + --build-arg "TARGETARCH=${arch}" \ + --file deploy/docker/Dockerfile.supervisor \ + --target supervisor \ + --tag "${image}" \ + . + ) + return 0 + fi + if [ "${image}" = "openshell/supervisor:dev" ] \ && [ -z "${OPENSHELL_SUPERVISOR_IMAGE:-}" ] \ && [ -z "${CI:-}" ]; then @@ -401,7 +434,18 @@ fi SUPERVISOR_IMAGE="$(resolve_podman_supervisor_image)" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" -echo "Using Podman supervisor image: ${SUPERVISOR_IMAGE}" +SUPERVISOR_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_IMAGE}")" +SUPERVISOR_IMAGE_ID="${SUPERVISOR_IMAGE_ID#sha256:}" +SUPERVISOR_RUNTIME_IMAGE="$(podman_cmd image inspect --format '{{index .RepoDigests 0}}' "${SUPERVISOR_IMAGE}")" +if ! [[ "${SUPERVISOR_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve immutable supervisor image ID for ${SUPERVISOR_IMAGE}." >&2 + exit 2 +fi +if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve digest-pinned supervisor image reference for ${SUPERVISOR_IMAGE}." >&2 + exit 2 +fi +echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID})" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" @@ -461,7 +505,7 @@ e2e_write_podman_gateway_config \ "${HOST_PORT}" \ "${SANDBOX_IMAGE}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ - "${SUPERVISOR_IMAGE}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" \ "${OPENSHELL_PODMAN_SOCKET:-}" \ "${OIDC_MODE}" \ @@ -474,11 +518,14 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds fi - printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s"}\n' \ + printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_runtime_image":"%s"}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ "${driver_transport}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ + "${SUPERVISOR_IMAGE}" \ + "${SUPERVISOR_IMAGE_ID}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" fi @@ -491,7 +538,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ - OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ + OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_PODMAN_TLS_CA="${PKI_DIR}/ca.crt" \ OPENSHELL_PODMAN_TLS_CERT="${PKI_DIR}/client/tls.crt" \ OPENSHELL_PODMAN_TLS_KEY="${PKI_DIR}/client/tls.key" \ @@ -535,7 +582,7 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_PID_FILE}" OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ -OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ +OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" >"${GATEWAY_LOG}" 2>&1 & GATEWAY_PID=$! From 6896aec73e745288b4203fbce90691bb726d4364 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 06:18:41 -0400 Subject: [PATCH 27/42] fix(e2e): use isolated supervisor tags Signed-off-by: Jesse Jaggars --- e2e/parity/test.sh | 7 ++++--- e2e/with-podman-gateway.sh | 27 ++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 3cb84bdcc0..bdba4d000a 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -89,9 +89,9 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then transport=remote_uds external=true fi -printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%064d","supervisor_runtime_image":"localhost/openshell/supervisor@sha256:%064d"}\n' \ +printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%064d","supervisor_image_digest":"sha256:%064d","supervisor_runtime_image":"%s"}\n' \ "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" "${external}" "${transport}" "${pull_policy}" \ - "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 \ + "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 "${OPENSHELL_SUPERVISOR_IMAGE}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then replacement="${OPENSHELL_GATEWAY_BIN}.replacement" @@ -194,7 +194,8 @@ assert_contains "${WORKDIR}/results/baseline.json" '"gateway_origin":"supplied_o assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"supplied_override"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' -assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor@sha256:' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_image_digest":"sha256:' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"openshell/supervisor:parity-baseline-' assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 1dcea1fbf9..553c902e17 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -319,6 +319,17 @@ ensure_podman_supervisor_image() { if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then local dockerfile=${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor} local context="${WORKDIR}/supervisor-image" arch + case "${image}" in + *:dev|*:latest) + echo "ERROR: supplied supervisor binaries require a unique versioned image tag, not ${image}." >&2 + exit 2 + ;; + *:*) ;; + *) + echo "ERROR: supplied supervisor binaries require an explicit versioned image tag: ${image}." >&2 + exit 2 + ;; + esac case "$(uname -m)" in x86_64|amd64) arch=amd64 ;; aarch64|arm64) arch=arm64 ;; @@ -436,16 +447,21 @@ SUPERVISOR_IMAGE="$(resolve_podman_supervisor_image)" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" SUPERVISOR_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_IMAGE}")" SUPERVISOR_IMAGE_ID="${SUPERVISOR_IMAGE_ID#sha256:}" -SUPERVISOR_RUNTIME_IMAGE="$(podman_cmd image inspect --format '{{index .RepoDigests 0}}' "${SUPERVISOR_IMAGE}")" +SUPERVISOR_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_IMAGE}")" if ! [[ "${SUPERVISOR_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]]; then echo "ERROR: could not resolve immutable supervisor image ID for ${SUPERVISOR_IMAGE}." >&2 exit 2 fi -if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ @sha256:[0-9a-f]{64}$ ]]; then - echo "ERROR: could not resolve digest-pinned supervisor image reference for ${SUPERVISOR_IMAGE}." >&2 +if ! [[ "${SUPERVISOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve supervisor image digest for ${SUPERVISOR_IMAGE}." >&2 exit 2 fi -echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID})" +# Locally built RepoDigest references make Podman's pull API contact a registry +# even when the corresponding image is present. The per-variant tag is unique +# to this exact source SHA and lives in an isolated store; its resolved ID and +# manifest digest attest the image that the driver consumes with policy=missing. +SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE}" +echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST})" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" @@ -518,13 +534,14 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds fi - printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_runtime_image":"%s"}\n' \ + printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s"}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ "${driver_transport}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ "${SUPERVISOR_IMAGE}" \ "${SUPERVISOR_IMAGE_ID}" \ + "${SUPERVISOR_IMAGE_DIGEST}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" fi From 1db9ba32395e80dda6a6e92964edb514ae3c5dc2 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 06:30:51 -0400 Subject: [PATCH 28/42] fix(e2e): qualify parity image tags Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 2 +- e2e/parity/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 4099eb1b4f..6fd4aa00c1 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -409,7 +409,7 @@ run_variant() { local variant=$1 source_sha=$2 schema=$3 gateway=$4 cli=$5 conformance=$6 external_driver=$7 supervisor=$8 supervisor_dockerfile=$9 local gateway_digest=${10} cli_digest=${11} conformance_digest=${12} external_driver_digest=${13} supervisor_digest=${14} supervisor_dockerfile_digest=${15} local result_status variant_home="${RUN_DIR}/${variant}" - local supervisor_image="openshell/supervisor:parity-${variant}-${source_sha:0:12}" + local supervisor_image="localhost/openshell/supervisor:parity-${variant}-${source_sha:0:12}" mkdir -p "${variant_home}/config" "${variant_home}/state" "${variant_home}/cache" "${variant_home}/data" echo "==> schema parity ${variant} (schema v${schema}, ${DRIVER}, ${SCENARIO})" local option_profile="" diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index bdba4d000a..3e436e64c5 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -195,7 +195,7 @@ assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"su assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_image_digest":"sha256:' -assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"openshell/supervisor:parity-baseline-' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor:parity-baseline-' assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' From 8ecfb3dee8f920f9083fef047993ebeef54e7c37 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 06:48:29 -0400 Subject: [PATCH 29/42] fix(e2e): serve parity supervisor locally Signed-off-by: Jesse Jaggars --- e2e/with-podman-gateway.sh | 47 ++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 553c902e17..b04bb1b09e 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -115,6 +115,8 @@ PODMAN_NETWORK_MANAGED=0 PODMAN_SERVICE_PID="" PODMAN_SERVICE_LOG="${WORKDIR}/podman-service.log" PODMAN_SOCKET="" +SUPERVISOR_REGISTRY_CONTAINER="" +SUPERVISOR_REGISTRY_PORT="" GPU_MODE="${OPENSHELL_E2E_PODMAN_GPU:-0}" OIDC_MODE="${OPENSHELL_E2E_OIDC_GATEWAY:-0}" OIDC_ISSUER="${OPENSHELL_E2E_OIDC_ISSUER:-}" @@ -170,6 +172,11 @@ cleanup() { done fi + if [ -n "${SUPERVISOR_REGISTRY_CONTAINER}" ] \ + && command -v podman >/dev/null 2>&1; then + podman_cmd rm -f "${SUPERVISOR_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true + fi + if [ "${PODMAN_NETWORK_MANAGED}" = "1" ] \ && [ -n "${PODMAN_NETWORK_NAME}" ] \ && command -v podman >/dev/null 2>&1; then @@ -434,6 +441,15 @@ if ! podman_cmd info >/dev/null 2>&1; then echo " Start it with 'podman machine start' on macOS, or the user service on Linux." >&2 exit 2 fi +if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then + SUPERVISOR_REGISTRY_PORT="$(e2e_pick_port)" + cat >"${WORKDIR}/registries.conf" <&2 exit 2 fi -# Locally built RepoDigest references make Podman's pull API contact a registry -# even when the corresponding image is present. The per-variant tag is unique -# to this exact source SHA and lives in an isolated store; its resolved ID and -# manifest digest attest the image that the driver consumes with policy=missing. SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE}" +if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then + # Podman's image-pull API contacts a registry even for a locally present + # image with policy=missing. Publish the exact staged image to a disposable + # loopback-only registry so both frozen and current drivers can resolve it + # without any external mutable-tag dependency. + SUPERVISOR_REGISTRY_CONTAINER="openshell-parity-registry-$$" + supervisor_registry_image="localhost:${SUPERVISOR_REGISTRY_PORT}/openshell/supervisor:${SUPERVISOR_IMAGE##*:}" + podman_cmd run --detach --name "${SUPERVISOR_REGISTRY_CONTAINER}" \ + --publish "127.0.0.1:${SUPERVISOR_REGISTRY_PORT}:5000" \ + docker.io/library/registry:2 >/dev/null + supervisor_registry_ready=0 + for _ in $(seq 1 30); do + if curl --noproxy '*' --silent --fail \ + "http://127.0.0.1:${SUPERVISOR_REGISTRY_PORT}/v2/" >/dev/null; then + supervisor_registry_ready=1 + break + fi + sleep 1 + done + if [ "${supervisor_registry_ready}" != 1 ]; then + echo "ERROR: disposable supervisor registry did not become ready." >&2 + exit 2 + fi + podman_cmd tag "${SUPERVISOR_IMAGE}" "${supervisor_registry_image}" + podman_cmd push --tls-verify=false "${supervisor_registry_image}" >/dev/null + SUPERVISOR_RUNTIME_IMAGE="${supervisor_registry_image}" +fi echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST})" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" From 7e4655a8c28eb47243d01bba0b91389757172e83 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 07:06:12 -0400 Subject: [PATCH 30/42] test(e2e): isolate parity podman services Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 2 + e2e/parity/test.sh | 3 +- e2e/with-podman-gateway.sh | 76 ++++++++++++-------------------------- 3 files changed, 27 insertions(+), 54 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 6fd4aa00c1..e4abcfbb2f 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -422,12 +422,14 @@ run_variant() { fi if env -u OPENSHELL_GATEWAY_ENDPOINT -u OPENSHELL_GATEWAY_CONFIG \ -u OPENSHELL_COMPUTE_DRIVER -u OPENSHELL_COMPUTE_DRIVER_SOCKET -u OPENSHELL_DRIVERS \ + -u OPENSHELL_PODMAN_SOCKET \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ OPENSHELL_EXTERNAL_DRIVER_BIN="${external_driver}" \ OPENSHELL_E2E_SUPERVISOR_BIN="${supervisor}" \ OPENSHELL_E2E_SUPERVISOR_DOCKERFILE="${supervisor_dockerfile}" \ + OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE=1 \ OPENSHELL_SUPERVISOR_IMAGE="${supervisor_image}" \ OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 3e436e64c5..0ee9a0b1b9 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -74,7 +74,7 @@ mkdir -p "${WORKDIR}/bin" cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF' #!/usr/bin/env bash set -euo pipefail -for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS; do +for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET; do [ -z "${!variable:-}" ] || exit 23 done printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" @@ -166,6 +166,7 @@ OPENSHELL_GATEWAY_CONFIG=/tmp/untrusted.toml \ OPENSHELL_COMPUTE_DRIVER=wrong \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \ OPENSHELL_DRIVERS=wrong \ +OPENSHELL_PODMAN_SOCKET=/tmp/untrusted-podman.sock \ run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index b04bb1b09e..582e05c2d6 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -115,8 +115,6 @@ PODMAN_NETWORK_MANAGED=0 PODMAN_SERVICE_PID="" PODMAN_SERVICE_LOG="${WORKDIR}/podman-service.log" PODMAN_SOCKET="" -SUPERVISOR_REGISTRY_CONTAINER="" -SUPERVISOR_REGISTRY_PORT="" GPU_MODE="${OPENSHELL_E2E_PODMAN_GPU:-0}" OIDC_MODE="${OPENSHELL_E2E_OIDC_GATEWAY:-0}" OIDC_ISSUER="${OPENSHELL_E2E_OIDC_ISSUER:-}" @@ -172,11 +170,6 @@ cleanup() { done fi - if [ -n "${SUPERVISOR_REGISTRY_CONTAINER}" ] \ - && command -v podman >/dev/null 2>&1; then - podman_cmd rm -f "${SUPERVISOR_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true - fi - if [ "${PODMAN_NETWORK_MANAGED}" = "1" ] \ && [ -n "${PODMAN_NETWORK_NAME}" ] \ && command -v podman >/dev/null 2>&1; then @@ -243,17 +236,21 @@ default_podman_socket_path() { } ensure_podman_api_socket() { - if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then - return 0 - fi + if [ "${OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE:-0}" != 1 ]; then + if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then + return 0 + fi - local default_socket - default_socket="$(default_podman_socket_path || true)" - if [ -n "${default_socket}" ] \ - && [ -S "${default_socket}" ] \ - && podman_cmd --url "unix://${default_socket}" info >/dev/null 2>&1; then - export OPENSHELL_PODMAN_SOCKET="${default_socket}" - return 0 + local default_socket + default_socket="$(default_podman_socket_path || true)" + if [ -n "${default_socket}" ] \ + && [ -S "${default_socket}" ] \ + && podman_cmd --url "unix://${default_socket}" info >/dev/null 2>&1; then + export OPENSHELL_PODMAN_SOCKET="${default_socket}" + return 0 + fi + else + unset OPENSHELL_PODMAN_SOCKET fi # `podman system service` is a Linux-only subcommand — the macOS client @@ -441,15 +438,6 @@ if ! podman_cmd info >/dev/null 2>&1; then echo " Start it with 'podman machine start' on macOS, or the user service on Linux." >&2 exit 2 fi -if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then - SUPERVISOR_REGISTRY_PORT="$(e2e_pick_port)" - cat >"${WORKDIR}/registries.conf" <&2 exit 2 fi -SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE}" -if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then - # Podman's image-pull API contacts a registry even for a locally present - # image with policy=missing. Publish the exact staged image to a disposable - # loopback-only registry so both frozen and current drivers can resolve it - # without any external mutable-tag dependency. - SUPERVISOR_REGISTRY_CONTAINER="openshell-parity-registry-$$" - supervisor_registry_image="localhost:${SUPERVISOR_REGISTRY_PORT}/openshell/supervisor:${SUPERVISOR_IMAGE##*:}" - podman_cmd run --detach --name "${SUPERVISOR_REGISTRY_CONTAINER}" \ - --publish "127.0.0.1:${SUPERVISOR_REGISTRY_PORT}:5000" \ - docker.io/library/registry:2 >/dev/null - supervisor_registry_ready=0 - for _ in $(seq 1 30); do - if curl --noproxy '*' --silent --fail \ - "http://127.0.0.1:${SUPERVISOR_REGISTRY_PORT}/v2/" >/dev/null; then - supervisor_registry_ready=1 - break - fi - sleep 1 - done - if [ "${supervisor_registry_ready}" != 1 ]; then - echo "ERROR: disposable supervisor registry did not become ready." >&2 - exit 2 - fi - podman_cmd tag "${SUPERVISOR_IMAGE}" "${supervisor_registry_image}" - podman_cmd push --tls-verify=false "${supervisor_registry_image}" >/dev/null - SUPERVISOR_RUNTIME_IMAGE="${supervisor_registry_image}" +# The parity harness forces a temporary Podman API service into the same +# isolated XDG store where this image was built. Address the local image by its +# immutable manifest digest so policy=missing cannot resolve a mutable tag or +# contact a registry for a different artifact. +SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%:*}" +SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE_REPOSITORY}@${SUPERVISOR_IMAGE_DIGEST}" +if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: supervisor runtime image is not digest-pinned: ${SUPERVISOR_RUNTIME_IMAGE}" >&2 + exit 2 fi echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST})" From 0bc049c7d0a78a4bc130be41674624cbc940e2e6 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 07:46:17 -0400 Subject: [PATCH 31/42] test(e2e): harden parity evidence provenance Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 8 + e2e/parity/test.sh | 36 +- e2e/parity/verify-results.py | 442 ++++++++++++++++++ e2e/support/podman-gateway-config.sh | 7 +- e2e/with-podman-gateway.sh | 52 ++- ...chema_v2_compute_boundary_verifier_test.py | 165 +++++++ 6 files changed, 694 insertions(+), 16 deletions(-) create mode 100644 e2e/parity/verify-results.py create mode 100644 python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index e4abcfbb2f..568391aac5 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -322,6 +322,10 @@ stage_artifact candidate supervisor.Dockerfile "${CANDIDATE_WORKTREE}/deploy/doc if [ "${SCENARIO}" = external-driver ]; then stage_executable baseline external-driver "${BASELINE_EXTERNAL_DRIVER}" BASELINE_EXTERNAL_DRIVER BASELINE_EXTERNAL_DRIVER_DIGEST stage_executable candidate external-driver "${CANDIDATE_EXTERNAL_DRIVER}" CANDIDATE_EXTERNAL_DRIVER CANDIDATE_EXTERNAL_DRIVER_DIGEST + if [ "${BASELINE_EXTERNAL_DRIVER_DIGEST}" = "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" ]; then + echo "ERROR: external-driver parity requires different baseline and candidate driver content." >&2 + exit 2 + fi fi require_executable "Podman parity wrapper" "${WRAPPER}" @@ -423,6 +427,9 @@ run_variant() { if env -u OPENSHELL_GATEWAY_ENDPOINT -u OPENSHELL_GATEWAY_CONFIG \ -u OPENSHELL_COMPUTE_DRIVER -u OPENSHELL_COMPUTE_DRIVER_SOCKET -u OPENSHELL_DRIVERS \ -u OPENSHELL_PODMAN_SOCKET \ + -u CONTAINER_HOST -u CONTAINER_CONNECTION -u CONTAINERS_STORAGE_CONF \ + -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ + -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ @@ -435,6 +442,7 @@ run_variant() { OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE="${RESULTS_DIR}/${variant}.gateway.toml" \ OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE="${RESULTS_DIR}/${variant}.launch.json" \ + OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE="${RESULTS_DIR}/artifacts/${variant}/supervisor.packages.txt" \ OPENSHELL_GATEWAY_BIN="${gateway}" \ OPENSHELL_BIN="${cli}" \ OPENSHELL_CONFORMANCE_BIN="${conformance}" \ diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 0ee9a0b1b9..5c37099e62 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -74,7 +74,12 @@ mkdir -p "${WORKDIR}/bin" cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF' #!/usr/bin/env bash set -euo pipefail -for variable in OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET; do +for variable in \ + OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER \ + OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET \ + CONTAINER_HOST CONTAINER_CONNECTION CONTAINERS_STORAGE_CONF CONTAINERS_CONF \ + CONTAINERS_REGISTRIES_CONF CONTAINERS_REGISTRIES_CONF_DIR CONTAINERS_POLICY \ + PODMAN_CONNECTIONS_CONF DOCKER_HOST; do [ -z "${!variable:-}" ] || exit 23 done printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" @@ -89,10 +94,12 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then transport=remote_uds external=true fi +runtime_image="localhost/openshell/supervisor@sha256:$(printf '%064d' 0)" printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%064d","supervisor_image_digest":"sha256:%064d","supervisor_runtime_image":"%s"}\n' \ "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" "${external}" "${transport}" "${pull_policy}" \ - "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 "${OPENSHELL_SUPERVISOR_IMAGE}" \ + "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 "${runtime_image}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" +printf 'fixture-package-1.0-r0\n' >"${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE}" if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then replacement="${OPENSHELL_GATEWAY_BIN}.replacement" printf '#!/usr/bin/env bash\nexit 0\n# mutated\n' >"${replacement}" @@ -126,10 +133,7 @@ fi printf '{"untrusted":"raw output is intentionally not normalized"}\n' EOF for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver baseline-supervisor candidate-supervisor; do - cat >"${WORKDIR}/bin/${artifact}" <<'EOF' -#!/usr/bin/env bash -exit 0 -EOF + printf '#!/usr/bin/env bash\n# %s\nexit 0\n' "${artifact}" >"${WORKDIR}/bin/${artifact}" done chmod +x "${WORKDIR}/bin/"* @@ -167,6 +171,15 @@ OPENSHELL_COMPUTE_DRIVER=wrong \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \ OPENSHELL_DRIVERS=wrong \ OPENSHELL_PODMAN_SOCKET=/tmp/untrusted-podman.sock \ +CONTAINER_HOST=tcp://untrusted.invalid:9999 \ +CONTAINER_CONNECTION=untrusted \ +CONTAINERS_STORAGE_CONF=/tmp/untrusted-storage.conf \ +CONTAINERS_CONF=/tmp/untrusted-containers.conf \ +CONTAINERS_REGISTRIES_CONF=/tmp/untrusted-registries.conf \ +CONTAINERS_REGISTRIES_CONF_DIR=/tmp/untrusted-registries.d \ +CONTAINERS_POLICY=/tmp/untrusted-policy.json \ +PODMAN_CONNECTIONS_CONF=/tmp/untrusted-connections.json \ +DOCKER_HOST=tcp://untrusted.invalid:2375 \ run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" @@ -196,7 +209,7 @@ assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"su assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_image_digest":"sha256:' -assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor:parity-baseline-' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor@sha256:' assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' @@ -215,6 +228,15 @@ set -e assert_status "${status}" 2 assert_contains "${WORKDIR}/same-driver.out" 'requires distinct baseline and candidate driver artifacts' +cp "${WORKDIR}/bin/baseline-driver" "${WORKDIR}/bin/same-content-driver" +set +e +OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/same-content-driver" \ + run_harness --scenario external-driver >"${WORKDIR}/same-driver-content.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/same-driver-content.out" 'requires different baseline and candidate driver content' + run_harness --scenario podman-options assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance|${ROOT}|podman-options" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance|${ROOT}|podman-options" diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py new file mode 100644 index 0000000000..ec68731c1b --- /dev/null +++ b/e2e/parity/verify-results.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify retained parity evidence and emit its normalized comparison.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tomllib +from pathlib import Path +from typing import Any + +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +DIGEST_REFERENCE_RE = re.compile(r"^[^@]+@sha256:([0-9a-f]{64})$") +ARTIFACT_FIELDS = { + "gateway_sha256": "gateway", + "cli_sha256": "cli", + "conformance_sha256": "conformance", + "supervisor_sha256": "supervisor", + "supervisor_dockerfile_sha256": "supervisor.Dockerfile", +} +ORACLE_MARKERS = ( + "][smoke/status] completed", + "][smoke/create] completed", + "][smoke/get-ready] completed", + "][smoke/list-visible/0] completed", + "][smoke/exec] completed", + "][smoke/delete] completed", + "][smoke/list-empty/query/0] completed", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as source: + value = json.load(source) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def verify_variant( + results_dir: Path, + variant: str, + expected_sha: str, + schema_version: int, + scenario: str, +) -> dict[str, Any]: + result_path = results_dir / f"{variant}.json" + launch_path = results_dir / f"{variant}.launch.json" + log_path = results_dir / f"{variant}.log" + config_path = results_dir / f"{variant}.gateway.toml" + result = load_json(result_path) + launch = load_json(launch_path) + require(config_path.is_file(), f"{config_path}: retained gateway config is missing") + with config_path.open("rb") as config_file: + config = tomllib.load(config_file) + + require(result.get("variant") == variant, f"{result_path}: variant mismatch") + require( + result.get("source_sha") == expected_sha, f"{result_path}: source SHA mismatch" + ) + require( + result.get("schema_version") == schema_version, + f"{result_path}: schema mismatch", + ) + require(result.get("scenario") == scenario, f"{result_path}: scenario mismatch") + require(result.get("driver") == "podman", f"{result_path}: driver mismatch") + expected_profile = "driver-free" if scenario == "external-driver" else "in-tree" + expected_features = ( + "--no-default-features --features telemetry" + if scenario == "external-driver" + else "default" + ) + require( + result.get("gateway_profile") == expected_profile, + f"{result_path}: gateway profile mismatch", + ) + require( + result.get("gateway_cargo_features") == expected_features, + f"{result_path}: gateway feature profile mismatch", + ) + require(result.get("success") is True, f"{result_path}: parity oracle did not pass") + require( + result.get("gateway_origin") == "built_by_harness", + f"{result_path}: gateway was not built by the harness", + ) + require( + result.get("cli_origin") == "built_by_harness", + f"{result_path}: CLI was not built by the harness", + ) + require( + result.get("conformance_origin") == "built_by_harness", + f"{result_path}: conformance runner was not built by the harness", + ) + require( + result.get("supervisor_origin") == "built_by_harness", + f"{result_path}: supervisor was not built by the harness", + ) + + external = scenario == "external-driver" + expected_driver_origin = "built_by_harness" if external else "not_applicable" + require( + result.get("external_driver_origin") == expected_driver_origin, + f"{result_path}: external driver origin mismatch", + ) + require( + launch.get("schema_version") == schema_version, + f"{launch_path}: schema mismatch", + ) + require( + launch.get("external_compute_driver") is external, + f"{launch_path}: topology mismatch", + ) + expected_transport = "remote_uds" if external else "in_tree" + require( + launch.get("compute_driver_transport") == expected_transport, + f"{launch_path}: transport mismatch", + ) + expected_policy = "missing" if schema_version == 1 else "if_not_present" + require( + launch.get("external_driver_pull_policy") == expected_policy, + f"{launch_path}: pull-policy mismatch", + ) + + openshell = config.get("openshell", {}) + gateway = openshell.get("gateway", {}) + podman_config = openshell.get("drivers", {}).get("podman", {}) + require( + openshell.get("version") == schema_version, f"{config_path}: schema mismatch" + ) + expected_selector = ["podman"] if schema_version == 1 else "podman" + selector_field = "compute_drivers" if schema_version == 1 else "compute_driver" + require( + gateway.get(selector_field) == expected_selector, + f"{config_path}: selected compute driver mismatch", + ) + require(isinstance(podman_config, dict), f"{config_path}: Podman table is missing") + if external: + require( + set(podman_config) == {"socket_path"}, + f"{config_path}: external gateway Podman table is not transport-only", + ) + else: + required_runtime_fields = { + "socket_path", + "network_name", + "default_image", + "image_pull_policy", + "supervisor_image", + } + require( + set(podman_config) >= required_runtime_fields, + f"{config_path}: in-tree Podman runtime fields are incomplete", + ) + require( + DIGEST_REFERENCE_RE.fullmatch(podman_config["default_image"]) is not None, + f"{config_path}: sandbox image is not digest-pinned", + ) + require( + DIGEST_REFERENCE_RE.fullmatch(podman_config["supervisor_image"]) + is not None, + f"{config_path}: supervisor image is not digest-pinned", + ) + + artifact_fields = dict(ARTIFACT_FIELDS) + if external: + artifact_fields["external_driver_sha256"] = "external-driver" + artifact_hashes: dict[str, str] = {} + for field, filename in artifact_fields.items(): + expected_hash = result.get(field) + require( + isinstance(expected_hash, str) + and SHA256_RE.fullmatch(expected_hash) is not None, + f"{result_path}: invalid {field}", + ) + artifact_path = results_dir / "artifacts" / variant / filename + require( + artifact_path.is_file(), f"{artifact_path}: retained artifact is missing" + ) + actual_hash = sha256(artifact_path) + require( + actual_hash == expected_hash, + f"{artifact_path}: retained artifact hash mismatch", + ) + artifact_hashes[filename] = actual_hash + + image_id = launch.get("supervisor_image_id") + image_digest = launch.get("supervisor_image_digest") + runtime_image = launch.get("supervisor_runtime_image") + require( + isinstance(image_id, str) and SHA256_RE.fullmatch(image_id) is not None, + f"{launch_path}: invalid supervisor image ID", + ) + require( + isinstance(image_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) is not None, + f"{launch_path}: invalid supervisor image digest", + ) + match = ( + DIGEST_REFERENCE_RE.fullmatch(runtime_image) + if isinstance(runtime_image, str) + else None + ) + require( + match is not None, + f"{launch_path}: supervisor runtime image is not digest-pinned", + ) + require( + f"sha256:{match.group(1)}" == image_digest, + f"{launch_path}: runtime reference digest mismatch", + ) + + sandbox_id = launch.get("sandbox_image_id") + sandbox_digest = launch.get("sandbox_image_digest") + sandbox_runtime = launch.get("sandbox_runtime_image") + sandbox_match = ( + DIGEST_REFERENCE_RE.fullmatch(sandbox_runtime) + if isinstance(sandbox_runtime, str) + else None + ) + require( + isinstance(sandbox_id, str) and SHA256_RE.fullmatch(sandbox_id) is not None, + f"{launch_path}: invalid sandbox image ID", + ) + require( + isinstance(sandbox_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", sandbox_digest) is not None, + f"{launch_path}: invalid sandbox image digest", + ) + require( + sandbox_match is not None + and f"sha256:{sandbox_match.group(1)}" == sandbox_digest, + f"{launch_path}: sandbox runtime image is not digest-pinned", + ) + if not external: + require( + podman_config["default_image"] == sandbox_runtime + and podman_config["supervisor_image"] == runtime_image, + f"{config_path}: runtime image references differ from launch evidence", + ) + + for field in ("supervisor_base_image_id", "supervisor_package_manifest_sha256"): + value = launch.get(field) + require( + isinstance(value, str) and SHA256_RE.fullmatch(value) is not None, + f"{launch_path}: invalid {field}", + ) + base_digest = launch.get("supervisor_base_image_digest") + require( + isinstance(base_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", base_digest) is not None, + f"{launch_path}: invalid supervisor base-image digest", + ) + package_path = results_dir / "artifacts" / variant / "supervisor.packages.txt" + require(package_path.is_file(), f"{package_path}: package manifest is missing") + require( + sha256(package_path) == launch["supervisor_package_manifest_sha256"], + f"{package_path}: package manifest hash mismatch", + ) + artifact_hashes["supervisor.packages.txt"] = sha256(package_path) + + require(log_path.is_file(), f"{log_path}: retained raw log is missing") + raw_log = log_path.read_text(encoding="utf-8", errors="replace") + for marker in ORACLE_MARKERS: + require( + re.search(re.escape(marker) + r".*exit 0", raw_log) is not None, + f"{log_path}: missing successful lifecycle oracle marker {marker}", + ) + require( + '"passed": true' in raw_log, + f"{log_path}: missing successful conformance result", + ) + launch_markers = ( + runtime_image, + image_id, + image_digest, + sandbox_runtime, + sandbox_id, + sandbox_digest, + launch["supervisor_base_image_id"], + base_digest, + launch["supervisor_package_manifest_sha256"], + ) + require( + all(marker in raw_log for marker in launch_markers), + f"{log_path}: launch provenance is absent from raw output", + ) + + return { + "schema_version": schema_version, + "source_sha": expected_sha, + "gateway_profile": result["gateway_profile"], + "gateway_cargo_features": result["gateway_cargo_features"], + "artifact_origins": { + "gateway": result["gateway_origin"], + "cli": result["cli_origin"], + "conformance": result["conformance_origin"], + "external_driver": result["external_driver_origin"], + "supervisor": result["supervisor_origin"], + }, + "artifact_sha256": artifact_hashes, + "launch_attestation": launch, + "raw_evidence_sha256": { + result_path.name: sha256(result_path), + launch_path.name: sha256(launch_path), + log_path.name: sha256(log_path), + config_path.name: sha256(config_path), + }, + "artifacts_verified": True, + "raw_output_verified": True, + "success": True, + } + + +def verify_topology( + results_dir: Path, + baseline_sha: str, + candidate_sha: str, + scenario: str, +) -> dict[str, Any]: + comparison_path = results_dir / "comparison.json" + comparison = load_json(comparison_path) + require( + comparison.get("scenario") == scenario, f"{comparison_path}: scenario mismatch" + ) + for field in ("baseline_success", "candidate_success", "parity", "accepted"): + require( + comparison.get(field) is True, f"{comparison_path}: {field} is not true" + ) + require( + comparison.get("classification") == "pass", + f"{comparison_path}: classification is not pass", + ) + + baseline = verify_variant(results_dir, "baseline", baseline_sha, 1, scenario) + candidate = verify_variant(results_dir, "candidate", candidate_sha, 2, scenario) + if scenario == "external-driver": + baseline_driver = results_dir / "artifacts/baseline/external-driver" + candidate_driver = results_dir / "artifacts/candidate/external-driver" + require( + baseline_driver.resolve() != candidate_driver.resolve(), + "external-driver artifacts resolve to the same path", + ) + require( + not baseline_driver.samefile(candidate_driver), + "external-driver artifacts share an inode", + ) + require( + baseline["artifact_sha256"]["external-driver"] + != candidate["artifact_sha256"]["external-driver"], + "external-driver artifacts have identical content", + ) + + return { + "baseline": baseline, + "candidate": candidate, + "comparison_sha256": sha256(comparison_path), + "classification": "pass", + "parity": True, + "accepted": True, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline-sha", required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--in-tree", required=True, type=Path) + parser.add_argument("--external-uds", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + require( + re.fullmatch(r"[0-9a-f]{40}", args.baseline_sha) is not None, + "invalid baseline SHA", + ) + require( + re.fullmatch(r"[0-9a-f]{40}", args.candidate_sha) is not None, + "invalid candidate SHA", + ) + + report = { + "manifest_version": 2, + "baseline_commit": args.baseline_sha, + "candidate_commit": args.candidate_sha, + "lane": "local-linux-x86_64-rootless-podman-5.8.2", + "oracle": { + "status": True, + "create": True, + "ready": True, + "list_visible": True, + "callback_exec_exact_marker": True, + "delete": True, + "list_empty": True, + }, + "in_tree": verify_topology( + args.in_tree, args.baseline_sha, args.candidate_sha, "smoke" + ), + "external_uds": verify_topology( + args.external_uds, args.baseline_sha, args.candidate_sha, "external-driver" + ), + "callback_listener": { + "in_tree_baseline_exec": True, + "in_tree_candidate_exec": True, + "external_baseline_exec": True, + "external_candidate_exec": True, + "classification": "pass", + }, + "classification": "pass", + "accepted": True, + "verification": { + "retained_artifact_hashes_recomputed": True, + "raw_lifecycle_output_inspected": True, + "digest_pinned_supervisor_runtime_verified": True, + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index a26ef54d54..dcf52d09c6 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -124,7 +124,12 @@ e2e_write_podman_gateway_config() { ;; 2) cp "${root}/deploy/rpm/gateway.toml.default" "${output}" - if [ "${option_profile}" = "podman-options" ]; then + if [ "${external_driver}" = "1" ]; then + # A remote UDS driver owns all runtime options. Keep the selected + # gateway table transport-only so no in-tree setting can be mistaken + # for executed external-driver configuration. + sed -i '/^health_check_interval_secs = /d' "${output}" + elif [ "${option_profile}" = "podman-options" ]; then sed -i 's/^health_check_interval_secs = .*/health_check_interval_secs = 7/' "${output}" fi # The v2 template opens the Podman table. Insert gateway-owned TLS diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 582e05c2d6..5aa23a7bb0 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -470,19 +470,47 @@ if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then echo "ERROR: supervisor runtime image is not digest-pinned: ${SUPERVISOR_RUNTIME_IMAGE}" >&2 exit 2 fi -echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST})" +SUPERVISOR_BASE_IMAGE="$(awk '$1 == "FROM" { print $2; exit }' "${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor}")" +SUPERVISOR_BASE_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_BASE_IMAGE}")" +SUPERVISOR_BASE_IMAGE_ID="${SUPERVISOR_BASE_IMAGE_ID#sha256:}" +SUPERVISOR_BASE_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_BASE_IMAGE}")" +if ! [[ "${SUPERVISOR_BASE_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${SUPERVISOR_BASE_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve supervisor base-image provenance for ${SUPERVISOR_BASE_IMAGE}." >&2 + exit 2 +fi +SUPERVISOR_PACKAGE_MANIFEST="${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE:-${WORKDIR}/supervisor.packages.txt}" +mkdir -p "$(dirname "${SUPERVISOR_PACKAGE_MANIFEST}")" +podman_cmd run --rm --network none --entrypoint /sbin/apk \ + "${SUPERVISOR_RUNTIME_IMAGE}" info -v | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" +SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" +echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -SANDBOX_IMAGE="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" +SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" PODMAN_STOP_TIMEOUT_SECS="${OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS:-15}" if ! [[ "${PODMAN_STOP_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then echo "ERROR: OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS must be a non-negative integer." >&2 exit 2 fi -if ! podman_cmd image exists "${SANDBOX_IMAGE}" 2>/dev/null; then - echo "Pulling ${SANDBOX_IMAGE}..." - podman_cmd pull "${SANDBOX_IMAGE}" +if ! podman_cmd image exists "${SANDBOX_IMAGE_REQUEST}" 2>/dev/null; then + echo "Pulling ${SANDBOX_IMAGE_REQUEST}..." + podman_cmd pull "${SANDBOX_IMAGE_REQUEST}" +fi +SANDBOX_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SANDBOX_IMAGE_REQUEST}")" +SANDBOX_IMAGE_ID="${SANDBOX_IMAGE_ID#sha256:}" +SANDBOX_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SANDBOX_IMAGE_REQUEST}")" +SANDBOX_IMAGE_REPOSITORY="${SANDBOX_IMAGE_REQUEST%%@*}" +case "${SANDBOX_IMAGE_REPOSITORY##*/}" in + *:*) SANDBOX_IMAGE_REPOSITORY="${SANDBOX_IMAGE_REPOSITORY%:*}" ;; +esac +SANDBOX_RUNTIME_IMAGE="${SANDBOX_IMAGE_REPOSITORY}@${SANDBOX_IMAGE_DIGEST}" +if ! [[ "${SANDBOX_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${SANDBOX_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve an immutable sandbox image for ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 fi +echo "Using Podman sandbox image: ${SANDBOX_RUNTIME_IMAGE} (ID ${SANDBOX_IMAGE_ID}, digest ${SANDBOX_IMAGE_DIGEST})" PKI_DIR="${WORKDIR}/pki" e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" "host.containers.internal" @@ -528,7 +556,7 @@ e2e_write_podman_gateway_config \ "${DRIVER_SOCKET}" \ "${PODMAN_NETWORK_NAME}" \ "${HOST_PORT}" \ - "${SANDBOX_IMAGE}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ "${PODMAN_STOP_TIMEOUT_SECS}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" \ @@ -543,7 +571,7 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds fi - printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s"}\n' \ + printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s"}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ "${driver_transport}" \ @@ -552,13 +580,21 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${SUPERVISOR_IMAGE_ID}" \ "${SUPERVISOR_IMAGE_DIGEST}" \ "${SUPERVISOR_RUNTIME_IMAGE}" \ + "${SUPERVISOR_BASE_IMAGE}" \ + "${SUPERVISOR_BASE_IMAGE_ID}" \ + "${SUPERVISOR_BASE_IMAGE_DIGEST}" \ + "${SUPERVISOR_PACKAGE_MANIFEST_SHA256}" \ + "${SANDBOX_IMAGE_REQUEST}" \ + "${SANDBOX_IMAGE_ID}" \ + "${SANDBOX_IMAGE_DIGEST}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ - OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py new file mode 100644 index 0000000000..bb7a755967 --- /dev/null +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the retained-artifact verifier used by schema-v2 Step 10.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from types import ModuleType + +REPO_ROOT = Path(__file__).resolve().parents[2] +VERIFIER_PATH = REPO_ROOT / "e2e/parity/verify-results.py" +BASELINE_SHA = "1" * 40 +CANDIDATE_SHA = "2" * 40 +IMAGE_ID = "3" * 64 +IMAGE_DIGEST = f"sha256:{'4' * 64}" +RUNTIME_IMAGE = f"localhost/openshell/supervisor@{IMAGE_DIGEST}" + + +def load_verifier() -> ModuleType: + spec = importlib.util.spec_from_file_location("parity_verifier", VERIFIER_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value) + "\n", encoding="utf-8") + + +def create_variant( + verifier: ModuleType, + results_dir: Path, + variant: str, + source_sha: str, + schema_version: int, +) -> None: + artifact_dir = results_dir / "artifacts" / variant + artifact_dir.mkdir(parents=True) + artifacts = { + "gateway": f"{variant}-gateway", + "cli": f"{variant}-cli", + "conformance": f"{variant}-conformance", + "supervisor": f"{variant}-supervisor", + "supervisor.Dockerfile": f"{variant}-dockerfile", + "external-driver": f"{variant}-external-driver", + "supervisor.packages.txt": "fixture-package-1.0-r0\n", + } + for filename, content in artifacts.items(): + (artifact_dir / filename).write_text(content, encoding="utf-8") + + write_json( + results_dir / f"{variant}.json", + { + "variant": variant, + "source_sha": source_sha, + "schema_version": schema_version, + "driver": "podman", + "scenario": "external-driver", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "gateway_origin": "built_by_harness", + "cli_origin": "built_by_harness", + "conformance_origin": "built_by_harness", + "external_driver_origin": "built_by_harness", + "supervisor_origin": "built_by_harness", + "gateway_sha256": verifier.sha256(artifact_dir / "gateway"), + "cli_sha256": verifier.sha256(artifact_dir / "cli"), + "conformance_sha256": verifier.sha256(artifact_dir / "conformance"), + "supervisor_sha256": verifier.sha256(artifact_dir / "supervisor"), + "supervisor_dockerfile_sha256": verifier.sha256( + artifact_dir / "supervisor.Dockerfile" + ), + "external_driver_sha256": verifier.sha256(artifact_dir / "external-driver"), + "success": True, + }, + ) + selector = ( + 'compute_drivers = ["podman"]' + if schema_version == 1 + else 'compute_driver = "podman"' + ) + (results_dir / f"{variant}.gateway.toml").write_text( + f"""[openshell] +version = {schema_version} +[openshell.gateway] +{selector} +[openshell.drivers.podman] +socket_path = "/tmp/{variant}.sock" +""", + encoding="utf-8", + ) + policy = "missing" if schema_version == 1 else "if_not_present" + package_hash = verifier.sha256(artifact_dir / "supervisor.packages.txt") + write_json( + results_dir / f"{variant}.launch.json", + { + "schema_version": schema_version, + "external_compute_driver": True, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": policy, + "supervisor_image_id": IMAGE_ID, + "supervisor_image_digest": IMAGE_DIGEST, + "supervisor_runtime_image": RUNTIME_IMAGE, + "supervisor_base_image": "alpine:fixture", + "supervisor_base_image_id": IMAGE_ID, + "supervisor_base_image_digest": IMAGE_DIGEST, + "supervisor_package_manifest_sha256": package_hash, + "sandbox_image_request": "example.invalid/sandbox:fixture", + "sandbox_image_id": IMAGE_ID, + "sandbox_image_digest": IMAGE_DIGEST, + "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + }, + ) + lifecycle = "\n".join( + f"[run fixture{marker} in 1ms: exit 0" for marker in verifier.ORACLE_MARKERS + ) + (results_dir / f"{variant}.log").write_text( + f"{lifecycle}\n{RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} " + f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', + encoding="utf-8", + ) + + +def create_external_bundle(verifier: ModuleType, results_dir: Path) -> None: + create_variant(verifier, results_dir, "baseline", BASELINE_SHA, 1) + create_variant(verifier, results_dir, "candidate", CANDIDATE_SHA, 2) + write_json( + results_dir / "comparison.json", + { + "scenario": "external-driver", + "baseline_success": True, + "candidate_success": True, + "parity": True, + "accepted": True, + "classification": "pass", + }, + ) + + +def test_verifier_recomputes_retained_artifact_hashes(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + + report = verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + assert report["baseline"]["artifacts_verified"] is True + assert report["candidate"]["raw_output_verified"] is True + + (tmp_path / "artifacts/candidate/gateway").write_text( + "mutated after execution", encoding="utf-8" + ) + with pytest.raises(ValueError, match="retained artifact hash mismatch"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) From b8297831aa2591d2fefd58539fd1d36cf42943eb Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 08:05:01 -0400 Subject: [PATCH 32/42] test(e2e): pin parity sandbox artifacts Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 57 ++++++++++++++++- e2e/parity/test.sh | 34 ++++++++-- e2e/parity/verify-results.py | 22 +++++++ e2e/with-podman-gateway.sh | 10 +++ ...chema_v2_compute_boundary_verifier_test.py | 63 ++++++++++++++++++- 5 files changed, 180 insertions(+), 6 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 568391aac5..36cd5ebe83 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -19,6 +19,9 @@ RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" PODMAN_OPTIONS_ORACLE="${OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE:-${ROOT}/e2e/parity/podman-options.sh}" PODMAN_BIN="${OPENSHELL_PARITY_PODMAN_BIN:-podman}" +DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}" +PARITY_SANDBOX_RUNTIME_IMAGE="" TEMP_WORKTREE="" RUN_DIR="" @@ -108,7 +111,8 @@ cleanup() { # isolated container store from Podman's user namespace before falling back # to ordinary cleanup for runs that never reached the container runtime. if { [ -d "${RUN_DIR}/baseline/data/containers/storage" ] \ - || [ -d "${RUN_DIR}/candidate/data/containers/storage" ]; } \ + || [ -d "${RUN_DIR}/candidate/data/containers/storage" ] \ + || [ -d "${RUN_DIR}/sandbox-resolver/data/containers/storage" ]; } \ && command -v "${PODMAN_BIN}" >/dev/null 2>&1; then "${PODMAN_BIN}" unshare rm -rf -- "${RUN_DIR}" >/dev/null 2>&1 || true fi @@ -334,6 +338,54 @@ if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; exit 2 fi +podman_in_resolver_store() { + local resolver_home="${RUN_DIR}/sandbox-resolver" + mkdir -p "${resolver_home}/config" "${resolver_home}/state" \ + "${resolver_home}/cache" "${resolver_home}/data" + env -u CONTAINER_HOST -u CONTAINER_CONNECTION -u CONTAINERS_STORAGE_CONF \ + -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ + -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ + XDG_CONFIG_HOME="${resolver_home}/config" \ + XDG_STATE_HOME="${resolver_home}/state" \ + XDG_CACHE_HOME="${resolver_home}/cache" \ + XDG_DATA_HOME="${resolver_home}/data" \ + "${PODMAN_BIN}" "$@" +} + +resolve_parity_sandbox_image() { + local image_id image_digest repository + if [ -n "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" ] \ + && ! [[ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE must be digest-pinned for parity runs." >&2 + exit 2 + fi + case "${SANDBOX_IMAGE_REQUEST}" in + */*) ;; + *) + echo "ERROR: parity sandbox image must use a fully qualified repository: ${SANDBOX_IMAGE_REQUEST}" >&2 + exit 2 + ;; + esac + echo "Resolving parity sandbox image once: ${SANDBOX_IMAGE_REQUEST}" + podman_in_resolver_store pull "${SANDBOX_IMAGE_REQUEST}" >/dev/null + image_id="$(podman_in_resolver_store image inspect --format '{{.Id}}' "${SANDBOX_IMAGE_REQUEST}")" + image_id="${image_id#sha256:}" + image_digest="$(podman_in_resolver_store image inspect --format '{{.Digest}}' "${SANDBOX_IMAGE_REQUEST}")" + repository="${SANDBOX_IMAGE_REQUEST%%@*}" + case "${repository##*/}" in + *:*) repository="${repository%:*}" ;; + esac + PARITY_SANDBOX_RUNTIME_IMAGE="${repository}@${image_digest}" + if ! [[ "${image_id}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${PARITY_SANDBOX_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve one immutable parity sandbox image from ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 + fi + echo "Using one immutable parity sandbox image for both variants: ${PARITY_SANDBOX_RUNTIME_IMAGE} (ID ${image_id})" +} + +resolve_parity_sandbox_image + write_result() { local variant=$1 source_sha=$2 schema=$3 status=$4 local gateway_digest=$5 cli_digest=$6 conformance_digest=$7 external_driver_digest_value=$8 supervisor_digest=$9 supervisor_dockerfile_digest=${10} @@ -430,6 +482,7 @@ run_variant() { -u CONTAINER_HOST -u CONTAINER_CONNECTION -u CONTAINERS_STORAGE_CONF \ -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ + -u OPENSHELL_SANDBOX_IMAGE -u OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ @@ -437,6 +490,8 @@ run_variant() { OPENSHELL_E2E_SUPERVISOR_BIN="${supervisor}" \ OPENSHELL_E2E_SUPERVISOR_DOCKERFILE="${supervisor_dockerfile}" \ OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE=1 \ + OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE=1 \ + OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE="${PARITY_SANDBOX_RUNTIME_IMAGE}" \ OPENSHELL_SUPERVISOR_IMAGE="${supervisor_image}" \ OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 5c37099e62..757a64308b 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -79,9 +79,12 @@ for variable in \ OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET \ CONTAINER_HOST CONTAINER_CONNECTION CONTAINERS_STORAGE_CONF CONTAINERS_CONF \ CONTAINERS_REGISTRIES_CONF CONTAINERS_REGISTRIES_CONF_DIR CONTAINERS_POLICY \ - PODMAN_CONNECTIONS_CONF DOCKER_HOST; do + PODMAN_CONNECTIONS_CONF DOCKER_HOST OPENSHELL_SANDBOX_IMAGE; do [ -z "${!variable:-}" ] || exit 23 done +expected_sandbox="ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:$(printf '%064d' 0)" +[ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = 1 ] || exit 24 +[ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" = "${expected_sandbox}" ] || exit 25 printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" case "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" in @@ -120,9 +123,22 @@ cat >"${WORKDIR}/bin/fake-podman" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >>"$OPENSHELL_PARITY_TEST_PODMAN_CALLS" -[ "$1" = unshare ] || exit 19 -shift -exec "$@" +case "$1" in + pull) exit 0 ;; + image) + [ "$2" = inspect ] || exit 19 + case "$4" in + '{{.Id}}') printf 'sha256:%064d\n' 0 ;; + '{{.Digest}}') printf 'sha256:%064d\n' 0 ;; + *) exit 19 ;; + esac + ;; + unshare) + shift + exec "$@" + ;; + *) exit 19 ;; +esac EOF cat >"${WORKDIR}/bin/fake-conformance" <<'EOF' #!/usr/bin/env bash @@ -180,6 +196,7 @@ CONTAINERS_REGISTRIES_CONF_DIR=/tmp/untrusted-registries.d \ CONTAINERS_POLICY=/tmp/untrusted-policy.json \ PODMAN_CONNECTIONS_CONF=/tmp/untrusted-connections.json \ DOCKER_HOST=tcp://untrusted.invalid:2375 \ +OPENSHELL_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \ run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" @@ -194,9 +211,18 @@ assert_contains "${WORKDIR}/results/candidate.json" '"success":true' assert_contains "${WORKDIR}/results/comparison.json" '"parity":true' assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' +assert_contains "${WORKDIR}/podman-calls" 'pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest' assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' +set +e +OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \ + run_harness >"${WORKDIR}/mutable-sandbox.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/mutable-sandbox.out" 'must be digest-pinned for parity runs' + run_harness --scenario external-driver assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/baseline/external-driver|${WORKDIR}/results/artifacts/baseline/supervisor" assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/candidate/external-driver|${WORKDIR}/results/artifacts/candidate/supervisor" diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index ec68731c1b..8d4528cea0 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -227,6 +227,7 @@ def verify_variant( f"{launch_path}: runtime reference digest mismatch", ) + sandbox_request = launch.get("sandbox_image_request") sandbox_id = launch.get("sandbox_image_id") sandbox_digest = launch.get("sandbox_image_digest") sandbox_runtime = launch.get("sandbox_runtime_image") @@ -249,6 +250,10 @@ def verify_variant( and f"sha256:{sandbox_match.group(1)}" == sandbox_digest, f"{launch_path}: sandbox runtime image is not digest-pinned", ) + require( + sandbox_request == sandbox_runtime, + f"{launch_path}: sandbox image request was not the resolved digest reference", + ) if not external: require( podman_config["default_image"] == sandbox_runtime @@ -351,6 +356,21 @@ def verify_topology( baseline = verify_variant(results_dir, "baseline", baseline_sha, 1, scenario) candidate = verify_variant(results_dir, "candidate", candidate_sha, 2, scenario) + baseline_launch = baseline["launch_attestation"] + candidate_launch = candidate["launch_attestation"] + for field, label in ( + ("sandbox_image_id", "sandbox image ID"), + ("sandbox_image_digest", "sandbox image digest"), + ("sandbox_runtime_image", "sandbox runtime image"), + ("supervisor_base_image", "supervisor base image"), + ("supervisor_base_image_id", "supervisor base-image ID"), + ("supervisor_base_image_digest", "supervisor base-image digest"), + ("supervisor_package_manifest_sha256", "supervisor package manifest"), + ): + require( + baseline_launch.get(field) == candidate_launch.get(field), + f"baseline and candidate {label} differ", + ) if scenario == "external-driver": baseline_driver = results_dir / "artifacts/baseline/external-driver" candidate_driver = results_dir / "artifacts/candidate/external-driver" @@ -432,6 +452,8 @@ def main() -> None: "retained_artifact_hashes_recomputed": True, "raw_lifecycle_output_inspected": True, "digest_pinned_supervisor_runtime_verified": True, + "same_immutable_sandbox_verified": True, + "supervisor_dependency_provenance_matched": True, }, } args.output.parent.mkdir(parents=True, exist_ok=True) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 5aa23a7bb0..1c6950ba37 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -488,6 +488,11 @@ echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISO DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ + && ! [[ "${SANDBOX_IMAGE_REQUEST}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: this e2e invocation requires a digest-pinned sandbox image: ${SANDBOX_IMAGE_REQUEST}" >&2 + exit 2 +fi PODMAN_STOP_TIMEOUT_SECS="${OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS:-15}" if ! [[ "${PODMAN_STOP_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then echo "ERROR: OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS must be a non-negative integer." >&2 @@ -510,6 +515,11 @@ if ! [[ "${SANDBOX_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ echo "ERROR: could not resolve an immutable sandbox image for ${SANDBOX_IMAGE_REQUEST}." >&2 exit 2 fi +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ + && [ "${SANDBOX_IMAGE_REQUEST}" != "${SANDBOX_RUNTIME_IMAGE}" ]; then + echo "ERROR: sandbox image digest changed while resolving ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 +fi echo "Using Podman sandbox image: ${SANDBOX_RUNTIME_IMAGE} (ID ${SANDBOX_IMAGE_ID}, digest ${SANDBOX_IMAGE_DIGEST})" PKI_DIR="${WORKDIR}/pki" diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index bb7a755967..51aeb1f8fb 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -114,7 +114,7 @@ def create_variant( "supervisor_base_image_id": IMAGE_ID, "supervisor_base_image_digest": IMAGE_DIGEST, "supervisor_package_manifest_sha256": package_hash, - "sandbox_image_request": "example.invalid/sandbox:fixture", + "sandbox_image_request": "example.invalid/sandbox@" + IMAGE_DIGEST, "sandbox_image_id": IMAGE_ID, "sandbox_image_digest": IMAGE_DIGEST, "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, @@ -163,3 +163,64 @@ def test_verifier_recomputes_retained_artifact_hashes(tmp_path: Path) -> None: verifier.verify_topology( tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" ) + + +def test_verifier_rejects_mutable_sandbox_request(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["sandbox_image_request"] = "example.invalid/sandbox:latest" + write_json(launch_path, launch) + + with pytest.raises( + ValueError, match="request was not the resolved digest reference" + ): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_different_sandbox_artifacts(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + other_id = "5" * 64 + other_digest = f"sha256:{'6' * 64}" + other_runtime = f"example.invalid/sandbox@{other_digest}" + launch.update( + { + "sandbox_image_request": other_runtime, + "sandbox_image_id": other_id, + "sandbox_image_digest": other_digest, + "sandbox_runtime_image": other_runtime, + } + ) + write_json(launch_path, launch) + with (tmp_path / "candidate.log").open("a", encoding="utf-8") as log: + log.write(f"{other_id} {other_digest} {other_runtime}\n") + + with pytest.raises(ValueError, match="sandbox image ID differ"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_different_supervisor_packages(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + package_path = tmp_path / "artifacts/candidate/supervisor.packages.txt" + package_path.write_text("fixture-package-2.0-r0\n", encoding="utf-8") + package_hash = verifier.sha256(package_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["supervisor_package_manifest_sha256"] = package_hash + write_json(launch_path, launch) + with (tmp_path / "candidate.log").open("a", encoding="utf-8") as log: + log.write(f"{package_hash}\n") + + with pytest.raises(ValueError, match="supervisor package manifest differ"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) From 7e40c2fb357661f14003bc331c8393b65d6be014 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 08:31:05 -0400 Subject: [PATCH 33/42] test(e2e): attest parity runtime inputs Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 53 ++++++++ e2e/parity/test.sh | 30 ++++- e2e/parity/verify-results.py | 114 +++++++++++++++++- e2e/with-podman-gateway.sh | 85 ++++++++++++- ...chema_v2_compute_boundary_verifier_test.py | 51 +++++++- 5 files changed, 323 insertions(+), 10 deletions(-) diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 36cd5ebe83..537f2ad366 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -22,6 +22,9 @@ PODMAN_BIN="${OPENSHELL_PARITY_PODMAN_BIN:-podman}" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}" PARITY_SANDBOX_RUNTIME_IMAGE="" +PARITY_SANDBOX_COMMUNITY_REGISTRY="" +PARITY_SUPERVISOR_BASE_IMAGE="" +PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE="" TEMP_WORKTREE="" RUN_DIR="" @@ -381,10 +384,35 @@ resolve_parity_sandbox_image() { echo "ERROR: could not resolve one immutable parity sandbox image from ${SANDBOX_IMAGE_REQUEST}." >&2 exit 2 fi + PARITY_SANDBOX_COMMUNITY_REGISTRY="${repository%/base}" + if [ "${PARITY_SANDBOX_COMMUNITY_REGISTRY}" = "${repository}" ]; then + echo "ERROR: parity sandbox repository must end in /base for the conformance alias: ${repository}." >&2 + exit 2 + fi echo "Using one immutable parity sandbox image for both variants: ${PARITY_SANDBOX_RUNTIME_IMAGE} (ID ${image_id})" } +resolve_parity_supervisor_base_image() { + local baseline_base candidate_base + baseline_base="$(awk '$1 == "FROM" { print $2; exit }' "${BASELINE_SUPERVISOR_DOCKERFILE}")" + candidate_base="$(awk '$1 == "FROM" { print $2; exit }' "${CANDIDATE_SUPERVISOR_DOCKERFILE}")" + if [ -z "${baseline_base}" ] || [ "${baseline_base}" != "${candidate_base}" ]; then + echo "ERROR: parity supervisor Dockerfiles must select the same base image." >&2 + exit 2 + fi + PARITY_SUPERVISOR_BASE_IMAGE="${baseline_base}" + echo "Resolving parity supervisor base image once: ${PARITY_SUPERVISOR_BASE_IMAGE}" + podman_in_resolver_store pull "${PARITY_SUPERVISOR_BASE_IMAGE}" >/dev/null + PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE="$(podman_in_resolver_store image inspect --format '{{index .RepoDigests 0}}' "${PARITY_SUPERVISOR_BASE_IMAGE}")" + if ! [[ "${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve one immutable supervisor base image from ${PARITY_SUPERVISOR_BASE_IMAGE}." >&2 + exit 2 + fi + echo "Using one immutable supervisor base image for both variants: ${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" +} + resolve_parity_sandbox_image +resolve_parity_supervisor_base_image write_result() { local variant=$1 source_sha=$2 schema=$3 status=$4 @@ -476,6 +504,14 @@ run_variant() { else command=("${conformance}" run --openshell-bin "${cli}" --output json) fi + verify_artifact_digest "${variant} gateway before execution" "${gateway}" "${gateway_digest}" || return 1 + verify_artifact_digest "${variant} CLI before execution" "${cli}" "${cli_digest}" || return 1 + verify_artifact_digest "${variant} conformance CLI before execution" "${conformance}" "${conformance_digest}" || return 1 + verify_artifact_digest "${variant} supervisor before execution" "${supervisor}" "${supervisor_digest}" || return 1 + verify_artifact_digest "${variant} supervisor Dockerfile before execution" "${supervisor_dockerfile}" "${supervisor_dockerfile_digest}" || return 1 + if [ -n "${external_driver}" ]; then + verify_artifact_digest "${variant} external driver before execution" "${external_driver}" "${external_driver_digest}" || return 1 + fi if env -u OPENSHELL_GATEWAY_ENDPOINT -u OPENSHELL_GATEWAY_CONFIG \ -u OPENSHELL_COMPUTE_DRIVER -u OPENSHELL_COMPUTE_DRIVER_SOCKET -u OPENSHELL_DRIVERS \ -u OPENSHELL_PODMAN_SOCKET \ @@ -483,6 +519,14 @@ run_variant() { -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ -u OPENSHELL_SANDBOX_IMAGE -u OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE \ + -u OPENSHELL_GRPC_ENDPOINT -u OPENSHELL_PODMAN_HOST_GATEWAY_IP \ + -u OPENSHELL_PODMAN_USERNS -u OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET \ + -u OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET -u OPENSHELL_APP_ARMOR_PROFILE \ + -u OPENSHELL_SANDBOX_HTTPS_PROXY -u OPENSHELL_SANDBOX_NO_PROXY \ + -u OPENSHELL_SANDBOX_PROXY_AUTH_FILE -u OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE \ + -u OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME -u OPENSHELL_SANDBOX_PROXY_CA_BUNDLE \ + -u OPENSHELL_OTLP_ENDPOINT -u OPENSHELL_GATEWAY_NAME -u OPENSHELL_COMPUTE_DRIVER_BIND \ + -u OPENSHELL_COMMUNITY_REGISTRY \ OPENSHELL_PARITY_VARIANT="${variant}" \ OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ @@ -492,6 +536,15 @@ run_variant() { OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE=1 \ OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE=1 \ OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE="${PARITY_SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_COMMUNITY_REGISTRY="${PARITY_SANDBOX_COMMUNITY_REGISTRY}" \ + OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE="${PARITY_SUPERVISOR_BASE_IMAGE}" \ + OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE="${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" \ + OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256="${gateway_digest}" \ + OPENSHELL_E2E_EXPECTED_CLI_SHA256="${cli_digest}" \ + OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256="${conformance_digest}" \ + OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256="${external_driver_digest}" \ + OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256="${supervisor_digest}" \ + OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256="${supervisor_dockerfile_digest}" \ OPENSHELL_SUPERVISOR_IMAGE="${supervisor_image}" \ OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index 757a64308b..c66513ac5b 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -79,12 +79,22 @@ for variable in \ OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET \ CONTAINER_HOST CONTAINER_CONNECTION CONTAINERS_STORAGE_CONF CONTAINERS_CONF \ CONTAINERS_REGISTRIES_CONF CONTAINERS_REGISTRIES_CONF_DIR CONTAINERS_POLICY \ - PODMAN_CONNECTIONS_CONF DOCKER_HOST OPENSHELL_SANDBOX_IMAGE; do + PODMAN_CONNECTIONS_CONF DOCKER_HOST OPENSHELL_SANDBOX_IMAGE \ + OPENSHELL_GRPC_ENDPOINT OPENSHELL_PODMAN_HOST_GATEWAY_IP OPENSHELL_PODMAN_USERNS \ + OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET \ + OPENSHELL_APP_ARMOR_PROFILE OPENSHELL_SANDBOX_HTTPS_PROXY OPENSHELL_SANDBOX_NO_PROXY \ + OPENSHELL_SANDBOX_PROXY_AUTH_FILE OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE \ + OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME OPENSHELL_SANDBOX_PROXY_CA_BUNDLE \ + OPENSHELL_OTLP_ENDPOINT OPENSHELL_GATEWAY_NAME OPENSHELL_COMPUTE_DRIVER_BIND; do [ -z "${!variable:-}" ] || exit 23 done expected_sandbox="ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:$(printf '%064d' 0)" [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = 1 ] || exit 24 [ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" = "${expected_sandbox}" ] || exit 25 +[ "${OPENSHELL_COMMUNITY_REGISTRY:-}" = "ghcr.io/nvidia/openshell-community/sandboxes" ] || exit 28 +expected_base="docker.io/library/alpine@sha256:$(printf '%064d' 0)" +[ "${OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE:-}" = alpine:3.22 ] || exit 26 +[ "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-}" = "${expected_base}" ] || exit 27 printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" mkdir -p "$XDG_DATA_HOME/containers/storage" case "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" in @@ -130,6 +140,7 @@ case "$1" in case "$4" in '{{.Id}}') printf 'sha256:%064d\n' 0 ;; '{{.Digest}}') printf 'sha256:%064d\n' 0 ;; + '{{index .RepoDigests 0}}') printf 'docker.io/library/alpine@sha256:%064d\n' 0 ;; *) exit 19 ;; esac ;; @@ -197,6 +208,22 @@ CONTAINERS_POLICY=/tmp/untrusted-policy.json \ PODMAN_CONNECTIONS_CONF=/tmp/untrusted-connections.json \ DOCKER_HOST=tcp://untrusted.invalid:2375 \ OPENSHELL_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \ +OPENSHELL_GRPC_ENDPOINT=http://untrusted.invalid:1 \ +OPENSHELL_PODMAN_HOST_GATEWAY_IP=192.0.2.1 \ +OPENSHELL_PODMAN_USERNS=keep-id \ +OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/tmp/untrusted-spiffe.sock \ +OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET=/tmp/untrusted-e2e-spiffe.sock \ +OPENSHELL_APP_ARMOR_PROFILE=Unconfined \ +OPENSHELL_SANDBOX_HTTPS_PROXY=http://untrusted.invalid:8080 \ +OPENSHELL_SANDBOX_NO_PROXY=untrusted.invalid \ +OPENSHELL_SANDBOX_PROXY_AUTH_FILE=/tmp/untrusted-proxy-auth \ +OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE=true \ +OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME=true \ +OPENSHELL_SANDBOX_PROXY_CA_BUNDLE=/tmp/untrusted-proxy-ca \ +OPENSHELL_OTLP_ENDPOINT=http://untrusted.invalid:4317 \ +OPENSHELL_GATEWAY_NAME=untrusted \ +OPENSHELL_COMPUTE_DRIVER_BIND=192.0.2.2:50061 \ +OPENSHELL_COMMUNITY_REGISTRY=untrusted.invalid/community \ run_harness assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" @@ -212,6 +239,7 @@ assert_contains "${WORKDIR}/results/comparison.json" '"parity":true' assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' assert_contains "${WORKDIR}/podman-calls" 'pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest' +assert_contains "${WORKDIR}/podman-calls" 'pull alpine:3.22' assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index 8d4528cea0..20d7be1e97 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -261,6 +261,16 @@ def verify_variant( f"{config_path}: runtime image references differ from launch evidence", ) + base_runtime = launch.get("supervisor_base_runtime_image") + base_runtime_match = ( + DIGEST_REFERENCE_RE.fullmatch(base_runtime) + if isinstance(base_runtime, str) + else None + ) + require( + base_runtime_match is not None, + f"{launch_path}: supervisor base runtime image is not digest-pinned", + ) for field in ("supervisor_base_image_id", "supervisor_package_manifest_sha256"): value = launch.get(field) require( @@ -281,6 +291,56 @@ def verify_variant( ) artifact_hashes["supervisor.packages.txt"] = sha256(package_path) + sandbox_alias = launch.get("sandbox_client_image_alias") + require( + isinstance(sandbox_alias, str) + and sandbox_alias == sandbox_runtime.rsplit("@", 1)[0] + ":latest" + and launch.get("sandbox_client_image_alias_id") == sandbox_id, + f"{launch_path}: sandbox client alias is not bound to the pinned image", + ) + + for launch_field, result_field in ( + ("gateway_sha256_before_execution", "gateway_sha256"), + ("cli_sha256_before_execution", "cli_sha256"), + ("conformance_sha256_before_execution", "conformance_sha256"), + ("supervisor_sha256_before_execution", "supervisor_sha256"), + ( + "supervisor_dockerfile_sha256_before_execution", + "supervisor_dockerfile_sha256", + ), + ): + require( + launch.get(launch_field) == result.get(result_field), + f"{launch_path}: {launch_field} does not bind the staged artifact", + ) + if external: + require( + launch.get("external_driver_sha256_before_execution") + == result.get("external_driver_sha256"), + f"{launch_path}: external driver pre-execution hash mismatch", + ) + gateway_port = launch.get("gateway_port") + require( + isinstance(gateway_port, int) + and 0 < gateway_port <= 65535 + and launch.get("external_driver_grpc_endpoint") + == f"https://host.containers.internal:{gateway_port}", + f"{launch_path}: external driver callback endpoint is not isolated", + ) + require( + launch.get("external_driver_host_gateway_ip") == "host-gateway" + and launch.get("external_driver_userns") is None + and launch.get("external_driver_spiffe") is False + and launch.get("external_driver_proxy") is False + and launch.get("external_driver_app_armor") is False, + f"{launch_path}: external driver effective configuration is tainted", + ) + else: + require( + launch.get("external_driver_sha256_before_execution") == "", + f"{launch_path}: unexpected external driver hash", + ) + require(log_path.is_file(), f"{log_path}: retained raw log is missing") raw_log = log_path.read_text(encoding="utf-8", errors="replace") for marker in ORACLE_MARKERS: @@ -299,8 +359,11 @@ def verify_variant( sandbox_runtime, sandbox_id, sandbox_digest, + sandbox_alias, + launch["sandbox_client_image_alias_id"], launch["supervisor_base_image_id"], base_digest, + base_runtime, launch["supervisor_package_manifest_sha256"], ) require( @@ -398,6 +461,41 @@ def verify_topology( } +def verify_four_run_provenance( + in_tree: dict[str, Any], external_uds: dict[str, Any] +) -> None: + variants = [ + in_tree["baseline"]["launch_attestation"], + in_tree["candidate"]["launch_attestation"], + external_uds["baseline"]["launch_attestation"], + external_uds["candidate"]["launch_attestation"], + ] + for fields, label in ( + ( + ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "sandbox_client_image_alias", + "sandbox_client_image_alias_id", + ), + "sandbox artifact", + ), + ( + ( + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_base_runtime_image", + "supervisor_package_manifest_sha256", + ), + "supervisor dependency provenance", + ), + ): + tuples = {tuple(launch.get(field) for field in fields) for launch in variants} + require(len(tuples) == 1, f"the four runs use different {label}") + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--baseline-sha", required=True) @@ -419,6 +517,14 @@ def main() -> None: "invalid candidate SHA", ) + in_tree = verify_topology( + args.in_tree, args.baseline_sha, args.candidate_sha, "smoke" + ) + external_uds = verify_topology( + args.external_uds, args.baseline_sha, args.candidate_sha, "external-driver" + ) + verify_four_run_provenance(in_tree, external_uds) + report = { "manifest_version": 2, "baseline_commit": args.baseline_sha, @@ -433,12 +539,8 @@ def main() -> None: "delete": True, "list_empty": True, }, - "in_tree": verify_topology( - args.in_tree, args.baseline_sha, args.candidate_sha, "smoke" - ), - "external_uds": verify_topology( - args.external_uds, args.baseline_sha, args.candidate_sha, "external-driver" - ), + "in_tree": in_tree, + "external_uds": external_uds, "callback_listener": { "in_tree_baseline_exec": True, "in_tree_candidate_exec": True, diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 1c6950ba37..4267fbc448 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -85,6 +85,20 @@ podman_cmd() { with_podman_config podman "$@" } +require_expected_sha256() { + local label=$1 path=$2 expected=$3 actual + [ -n "${expected}" ] || return 0 + if [ ! -f "${path}" ]; then + echo "ERROR: ${label} is missing before execution: ${path}" >&2 + exit 2 + fi + actual="$(sha256sum "${path}" | cut -d' ' -f1)" + if [ "${actual}" != "${expected}" ]; then + echo "ERROR: ${label} hash changed before execution." >&2 + exit 2 + fi +} + WORKDIR_PARENT="${TMPDIR:-/tmp}" WORKDIR_PARENT="${WORKDIR_PARENT%/}" WORKDIR="$(mktemp -d "${WORKDIR_PARENT}/openshell-e2e-podman.XXXXXX")" @@ -347,14 +361,33 @@ ensure_podman_supervisor_image() { echo "ERROR: supervisor Dockerfile not found: ${dockerfile}" >&2 exit 2 fi + require_expected_sha256 "supervisor binary" "${OPENSHELL_E2E_SUPERVISOR_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256:-}" + require_expected_sha256 "supervisor Dockerfile" "${dockerfile}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256:-}" mkdir -p "${context}/deploy/docker/.build/prebuilt-binaries/${arch}" install -m 0555 "${OPENSHELL_E2E_SUPERVISOR_BIN}" \ "${context}/deploy/docker/.build/prebuilt-binaries/${arch}/openshell-sandbox" cp "${dockerfile}" "${context}/deploy/docker/Dockerfile.supervisor" + local -a pull_option=() + if [ -n "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-}" ]; then + local dockerfile_base + dockerfile_base="$(awk '$1 == "FROM" { print $2; exit }' "${dockerfile}")" + if [ "${dockerfile_base}" != "${OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE:-}" ] \ + || ! [[ "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: supervisor base-image attestation does not match the Dockerfile." >&2 + exit 2 + fi + echo "Pulling pinned supervisor base image ${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}..." + podman_cmd pull "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" + podman_cmd tag "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" "${dockerfile_base}" + pull_option=(--pull=never) + fi echo "Building Podman supervisor image ${image} from supplied binary..." ( cd "${context}" podman_cmd build \ + "${pull_option[@]}" \ --build-arg "TARGETARCH=${arch}" \ --file deploy/docker/Dockerfile.supervisor \ --target supervisor \ @@ -520,7 +553,19 @@ if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ echo "ERROR: sandbox image digest changed while resolving ${SANDBOX_IMAGE_REQUEST}." >&2 exit 2 fi -echo "Using Podman sandbox image: ${SANDBOX_RUNTIME_IMAGE} (ID ${SANDBOX_IMAGE_ID}, digest ${SANDBOX_IMAGE_DIGEST})" +SANDBOX_CLIENT_IMAGE_ALIAS="" +SANDBOX_CLIENT_IMAGE_ALIAS_ID="" +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ]; then + SANDBOX_CLIENT_IMAGE_ALIAS="${SANDBOX_IMAGE_REPOSITORY}:latest" + podman_cmd tag "${SANDBOX_RUNTIME_IMAGE}" "${SANDBOX_CLIENT_IMAGE_ALIAS}" + SANDBOX_CLIENT_IMAGE_ALIAS_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SANDBOX_CLIENT_IMAGE_ALIAS}")" + SANDBOX_CLIENT_IMAGE_ALIAS_ID="${SANDBOX_CLIENT_IMAGE_ALIAS_ID#sha256:}" + if [ "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" != "${SANDBOX_IMAGE_ID}" ]; then + echo "ERROR: sandbox client alias does not resolve to the pinned sandbox image." >&2 + exit 2 + fi +fi +echo "Using Podman sandbox image: ${SANDBOX_RUNTIME_IMAGE} (ID ${SANDBOX_IMAGE_ID}, digest ${SANDBOX_IMAGE_DIGEST}, client alias ${SANDBOX_CLIENT_IMAGE_ALIAS:-none} ID ${SANDBOX_CLIENT_IMAGE_ALIAS_ID:-none})" PKI_DIR="${WORKDIR}/pki" e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" "host.containers.internal" @@ -578,11 +623,20 @@ if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then fi if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then driver_transport=in_tree + external_driver_grpc_endpoint=null + external_driver_host_gateway_ip=null + external_driver_userns=null + external_driver_spiffe=false + external_driver_proxy=false + external_driver_app_armor=false if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds + external_driver_grpc_endpoint="\"https://host.containers.internal:${HOST_PORT}\"" + external_driver_host_gateway_ip='"host-gateway"' fi - printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s"}\n' \ + printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s}\n' \ "${CONFIG_SCHEMA_VERSION}" \ + "${HOST_PORT}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ "${driver_transport}" \ "${EXTERNAL_DRIVER_PULL_POLICY}" \ @@ -593,20 +647,39 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${SUPERVISOR_BASE_IMAGE}" \ "${SUPERVISOR_BASE_IMAGE_ID}" \ "${SUPERVISOR_BASE_IMAGE_DIGEST}" \ + "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-${SUPERVISOR_BASE_IMAGE}@${SUPERVISOR_BASE_IMAGE_DIGEST}}" \ "${SUPERVISOR_PACKAGE_MANIFEST_SHA256}" \ "${SANDBOX_IMAGE_REQUEST}" \ "${SANDBOX_IMAGE_ID}" \ "${SANDBOX_IMAGE_DIGEST}" \ "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_CLIENT_IMAGE_ALIAS}" \ + "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" \ + "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256:-}" \ + "${external_driver_grpc_endpoint}" \ + "${external_driver_host_gateway_ip}" \ + "${external_driver_userns}" \ + "${external_driver_spiffe}" \ + "${external_driver_proxy}" \ + "${external_driver_app_armor}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + require_expected_sha256 "external compute driver" "${DRIVER_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" + env -i \ OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ + OPENSHELL_GRPC_ENDPOINT="https://host.containers.internal:${HOST_PORT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ @@ -653,6 +726,8 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_LOG}" \ "${GATEWAY_PID_FILE}" +require_expected_sha256 "gateway binary" "${GATEWAY_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ @@ -704,5 +779,11 @@ if [ "${elapsed}" -ge "${timeout}" ]; then exit 1 fi +require_expected_sha256 "OpenShell CLI" "${CLI_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_SHA256:-}" +if [ -n "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256:-}" ]; then + require_expected_sha256 "conformance CLI" "${OPENSHELL_CONFORMANCE_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256}" +fi echo "Running e2e command against ${CLI_GATEWAY_ENDPOINT}: $*" "$@" diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index 51aeb1f8fb..48257b2f01 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -22,6 +22,7 @@ IMAGE_ID = "3" * 64 IMAGE_DIGEST = f"sha256:{'4' * 64}" RUNTIME_IMAGE = f"localhost/openshell/supervisor@{IMAGE_DIGEST}" +BASE_RUNTIME_IMAGE = f"docker.io/library/alpine@{IMAGE_DIGEST}" def load_verifier() -> ModuleType: @@ -100,10 +101,12 @@ def create_variant( ) policy = "missing" if schema_version == 1 else "if_not_present" package_hash = verifier.sha256(artifact_dir / "supervisor.packages.txt") + result = json.loads((results_dir / f"{variant}.json").read_text(encoding="utf-8")) write_json( results_dir / f"{variant}.launch.json", { "schema_version": schema_version, + "gateway_port": 18181, "external_compute_driver": True, "compute_driver_transport": "remote_uds", "external_driver_pull_policy": policy, @@ -113,18 +116,35 @@ def create_variant( "supervisor_base_image": "alpine:fixture", "supervisor_base_image_id": IMAGE_ID, "supervisor_base_image_digest": IMAGE_DIGEST, + "supervisor_base_runtime_image": BASE_RUNTIME_IMAGE, "supervisor_package_manifest_sha256": package_hash, "sandbox_image_request": "example.invalid/sandbox@" + IMAGE_DIGEST, "sandbox_image_id": IMAGE_ID, "sandbox_image_digest": IMAGE_DIGEST, "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_client_image_alias": "example.invalid/sandbox:latest", + "sandbox_client_image_alias_id": IMAGE_ID, + "gateway_sha256_before_execution": result["gateway_sha256"], + "cli_sha256_before_execution": result["cli_sha256"], + "conformance_sha256_before_execution": result["conformance_sha256"], + "external_driver_sha256_before_execution": result["external_driver_sha256"], + "supervisor_sha256_before_execution": result["supervisor_sha256"], + "supervisor_dockerfile_sha256_before_execution": result[ + "supervisor_dockerfile_sha256" + ], + "external_driver_grpc_endpoint": "https://host.containers.internal:18181", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": None, + "external_driver_spiffe": False, + "external_driver_proxy": False, + "external_driver_app_armor": False, }, ) lifecycle = "\n".join( f"[run fixture{marker} in 1ms: exit 0" for marker in verifier.ORACLE_MARKERS ) (results_dir / f"{variant}.log").write_text( - f"{lifecycle}\n{RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} " + f"{lifecycle}\n{RUNTIME_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', encoding="utf-8", ) @@ -195,6 +215,7 @@ def test_verifier_rejects_different_sandbox_artifacts(tmp_path: Path) -> None: "sandbox_image_id": other_id, "sandbox_image_digest": other_digest, "sandbox_runtime_image": other_runtime, + "sandbox_client_image_alias_id": other_id, } ) write_json(launch_path, launch) @@ -224,3 +245,31 @@ def test_verifier_rejects_different_supervisor_packages(tmp_path: Path) -> None: verifier.verify_topology( tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" ) + + +def test_verifier_rejects_cross_topology_sandbox_drift() -> None: + verifier = load_verifier() + launch = { + "sandbox_image_id": IMAGE_ID, + "sandbox_image_digest": IMAGE_DIGEST, + "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_client_image_alias": "example.invalid/sandbox:latest", + "sandbox_client_image_alias_id": IMAGE_ID, + "supervisor_base_image": "alpine:fixture", + "supervisor_base_image_id": IMAGE_ID, + "supervisor_base_image_digest": IMAGE_DIGEST, + "supervisor_base_runtime_image": BASE_RUNTIME_IMAGE, + "supervisor_package_manifest_sha256": "7" * 64, + } + in_tree = { + "baseline": {"launch_attestation": dict(launch)}, + "candidate": {"launch_attestation": dict(launch)}, + } + external = { + "baseline": {"launch_attestation": dict(launch)}, + "candidate": {"launch_attestation": dict(launch)}, + } + external["candidate"]["launch_attestation"]["sandbox_image_id"] = "8" * 64 + + with pytest.raises(ValueError, match="four runs use different sandbox artifact"): + verifier.verify_four_run_provenance(in_tree, external) From 4688e8418e9b2128757fdf789c532ab2378fae2f Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 09:04:26 -0400 Subject: [PATCH 34/42] test(e2e): bind parity runtime evidence Signed-off-by: Jesse Jaggars --- e2e/parity/run.sh | 35 +++- e2e/parity/test.sh | 19 ++ e2e/parity/trace-cli.sh | 31 ++++ e2e/parity/verify-results.py | 172 +++++++++++++++++- e2e/with-podman-gateway.sh | 49 ++++- ...chema_v2_compute_boundary_verifier_test.py | 149 ++++++++++++++- 6 files changed, 425 insertions(+), 30 deletions(-) create mode 100755 e2e/parity/trace-cli.sh diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh index 537f2ad366..b83784d7f7 100755 --- a/e2e/parity/run.sh +++ b/e2e/parity/run.sh @@ -314,18 +314,20 @@ stage_executable() { stage_artifact "$1" "$2" "$3" 0555 "$4" "$5" } -BASELINE_GATEWAY_DIGEST="" BASELINE_CLI_DIGEST="" BASELINE_CONFORMANCE_DIGEST="" BASELINE_EXTERNAL_DRIVER_DIGEST="" BASELINE_SUPERVISOR_DIGEST="" BASELINE_SUPERVISOR_DOCKERFILE="" BASELINE_SUPERVISOR_DOCKERFILE_DIGEST="" -CANDIDATE_GATEWAY_DIGEST="" CANDIDATE_CLI_DIGEST="" CANDIDATE_CONFORMANCE_DIGEST="" CANDIDATE_EXTERNAL_DRIVER_DIGEST="" CANDIDATE_SUPERVISOR_DIGEST="" CANDIDATE_SUPERVISOR_DOCKERFILE="" CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST="" +BASELINE_GATEWAY_DIGEST="" BASELINE_CLI_DIGEST="" BASELINE_CONFORMANCE_DIGEST="" BASELINE_EXTERNAL_DRIVER_DIGEST="" BASELINE_SUPERVISOR_DIGEST="" BASELINE_SUPERVISOR_DOCKERFILE="" BASELINE_SUPERVISOR_DOCKERFILE_DIGEST="" BASELINE_CLI_TRACE_WRAPPER="" BASELINE_CLI_TRACE_WRAPPER_DIGEST="" +CANDIDATE_GATEWAY_DIGEST="" CANDIDATE_CLI_DIGEST="" CANDIDATE_CONFORMANCE_DIGEST="" CANDIDATE_EXTERNAL_DRIVER_DIGEST="" CANDIDATE_SUPERVISOR_DIGEST="" CANDIDATE_SUPERVISOR_DOCKERFILE="" CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST="" CANDIDATE_CLI_TRACE_WRAPPER="" CANDIDATE_CLI_TRACE_WRAPPER_DIGEST="" stage_executable baseline gateway "${BASELINE_GATEWAY}" BASELINE_GATEWAY BASELINE_GATEWAY_DIGEST stage_executable baseline cli "${BASELINE_CLI}" BASELINE_CLI BASELINE_CLI_DIGEST stage_executable baseline conformance "${BASELINE_CONFORMANCE}" BASELINE_CONFORMANCE BASELINE_CONFORMANCE_DIGEST stage_executable baseline supervisor "${BASELINE_SUPERVISOR}" BASELINE_SUPERVISOR BASELINE_SUPERVISOR_DIGEST stage_artifact baseline supervisor.Dockerfile "${BASELINE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 BASELINE_SUPERVISOR_DOCKERFILE BASELINE_SUPERVISOR_DOCKERFILE_DIGEST +stage_artifact baseline cli-trace-wrapper "${ROOT}/e2e/parity/trace-cli.sh" 0555 BASELINE_CLI_TRACE_WRAPPER BASELINE_CLI_TRACE_WRAPPER_DIGEST stage_executable candidate gateway "${CANDIDATE_GATEWAY}" CANDIDATE_GATEWAY CANDIDATE_GATEWAY_DIGEST stage_executable candidate cli "${CANDIDATE_CLI}" CANDIDATE_CLI CANDIDATE_CLI_DIGEST stage_executable candidate conformance "${CANDIDATE_CONFORMANCE}" CANDIDATE_CONFORMANCE CANDIDATE_CONFORMANCE_DIGEST stage_executable candidate supervisor "${CANDIDATE_SUPERVISOR}" CANDIDATE_SUPERVISOR CANDIDATE_SUPERVISOR_DIGEST stage_artifact candidate supervisor.Dockerfile "${CANDIDATE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 CANDIDATE_SUPERVISOR_DOCKERFILE CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST +stage_artifact candidate cli-trace-wrapper "${ROOT}/e2e/parity/trace-cli.sh" 0555 CANDIDATE_CLI_TRACE_WRAPPER CANDIDATE_CLI_TRACE_WRAPPER_DIGEST if [ "${SCENARIO}" = external-driver ]; then stage_executable baseline external-driver "${BASELINE_EXTERNAL_DRIVER}" BASELINE_EXTERNAL_DRIVER BASELINE_EXTERNAL_DRIVER_DIGEST stage_executable candidate external-driver "${CANDIDATE_EXTERNAL_DRIVER}" CANDIDATE_EXTERNAL_DRIVER CANDIDATE_EXTERNAL_DRIVER_DIGEST @@ -416,7 +418,7 @@ resolve_parity_supervisor_base_image write_result() { local variant=$1 source_sha=$2 schema=$3 status=$4 - local gateway_digest=$5 cli_digest=$6 conformance_digest=$7 external_driver_digest_value=$8 supervisor_digest=$9 supervisor_dockerfile_digest=${10} + local gateway_digest=$5 cli_digest=$6 conformance_digest=$7 external_driver_digest_value=$8 supervisor_digest=$9 supervisor_dockerfile_digest=${10} cli_trace_wrapper_digest=${11} local normalized_result="" external_driver_digest="" gateway_profile="in-tree" local gateway_features=default local gateway_origin cli_origin conformance_origin external_driver_origin supervisor_origin @@ -440,7 +442,7 @@ write_result() { fi if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi cat >"${RESULTS_DIR}/${variant}.json" <&2 + result_status=false + fi + if [ "${SCENARIO}" = external-driver ] && [ ! -s "${RESULTS_DIR}/${variant}.driver.log" ]; then + echo "ERROR: ${variant} external driver log was not retained." >&2 + result_status=false + fi + write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${gateway_digest}" "${cli_digest}" "${conformance_digest}" "${external_driver_digest}" "${supervisor_digest}" "${supervisor_dockerfile_digest}" "${cli_trace_wrapper_digest}" [ "${result_status}" = true ] } baseline_exit=0 candidate_exit=0 -run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" "${BASELINE_SUPERVISOR}" "${BASELINE_SUPERVISOR_DOCKERFILE}" "${BASELINE_GATEWAY_DIGEST}" "${BASELINE_CLI_DIGEST}" "${BASELINE_CONFORMANCE_DIGEST}" "${BASELINE_EXTERNAL_DRIVER_DIGEST}" "${BASELINE_SUPERVISOR_DIGEST}" "${BASELINE_SUPERVISOR_DOCKERFILE_DIGEST}" || baseline_exit=$? +run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" "${BASELINE_SUPERVISOR}" "${BASELINE_SUPERVISOR_DOCKERFILE}" "${BASELINE_GATEWAY_DIGEST}" "${BASELINE_CLI_DIGEST}" "${BASELINE_CONFORMANCE_DIGEST}" "${BASELINE_EXTERNAL_DRIVER_DIGEST}" "${BASELINE_SUPERVISOR_DIGEST}" "${BASELINE_SUPERVISOR_DOCKERFILE_DIGEST}" "${BASELINE_CLI_TRACE_WRAPPER}" "${BASELINE_CLI_TRACE_WRAPPER_DIGEST}" || baseline_exit=$? # Do not short-circuit: a candidate result is useful even when the frozen # baseline failed, and two equal failures must never constitute parity. -run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" "${CANDIDATE_SUPERVISOR}" "${CANDIDATE_SUPERVISOR_DOCKERFILE}" "${CANDIDATE_GATEWAY_DIGEST}" "${CANDIDATE_CLI_DIGEST}" "${CANDIDATE_CONFORMANCE_DIGEST}" "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" "${CANDIDATE_SUPERVISOR_DIGEST}" "${CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST}" || candidate_exit=$? +run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" "${CANDIDATE_SUPERVISOR}" "${CANDIDATE_SUPERVISOR_DOCKERFILE}" "${CANDIDATE_GATEWAY_DIGEST}" "${CANDIDATE_CLI_DIGEST}" "${CANDIDATE_CONFORMANCE_DIGEST}" "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" "${CANDIDATE_SUPERVISOR_DIGEST}" "${CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST}" "${CANDIDATE_CLI_TRACE_WRAPPER}" "${CANDIDATE_CLI_TRACE_WRAPPER_DIGEST}" || candidate_exit=$? baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh index c66513ac5b..5a4541930d 100755 --- a/e2e/parity/test.sh +++ b/e2e/parity/test.sh @@ -64,6 +64,21 @@ set -e assert_status "${status}" 2 assert_contains "${WORKDIR}/wrapper-schema.out" 'must be 1 or 2' +cat >"${WORKDIR}/trace-cli-fixture" <<'EOF' +#!/usr/bin/env bash +printf 'openshell-conformance-tracefixture\n' +printf 'trace fixture stderr\n' >&2 +EOF +chmod +x "${WORKDIR}/trace-cli-fixture" +OPENSHELL_PARITY_REAL_CLI="${WORKDIR}/trace-cli-fixture" \ +OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE="${WORKDIR}/trace-cli.stdout" \ + bash "${ROOT}/e2e/parity/trace-cli.sh" sandbox exec -- echo marker \ + >"${WORKDIR}/trace-cli.forwarded.stdout" \ + 2>"${WORKDIR}/trace-cli.forwarded.stderr" +cmp -s "${WORKDIR}/trace-cli.stdout" "${WORKDIR}/trace-cli.forwarded.stdout" \ + || fail 'CLI trace wrapper did not preserve exact exec stdout' +assert_contains "${WORKDIR}/trace-cli.forwarded.stderr" 'trace fixture stderr' + HEAD_SHA="$(git -C "${ROOT}" rev-parse HEAD)" cat >"${WORKDIR}/manifest.toml" <"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" printf 'fixture-package-1.0-r0\n' >"${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE}" +printf 'fixture exec stdout\n' >"${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then + printf 'fixture external driver log\n' >"${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE}" +fi if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then replacement="${OPENSHELL_GATEWAY_BIN}.replacement" printf '#!/usr/bin/env bash\nexit 0\n# mutated\n' >"${replacement}" diff --git a/e2e/parity/trace-cli.sh b/e2e/parity/trace-cli.sh new file mode 100755 index 0000000000..372c1b1d3a --- /dev/null +++ b/e2e/parity/trace-cli.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Transparently invoke the staged OpenShell CLI while retaining the exact +# stdout bytes produced by the conformance exec probe. + +set -uo pipefail + +: "${OPENSHELL_PARITY_REAL_CLI:?OPENSHELL_PARITY_REAL_CLI is required}" +: "${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE:?OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE is required}" + +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-cli.XXXXXX")" +cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +set +e +"${OPENSHELL_PARITY_REAL_CLI}" "$@" >"${tmpdir}/stdout" 2>"${tmpdir}/stderr" +status=$? +set -e + +cat "${tmpdir}/stdout" +cat "${tmpdir}/stderr" >&2 + +if [ "${1:-}" = sandbox ] && [ "${2:-}" = exec ]; then + install -m 0444 "${tmpdir}/stdout" "${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE}" +fi + +exit "${status}" diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py index 20d7be1e97..4ef8edece3 100644 --- a/e2e/parity/verify-results.py +++ b/e2e/parity/verify-results.py @@ -22,6 +22,7 @@ "conformance_sha256": "conformance", "supervisor_sha256": "supervisor", "supervisor_dockerfile_sha256": "supervisor.Dockerfile", + "cli_trace_wrapper_sha256": "cli-trace-wrapper", } ORACLE_MARKERS = ( "][smoke/status] completed", @@ -157,6 +158,11 @@ def verify_variant( set(podman_config) == {"socket_path"}, f"{config_path}: external gateway Podman table is not transport-only", ) + require( + isinstance(podman_config["socket_path"], str) + and Path(podman_config["socket_path"]).is_absolute(), + f"{config_path}: external driver socket path is not absolute", + ) else: required_runtime_fields = { "socket_path", @@ -308,6 +314,7 @@ def verify_variant( "supervisor_dockerfile_sha256_before_execution", "supervisor_dockerfile_sha256", ), + ("cli_trace_wrapper_sha256_before_execution", "cli_trace_wrapper_sha256"), ): require( launch.get(launch_field) == result.get(result_field), @@ -320,11 +327,11 @@ def verify_variant( f"{launch_path}: external driver pre-execution hash mismatch", ) gateway_port = launch.get("gateway_port") + callback_endpoint = f"https://host.containers.internal:{gateway_port}" require( isinstance(gateway_port, int) and 0 < gateway_port <= 65535 - and launch.get("external_driver_grpc_endpoint") - == f"https://host.containers.internal:{gateway_port}", + and launch.get("external_driver_grpc_endpoint") == callback_endpoint, f"{launch_path}: external driver callback endpoint is not isolated", ) require( @@ -335,6 +342,76 @@ def verify_variant( and launch.get("external_driver_app_armor") is False, f"{launch_path}: external driver effective configuration is tainted", ) + driver_environment = launch.get("external_driver_environment") + expected_environment_keys = { + "OPENSHELL_COMPUTE_DRIVER_SOCKET", + "OPENSHELL_PODMAN_SOCKET", + "OPENSHELL_SANDBOX_IMAGE", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + "OPENSHELL_GRPC_ENDPOINT", + "OPENSHELL_GATEWAY_PORT", + "OPENSHELL_NETWORK_NAME", + "OPENSHELL_STOP_TIMEOUT", + "OPENSHELL_SUPERVISOR_IMAGE", + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + "OPENSHELL_ENABLE_BIND_MOUNTS", + } + require( + isinstance(driver_environment, dict) + and set(driver_environment) == expected_environment_keys, + f"{launch_path}: external driver allowlisted environment is incomplete", + ) + require( + driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] + == podman_config["socket_path"], + f"{launch_path}: external driver socket differs from gateway TOML", + ) + podman_socket = driver_environment["OPENSHELL_PODMAN_SOCKET"] + require( + isinstance(podman_socket, str) + and Path(podman_socket).is_absolute() + and podman_socket != podman_config["socket_path"], + f"{launch_path}: external driver Podman socket is not isolated", + ) + require( + driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_runtime + and driver_environment["OPENSHELL_SANDBOX_IMAGE_PULL_POLICY"] + == expected_policy + and driver_environment["OPENSHELL_HEALTH_CHECK_INTERVAL_SECS"] == 10 + and driver_environment["OPENSHELL_GRPC_ENDPOINT"] == callback_endpoint + and driver_environment["OPENSHELL_GATEWAY_PORT"] == gateway_port + and isinstance(driver_environment["OPENSHELL_NETWORK_NAME"], str) + and driver_environment["OPENSHELL_NETWORK_NAME"] + and isinstance(driver_environment["OPENSHELL_STOP_TIMEOUT"], int) + and driver_environment["OPENSHELL_STOP_TIMEOUT"] >= 0 + and driver_environment["OPENSHELL_SUPERVISOR_IMAGE"] == runtime_image + and driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True, + f"{launch_path}: external driver allowlisted runtime inputs differ", + ) + tls_paths: set[str] = set() + for field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + tls_input = driver_environment[field] + require( + isinstance(tls_input, dict) + and set(tls_input) == {"path", "sha256"} + and isinstance(tls_input["path"], str) + and Path(tls_input["path"]).is_absolute() + and isinstance(tls_input["sha256"], str) + and SHA256_RE.fullmatch(tls_input["sha256"]) is not None, + f"{launch_path}: invalid external driver callback TLS input {field}", + ) + tls_paths.add(tls_input["path"]) + require( + len(tls_paths) == 3, + f"{launch_path}: external driver callback TLS paths are not distinct", + ) else: require( launch.get("external_driver_sha256_before_execution") == "", @@ -352,6 +429,31 @@ def verify_variant( '"passed": true' in raw_log, f"{log_path}: missing successful conformance result", ) + require( + re.search( + r"gateway preflight connected: .*authentication=authenticated(?:\n|$)", + raw_log, + ) + is not None, + f"{log_path}: authenticated gateway preflight is missing", + ) + run_ids = re.findall( + r"^CLI conformance run ID: ([a-z0-9]+)$", raw_log, re.MULTILINE + ) + require( + len(run_ids) == 1, + f"{log_path}: expected exactly one conformance run ID", + ) + exec_stdout_path = results_dir / f"{variant}.exec.stdout" + require( + exec_stdout_path.is_file(), + f"{exec_stdout_path}: retained exec stdout is missing", + ) + expected_exec_stdout = f"openshell-conformance-{run_ids[0]}\n".encode() + require( + exec_stdout_path.read_bytes() == expected_exec_stdout, + f"{exec_stdout_path}: callback exec stdout is not the exact marker", + ) launch_markers = ( runtime_image, image_id, @@ -371,6 +473,21 @@ def verify_variant( f"{log_path}: launch provenance is absent from raw output", ) + raw_evidence_hashes = { + result_path.name: sha256(result_path), + launch_path.name: sha256(launch_path), + log_path.name: sha256(log_path), + config_path.name: sha256(config_path), + exec_stdout_path.name: sha256(exec_stdout_path), + } + if external: + driver_log_path = results_dir / f"{variant}.driver.log" + require( + driver_log_path.is_file() and driver_log_path.stat().st_size > 0, + f"{driver_log_path}: retained external driver log is missing or empty", + ) + raw_evidence_hashes[driver_log_path.name] = sha256(driver_log_path) + return { "schema_version": schema_version, "source_sha": expected_sha, @@ -385,12 +502,7 @@ def verify_variant( }, "artifact_sha256": artifact_hashes, "launch_attestation": launch, - "raw_evidence_sha256": { - result_path.name: sha256(result_path), - launch_path.name: sha256(launch_path), - log_path.name: sha256(log_path), - config_path.name: sha256(config_path), - }, + "raw_evidence_sha256": raw_evidence_hashes, "artifacts_verified": True, "raw_output_verified": True, "success": True, @@ -450,6 +562,26 @@ def verify_topology( != candidate["artifact_sha256"]["external-driver"], "external-driver artifacts have identical content", ) + baseline_env = baseline_launch["external_driver_environment"] + candidate_env = candidate_launch["external_driver_environment"] + for field, label in ( + ("OPENSHELL_COMPUTE_DRIVER_SOCKET", "compute-driver UDS"), + ("OPENSHELL_PODMAN_SOCKET", "Podman API UDS"), + ("OPENSHELL_NETWORK_NAME", "Podman network"), + ): + require( + baseline_env[field] != candidate_env[field], + f"baseline and candidate reuse the same external {label}", + ) + for field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + require( + baseline_env[field]["path"] != candidate_env[field]["path"], + "baseline and candidate reuse the same external callback TLS path", + ) return { "baseline": baseline, @@ -495,6 +627,21 @@ def verify_four_run_provenance( tuples = {tuple(launch.get(field) for field in fields) for launch in variants} require(len(tuples) == 1, f"the four runs use different {label}") + for variant in ("baseline", "candidate"): + in_tree_artifacts = in_tree[variant]["artifact_sha256"] + external_artifacts = external_uds[variant]["artifact_sha256"] + for artifact in ( + "cli", + "conformance", + "supervisor", + "supervisor.Dockerfile", + "cli-trace-wrapper", + ): + require( + in_tree_artifacts[artifact] == external_artifacts[artifact], + f"{variant} {artifact} differs across topologies", + ) + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() @@ -530,6 +677,10 @@ def main() -> None: "baseline_commit": args.baseline_sha, "candidate_commit": args.candidate_sha, "lane": "local-linux-x86_64-rootless-podman-5.8.2", + "retained_evidence_bundles": { + "in_tree": args.in_tree.as_posix(), + "external_uds": args.external_uds.as_posix(), + }, "oracle": { "status": True, "create": True, @@ -553,6 +704,11 @@ def main() -> None: "verification": { "retained_artifact_hashes_recomputed": True, "raw_lifecycle_output_inspected": True, + "authenticated_preflight_verified": True, + "exact_callback_exec_stdout_verified": True, + "external_driver_allowlist_verified": True, + "external_driver_logs_retained": True, + "external_uds_isolation_verified": True, "digest_pinned_supervisor_runtime_verified": True, "same_immutable_sandbox_verified": True, "supervisor_dependency_provenance_matched": True, diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 4267fbc448..76fe699bc8 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -121,7 +121,8 @@ GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" DRIVER_BIN="" DRIVER_PID="" -DRIVER_LOG="${WORKDIR}/podman-driver.log" +DRIVER_LOG="${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE:-${WORKDIR}/podman-driver.log}" +mkdir -p "$(dirname "${DRIVER_LOG}")" DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" @@ -621,6 +622,12 @@ e2e_write_podman_gateway_config \ if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then cp "${GATEWAY_CONFIG}" "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE}" fi +EXTERNAL_DRIVER_CALLBACK_ENDPOINT="https://host.containers.internal:${HOST_PORT}" +EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS=10 +EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS=true +EXTERNAL_DRIVER_TLS_CA="${PKI_DIR}/ca.crt" +EXTERNAL_DRIVER_TLS_CERT="${PKI_DIR}/client/tls.crt" +EXTERNAL_DRIVER_TLS_KEY="${PKI_DIR}/client/tls.key" if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then driver_transport=in_tree external_driver_grpc_endpoint=null @@ -629,12 +636,34 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then external_driver_spiffe=false external_driver_proxy=false external_driver_app_armor=false + external_driver_environment=null if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then driver_transport=remote_uds - external_driver_grpc_endpoint="\"https://host.containers.internal:${HOST_PORT}\"" + external_driver_grpc_endpoint="\"${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}\"" external_driver_host_gateway_ip='"host-gateway"' + driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" + driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" + driver_tls_key_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_KEY}" | cut -d' ' -f1)" + external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + "${DRIVER_SOCKET}" \ + "${OPENSHELL_PODMAN_SOCKET:-}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ + "${EXTERNAL_DRIVER_PULL_POLICY}" \ + "${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ + "${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ + "${HOST_PORT}" \ + "${PODMAN_NETWORK_NAME}" \ + "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ + "${EXTERNAL_DRIVER_TLS_CA}" \ + "${driver_tls_ca_sha256}" \ + "${EXTERNAL_DRIVER_TLS_CERT}" \ + "${driver_tls_cert_sha256}" \ + "${EXTERNAL_DRIVER_TLS_KEY}" \ + "${driver_tls_key_sha256}" \ + "${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}")" fi - printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s}\n' \ + printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ "${CONFIG_SCHEMA_VERSION}" \ "${HOST_PORT}" \ "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ @@ -661,12 +690,14 @@ if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" \ "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256:-}" \ "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_TRACE_WRAPPER_SHA256:-}" \ "${external_driver_grpc_endpoint}" \ "${external_driver_host_gateway_ip}" \ "${external_driver_userns}" \ "${external_driver_spiffe}" \ "${external_driver_proxy}" \ "${external_driver_app_armor}" \ + "${external_driver_environment}" \ >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" fi @@ -678,16 +709,16 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ - OPENSHELL_HEALTH_CHECK_INTERVAL_SECS=10 \ - OPENSHELL_GRPC_ENDPOINT="https://host.containers.internal:${HOST_PORT}" \ + OPENSHELL_HEALTH_CHECK_INTERVAL_SECS="${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ + OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ - OPENSHELL_PODMAN_TLS_CA="${PKI_DIR}/ca.crt" \ - OPENSHELL_PODMAN_TLS_CERT="${PKI_DIR}/client/tls.crt" \ - OPENSHELL_PODMAN_TLS_KEY="${PKI_DIR}/client/tls.key" \ - OPENSHELL_ENABLE_BIND_MOUNTS=true \ + OPENSHELL_PODMAN_TLS_CA="${EXTERNAL_DRIVER_TLS_CA}" \ + OPENSHELL_PODMAN_TLS_CERT="${EXTERNAL_DRIVER_TLS_CERT}" \ + OPENSHELL_PODMAN_TLS_KEY="${EXTERNAL_DRIVER_TLS_KEY}" \ + OPENSHELL_ENABLE_BIND_MOUNTS="${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}" \ "${DRIVER_BIN}" >"${DRIVER_LOG}" 2>&1 & DRIVER_PID=$! e2e_wait_for_socket \ diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py index 48257b2f01..801703e041 100644 --- a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -52,6 +52,7 @@ def create_variant( "conformance": f"{variant}-conformance", "supervisor": f"{variant}-supervisor", "supervisor.Dockerfile": f"{variant}-dockerfile", + "cli-trace-wrapper": "shared-cli-trace-wrapper", "external-driver": f"{variant}-external-driver", "supervisor.packages.txt": "fixture-package-1.0-r0\n", } @@ -80,6 +81,9 @@ def create_variant( "supervisor_dockerfile_sha256": verifier.sha256( artifact_dir / "supervisor.Dockerfile" ), + "cli_trace_wrapper_sha256": verifier.sha256( + artifact_dir / "cli-trace-wrapper" + ), "external_driver_sha256": verifier.sha256(artifact_dir / "external-driver"), "success": True, }, @@ -132,22 +136,58 @@ def create_variant( "supervisor_dockerfile_sha256_before_execution": result[ "supervisor_dockerfile_sha256" ], + "cli_trace_wrapper_sha256_before_execution": result[ + "cli_trace_wrapper_sha256" + ], "external_driver_grpc_endpoint": "https://host.containers.internal:18181", "external_driver_host_gateway_ip": "host-gateway", "external_driver_userns": None, "external_driver_spiffe": False, "external_driver_proxy": False, "external_driver_app_armor": False, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": f"/tmp/{variant}.sock", + "OPENSHELL_PODMAN_SOCKET": f"/tmp/{variant}-podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "example.invalid/sandbox@" + IMAGE_DIGEST, + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": policy, + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:18181", + "OPENSHELL_GATEWAY_PORT": 18181, + "OPENSHELL_NETWORK_NAME": f"{variant}-network", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": RUNTIME_IMAGE, + "OPENSHELL_PODMAN_TLS_CA": { + "path": f"/tmp/{variant}-pki/ca.crt", + "sha256": "8" * 64, + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": f"/tmp/{variant}-pki/tls.crt", + "sha256": "9" * 64, + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": f"/tmp/{variant}-pki/tls.key", + "sha256": "a" * 64, + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": True, + }, }, ) lifecycle = "\n".join( f"[run fixture{marker} in 1ms: exit 0" for marker in verifier.ORACLE_MARKERS ) (results_dir / f"{variant}.log").write_text( + f"CLI conformance run ID: fixture\n" + f"gateway preflight connected: gateway=fixture, authentication=authenticated\n" f"{lifecycle}\n{RUNTIME_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', encoding="utf-8", ) + (results_dir / f"{variant}.exec.stdout").write_bytes( + b"openshell-conformance-fixture\n" + ) + (results_dir / f"{variant}.driver.log").write_text( + "external driver fixture started\n", encoding="utf-8" + ) def create_external_bundle(verifier: ModuleType, results_dir: Path) -> None: @@ -218,6 +258,7 @@ def test_verifier_rejects_different_sandbox_artifacts(tmp_path: Path) -> None: "sandbox_client_image_alias_id": other_id, } ) + launch["external_driver_environment"]["OPENSHELL_SANDBOX_IMAGE"] = other_runtime write_json(launch_path, launch) with (tmp_path / "candidate.log").open("a", encoding="utf-8") as log: log.write(f"{other_id} {other_digest} {other_runtime}\n") @@ -247,6 +288,87 @@ def test_verifier_rejects_different_supervisor_packages(tmp_path: Path) -> None: ) +def test_verifier_rejects_unattested_external_driver_input(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + del launch["external_driver_environment"]["OPENSHELL_STOP_TIMEOUT"] + write_json(launch_path, launch) + + with pytest.raises(ValueError, match="allowlisted environment is incomplete"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_binds_gateway_and_driver_uds(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["external_driver_environment"]["OPENSHELL_COMPUTE_DRIVER_SOCKET"] = ( + "/tmp/different.sock" + ) + write_json(launch_path, launch) + + with pytest.raises(ValueError, match="driver socket differs from gateway TOML"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_reused_external_uds(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["external_driver_environment"]["OPENSHELL_COMPUTE_DRIVER_SOCKET"] = ( + "/tmp/baseline.sock" + ) + write_json(launch_path, launch) + config_path = tmp_path / "candidate.gateway.toml" + config_path.write_text( + config_path.read_text(encoding="utf-8").replace( + "/tmp/candidate.sock", "/tmp/baseline.sock" + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="reuse the same external compute-driver UDS"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_requires_exact_exec_stdout(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + (tmp_path / "candidate.exec.stdout").write_text("wrong marker\n", encoding="utf-8") + + with pytest.raises(ValueError, match="stdout is not the exact marker"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_requires_authenticated_preflight(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + log_path = tmp_path / "candidate.log" + log_path.write_text( + log_path.read_text(encoding="utf-8").replace( + "authentication=authenticated", "authentication=unauthenticated" + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="authenticated gateway preflight is missing"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + def test_verifier_rejects_cross_topology_sandbox_drift() -> None: verifier = load_verifier() launch = { @@ -261,13 +383,32 @@ def test_verifier_rejects_cross_topology_sandbox_drift() -> None: "supervisor_base_runtime_image": BASE_RUNTIME_IMAGE, "supervisor_package_manifest_sha256": "7" * 64, } + shared_artifacts = { + "cli": "b" * 64, + "conformance": "c" * 64, + "supervisor": "d" * 64, + "supervisor.Dockerfile": "e" * 64, + "cli-trace-wrapper": "f" * 64, + } in_tree = { - "baseline": {"launch_attestation": dict(launch)}, - "candidate": {"launch_attestation": dict(launch)}, + "baseline": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + "candidate": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, } external = { - "baseline": {"launch_attestation": dict(launch)}, - "candidate": {"launch_attestation": dict(launch)}, + "baseline": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + "candidate": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, } external["candidate"]["launch_attestation"]["sandbox_image_id"] = "8" * 64 From d904600909e29e9131c88b8f7f6f1a40bcde34bc Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 09:20:28 -0400 Subject: [PATCH 35/42] test(e2e): record compute boundary parity Signed-off-by: Jesse Jaggars --- ...schema-v2-compute-boundary-comparison.json | 376 ++++++++++++++++++ .../gateway/schema-v2-live-results.toml | 19 + .../gateway_schema_v2_live_results_test.py | 154 +++++++ 3 files changed, 549 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-compute-boundary-comparison.json diff --git a/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json b/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json new file mode 100644 index 0000000000..783bb00932 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json @@ -0,0 +1,376 @@ +{ + "manifest_version": 2, + "baseline_commit": "74960ebfaeec4673885089ed995fad902459749f", + "candidate_commit": "4a39da510e4d278a24dd60291149519c9a570b46", + "lane": "local-linux-x86_64-rootless-podman-5.8.2", + "retained_evidence_bundles": { + "in_tree": "target/parity/step10-intree-4a39da51", + "external_uds": "target/parity/step10-external-4a39da51" + }, + "oracle": { + "status": true, + "create": true, + "ready": true, + "list_visible": true, + "callback_exec_exact_marker": true, + "delete": true, + "list_empty": true + }, + "in_tree": { + "baseline": { + "schema_version": 1, + "source_sha": "74960ebfaeec4673885089ed995fad902459749f", + "gateway_profile": "in-tree", + "gateway_cargo_features": "default", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "not_applicable", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "264cf3d809bd1d54633bce9251ec1a5540b567b8150640091df85bdfbe61797a", + "cli": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "supervisor": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 1, + "gateway_port": 32893, + "external_compute_driver": false, + "compute_driver_transport": "in_tree", + "external_driver_pull_policy": "missing", + "supervisor_image": "localhost/openshell/supervisor:parity-baseline-74960ebfaeec", + "supervisor_image_id": "b5bcaae227a1d6ea90fd146fc2904d4166725c8eb1b9590bdfd85b14c7b9caef", + "supervisor_image_digest": "sha256:0f11b6ca65ff33c99d03d6ebd8af239ac9edc148f849201a692a06d722853030", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:0f11b6ca65ff33c99d03d6ebd8af239ac9edc148f849201a692a06d722853030", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "264cf3d809bd1d54633bce9251ec1a5540b567b8150640091df85bdfbe61797a", + "cli_sha256_before_execution": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance_sha256_before_execution": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "external_driver_sha256_before_execution": "", + "supervisor_sha256_before_execution": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": null, + "external_driver_host_gateway_ip": null, + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": null + }, + "raw_evidence_sha256": { + "baseline.json": "d7e7f18607d1eab7fab65b1643febe46499aeaaa00c100b66f92b11101d798a1", + "baseline.launch.json": "a56fe7df2c454137cebcea17bc022adcb9b00f3a2e7403611f61fede62a9b329", + "baseline.log": "e44a39cf19296908e99c58b661bd1c162f0aedd2fafb9c70a1a875d91e712cbe", + "baseline.gateway.toml": "9d11c7c8455308c21354f2a41b546068f06e4605f9127e0156467c321bac550c", + "baseline.exec.stdout": "f3a45a1114b92419a9c4c91364d584de4161ec171a1a9a73dae5b4daca89f8a9" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "candidate": { + "schema_version": 2, + "source_sha": "4a39da510e4d278a24dd60291149519c9a570b46", + "gateway_profile": "in-tree", + "gateway_cargo_features": "default", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "not_applicable", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "2bd42b145f0438f48a579b010537f501e036035b0e22a4ff70e37db733a6e6ff", + "cli": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "supervisor": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 2, + "gateway_port": 55987, + "external_compute_driver": false, + "compute_driver_transport": "in_tree", + "external_driver_pull_policy": "if_not_present", + "supervisor_image": "localhost/openshell/supervisor:parity-candidate-4a39da510e4d", + "supervisor_image_id": "489ba15df40c47957b400e54633c250b4322dc37f2798a59717b57c75421330a", + "supervisor_image_digest": "sha256:6de7479f138e88da131741e64c170aa75ae14b4e7b9b49de2d11f51af7d1b6d7", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:6de7479f138e88da131741e64c170aa75ae14b4e7b9b49de2d11f51af7d1b6d7", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "2bd42b145f0438f48a579b010537f501e036035b0e22a4ff70e37db733a6e6ff", + "cli_sha256_before_execution": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance_sha256_before_execution": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "external_driver_sha256_before_execution": "", + "supervisor_sha256_before_execution": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": null, + "external_driver_host_gateway_ip": null, + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": null + }, + "raw_evidence_sha256": { + "candidate.json": "ed7d6334cccb40488291e35e9b7ad4fb45dd47b55b7e27925384d22907bedefd", + "candidate.launch.json": "384ee89066973cacdb43160a39542096bc3d02da051ac0417fefc0be41c2f619", + "candidate.log": "719aaedc019492ccdc2f1e003af6b12af5bf6d2507fc2573135fca529a9f398f", + "candidate.gateway.toml": "504bbea27b56cc630884ba3eb17eb518f7c55cb7886b10ff353cc777e3d08362", + "candidate.exec.stdout": "d903d8a1f84caca8daa2a5a047c3fe90e2501690fba7c0b62e14d3892b5aadc3" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "comparison_sha256": "e97312770ed8cbba6325b901885d083c7f82ef979b5b89e4e0ed2bd32478d9fb", + "classification": "pass", + "parity": true, + "accepted": true + }, + "external_uds": { + "baseline": { + "schema_version": 1, + "source_sha": "74960ebfaeec4673885089ed995fad902459749f", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "built_by_harness", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "5cc2f700d93dde5dd09060d9c9190ae44a99440b1399530a2b23941ec4e88f85", + "cli": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "supervisor": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external-driver": "d976be0580ab5a2e006ca9e1bc1556b84fa810481f6e1c2132793f8e38c7194c", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 1, + "gateway_port": 50871, + "external_compute_driver": true, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": "missing", + "supervisor_image": "localhost/openshell/supervisor:parity-baseline-74960ebfaeec", + "supervisor_image_id": "b2e8f8dae82296a54e7500beb0b2b55d1f8d4ac4afe6ed8de5fba4b3b65ba748", + "supervisor_image_digest": "sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "5cc2f700d93dde5dd09060d9c9190ae44a99440b1399530a2b23941ec4e88f85", + "cli_sha256_before_execution": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance_sha256_before_execution": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "external_driver_sha256_before_execution": "d976be0580ab5a2e006ca9e1bc1556b84fa810481f6e1c2132793f8e38c7194c", + "supervisor_sha256_before_execution": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": "https://host.containers.internal:50871", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": "/tmp/openshell-e2e-podman.H8oMaO/compute-driver.sock", + "OPENSHELL_PODMAN_SOCKET": "/tmp/openshell-e2e-podman.H8oMaO/podman/podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": "missing", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:50871", + "OPENSHELL_GATEWAY_PORT": 50871, + "OPENSHELL_NETWORK_NAME": "e2e-podman-659291-50871", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": "localhost/openshell/supervisor@sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "OPENSHELL_PODMAN_TLS_CA": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/ca.crt", + "sha256": "1a77771ebd83be7ed988263ea4a8e42d6b2af94c46861693911c4d5bff6a3fdd" + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/client/tls.crt", + "sha256": "0e9ad610227c223ebca348599a113e07f46ab0c6ff6f4ea5598e4c15673e25fd" + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/client/tls.key", + "sha256": "d5bbf26f73a8fdfe60e16783bb99275bf35c617b1f229b7c48afb0a3edbacbc0" + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": true + } + }, + "raw_evidence_sha256": { + "baseline.json": "2b48dd376fd02d46c93d6ae0831d3a81d7337893794ca97c6559676e1b8302de", + "baseline.launch.json": "babe872dd643f024da5e4d0697a3a946ba21432d16e7b7abb2e343e82459ec79", + "baseline.log": "1718593d49cd26f76b7ce1d3be5cfca7c70fae7c44976aa1485796b52048ab41", + "baseline.gateway.toml": "8d4f2579c58ba140e95dc1151216cf9962ac211272ce26d2f811b6f3a1a659d0", + "baseline.exec.stdout": "f740ac1e211850739407334863f73311e7ff508bb4e507b7d7fcaf8d8dc2de32", + "baseline.driver.log": "877ab38e85532d3b772ec080adc727ef517203b1b553e75c9b6f29ad06e3374a" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "candidate": { + "schema_version": 2, + "source_sha": "4a39da510e4d278a24dd60291149519c9a570b46", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "built_by_harness", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "fdefc047c1b675c311b1796db9f1b1a944456fc34b76547efc0352c6a75a7707", + "cli": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "supervisor": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external-driver": "9accb17523a33e8fc4df93b3b3dec619f1f5ad6f5ff3f51abb2a8a5718278f8c", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 2, + "gateway_port": 32901, + "external_compute_driver": true, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": "if_not_present", + "supervisor_image": "localhost/openshell/supervisor:parity-candidate-4a39da510e4d", + "supervisor_image_id": "9c7f51faec648947fe1515ec4bcbab52234e4e143a53bd1bf0a358eadd11440e", + "supervisor_image_digest": "sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "fdefc047c1b675c311b1796db9f1b1a944456fc34b76547efc0352c6a75a7707", + "cli_sha256_before_execution": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance_sha256_before_execution": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "external_driver_sha256_before_execution": "9accb17523a33e8fc4df93b3b3dec619f1f5ad6f5ff3f51abb2a8a5718278f8c", + "supervisor_sha256_before_execution": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": "https://host.containers.internal:32901", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": "/tmp/openshell-e2e-podman.M8xsgQ/compute-driver.sock", + "OPENSHELL_PODMAN_SOCKET": "/tmp/openshell-e2e-podman.M8xsgQ/podman/podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": "if_not_present", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:32901", + "OPENSHELL_GATEWAY_PORT": 32901, + "OPENSHELL_NETWORK_NAME": "e2e-podman-660516-32901", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": "localhost/openshell/supervisor@sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "OPENSHELL_PODMAN_TLS_CA": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/ca.crt", + "sha256": "90b268e741240a39838b7074b66ffa30576c053f5c8f5659e1e2e3c0e22809df" + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/client/tls.crt", + "sha256": "a89f555a5e53cf155f122bd2c8b58e0856019468cdf1cf80cafb1c832979f40f" + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/client/tls.key", + "sha256": "a53d8aaea929dec1b355a5e6ae45eb5001f9c3845d3d2cbbff9ca5a7e468be06" + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": true + } + }, + "raw_evidence_sha256": { + "candidate.json": "a68a9a3f78a8ba0fbf33c81e9960e705e554e82077fd4e0e8565a3bc8598790b", + "candidate.launch.json": "5be63290c313302ec2fa35cdd55ec641742d2d7009f7c2746842c42072425f1a", + "candidate.log": "d19a44064c8d90b078e4d34d711c599ae424834de433b355805105ae3f272cf2", + "candidate.gateway.toml": "862322125b3bc9438b87b35448e107d87850f4f3cd61c9c36f045560c963385e", + "candidate.exec.stdout": "fc1085830f6824809d3fdec6ecf518d54a7dde92ec4441e134f13abd5a6a6522", + "candidate.driver.log": "0b977d9c2b69a3e9665d457dd88fe231b3da7a55d74d4ace4c1a467e7cf716d1" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "comparison_sha256": "6b30e24a1cf0f9b7d8b5a5de206cfbbfb01c19a7921d6a449900af8448abec4c", + "classification": "pass", + "parity": true, + "accepted": true + }, + "callback_listener": { + "in_tree_baseline_exec": true, + "in_tree_candidate_exec": true, + "external_baseline_exec": true, + "external_candidate_exec": true, + "classification": "pass" + }, + "classification": "pass", + "accepted": true, + "verification": { + "retained_artifact_hashes_recomputed": true, + "raw_lifecycle_output_inspected": true, + "authenticated_preflight_verified": true, + "exact_callback_exec_stdout_verified": true, + "external_driver_allowlist_verified": true, + "external_driver_logs_retained": true, + "external_uds_isolation_verified": true, + "digest_pinned_supervisor_runtime_verified": true, + "same_immutable_sandbox_verified": true, + "supervisor_dependency_provenance_matched": true + } +} diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index 5af2a2f2ff..51e5a9c7f6 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -177,3 +177,22 @@ status = "platform_blocked" owner = "OpenShell Linux VM security CI lane" lane = "linux-x86_64-kvm-libkrun-proxy-spiffe" blocker = "The guest JWT mode and visibility, private owner-marker and sandbox-state persistence, callback recovery, TLS key permissions, proxy credential containment, and guest-reachable SPIFFE TCP behavior require a booted VM. The assigned lane must add authenticated-proxy and Workload API TCP fixtures to the paired libkrun/KVM run." + +[[result]] +id = "compute-driver-boundary-parity" +step = 10 +capability = "In-tree and external-UDS compute-driver execution with sandbox callback connectivity" +driver = "podman" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "4a39da510e4d278a24dd60291149519c9a570b46" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh exact-source gateway, CLI, conformance, Podman driver, and supervisor artifacts were staged before execution, hashed, executed from retained staging directories, and verified unchanged after execution.", + "Driver-free baseline and candidate gateways completed the same authenticated status, create, Ready, list, callback-exec, delete, and list-empty oracle through distinct-content external-driver executables over separate UDS endpoints; in-tree gateways completed the same oracle.", + "External drivers ran under env -i with a complete allowlisted environment attestation covering compute and Podman sockets, callback endpoint and TLS-file hashes, pull policy, images, network, timeouts, and bind mounts; successful driver logs were retained and hashed.", + "The verifier bound each transport-only gateway socket_path to the driver process input and required distinct baseline and candidate compute sockets, Podman sockets, networks, and callback TLS paths.", + "All four runs executed one repository-digest-pinned sandbox image and matched its image ID and manifest digest. They also matched the supervisor base-image identity, digest-pinned base reference, source Dockerfile, installed-package manifest, and same-source runtime artifacts across topologies.", + "Each variant built its supervisor image from its own staged supervisor binary in a forced temporary Podman service and isolated store; all supervisor runtime image references were digest-pinned.", + "A staged and hashed transparent CLI wrapper retained exact exec stdout bytes; the verifier tied them to the unique conformance run ID, required authenticated preflight and every lifecycle exit 0, and hashed result, launch, driver, raw-log, exec-stdout, gateway-configuration, package, and staged-artifact evidence.", +] diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index 68fe89fa51..591257260b 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -5,6 +5,9 @@ from __future__ import annotations +import hashlib +import json +import re import subprocess import tomllib from pathlib import Path @@ -13,6 +16,9 @@ REPO_ROOT = Path(__file__).resolve().parents[2] RESULTS_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-live-results.toml" CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" +COMPUTE_BOUNDARY_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-compute-boundary-comparison.json" +) REQUIRED_HEADER_FIELDS = { "manifest_version", @@ -44,6 +50,15 @@ "vm-guest-security-and-spiffe", "vm-launch-and-resource-configuration", } +REQUIRED_STEP_10_IDS = {"compute-driver-boundary-parity"} +STEP_10_CANDIDATE_COMMIT = "4a39da510e4d278a24dd60291149519c9a570b46" +STEP_10_REPORT_SHA256 = ( + "65541eec5f642461a88b04b5459474fd7a475adeb7071a65a53fe183caad6a01" +) +STEP_10_EVIDENCE_BUNDLES = { + "in_tree": "target/parity/step10-intree-4a39da51", + "external_uds": "target/parity/step10-external-4a39da51", +} ALLOWED_STATUSES = { "pass", "intentional_change", @@ -181,6 +196,145 @@ def test_step_9_records_vm_runtime_dispositions() -> None: assert all(result["driver"] == "vm" for result in results) +def test_step_10_records_verified_compute_boundary_parity(tmp_path: Path) -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 10 + ] + assert {result["id"] for result in results} == REQUIRED_STEP_10_IDS + assert results[0]["status"] == "pass" + + with COMPUTE_BOUNDARY_PATH.open(encoding="utf-8") as report_file: + report = json.load(report_file) + assert report["manifest_version"] == 2 + assert report["baseline_commit"] == load_toml(RESULTS_PATH)["baseline_commit"] + assert report["candidate_commit"] == results[0]["validated_candidate_commit"] + assert report["candidate_commit"] == STEP_10_CANDIDATE_COMMIT + assert report["retained_evidence_bundles"] == STEP_10_EVIDENCE_BUNDLES + assert ( + hashlib.sha256(COMPUTE_BOUNDARY_PATH.read_bytes()).hexdigest() + == STEP_10_REPORT_SHA256 + ) + assert report["classification"] == "pass" + assert report["accepted"] is True + assert all(report["oracle"].values()) + assert all(report["verification"].values()) + + launch_attestations = [] + for topology_name in ("in_tree", "external_uds"): + topology = report[topology_name] + assert topology["classification"] == "pass" + assert topology["parity"] is True + assert topology["accepted"] is True + assert re.fullmatch(r"[0-9a-f]{64}", topology["comparison_sha256"]) + baseline_launch = topology["baseline"]["launch_attestation"] + candidate_launch = topology["candidate"]["launch_attestation"] + for field in ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_package_manifest_sha256", + ): + assert baseline_launch[field] == candidate_launch[field] + launch_attestations.extend((baseline_launch, candidate_launch)) + + for variant_name, schema_version in (("baseline", 1), ("candidate", 2)): + variant = topology[variant_name] + assert variant["schema_version"] == schema_version + assert variant["success"] is True + assert variant["artifacts_verified"] is True + assert variant["raw_output_verified"] is True + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) + for digest in variant["artifact_sha256"].values() + ) + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) + for digest in variant["raw_evidence_sha256"].values() + ) + launch = variant["launch_attestation"] + supervisor_digest = launch["supervisor_image_digest"] + assert re.fullmatch(r"sha256:[0-9a-f]{64}", supervisor_digest) + assert launch["supervisor_runtime_image"].endswith(f"@{supervisor_digest}") + sandbox_digest = launch["sandbox_image_digest"] + assert re.fullmatch(r"sha256:[0-9a-f]{64}", sandbox_digest) + assert launch["sandbox_image_request"] == launch["sandbox_runtime_image"] + assert launch["sandbox_runtime_image"].endswith(f"@{sandbox_digest}") + + for field in ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_base_runtime_image", + "supervisor_package_manifest_sha256", + "supervisor_dockerfile_sha256_before_execution", + ): + assert len({launch[field] for launch in launch_attestations}) == 1, field + + external_sockets = [] + for variant_name in ("baseline", "candidate"): + external_variant = report["external_uds"][variant_name] + external_launch = external_variant["launch_attestation"] + assert external_launch["compute_driver_transport"] == "remote_uds" + assert external_launch["external_compute_driver"] is True + assert external_launch["external_driver_grpc_endpoint"].startswith("https://") + assert external_launch["external_driver_host_gateway_ip"] == "host-gateway" + assert external_launch["external_driver_userns"] is None + assert external_launch["external_driver_spiffe"] is False + assert external_launch["external_driver_proxy"] is False + assert external_launch["external_driver_app_armor"] is False + driver_environment = external_launch["external_driver_environment"] + assert driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] + assert driver_environment["OPENSHELL_PODMAN_SOCKET"] + assert driver_environment["OPENSHELL_GRPC_ENDPOINT"].startswith("https://") + assert driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True + for tls_field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + assert re.fullmatch( + r"[0-9a-f]{64}", driver_environment[tls_field]["sha256"] + ) + external_sockets.append(driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"]) + assert f"{variant_name}.driver.log" in external_variant["raw_evidence_sha256"] + assert f"{variant_name}.exec.stdout" in external_variant["raw_evidence_sha256"] + assert len(set(external_sockets)) == 2 + + evidence_paths = { + name: REPO_ROOT / relative_path + for name, relative_path in STEP_10_EVIDENCE_BUNDLES.items() + } + present = {name: path.is_dir() for name, path in evidence_paths.items()} + assert len(set(present.values())) == 1, "Step 10 retained evidence is incomplete" + if all(present.values()): + reproduced = tmp_path / "schema-v2-compute-boundary-comparison.json" + subprocess.run( + [ + "python3", + str(REPO_ROOT / "e2e/parity/verify-results.py"), + "--baseline-sha", + report["baseline_commit"], + "--candidate-sha", + STEP_10_CANDIDATE_COMMIT, + "--in-tree", + STEP_10_EVIDENCE_BUNDLES["in_tree"], + "--external-uds", + STEP_10_EVIDENCE_BUNDLES["external_uds"], + "--output", + str(reproduced), + ], + cwd=REPO_ROOT, + check=True, + ) + assert reproduced.read_bytes() == COMPUTE_BOUNDARY_PATH.read_bytes() + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": From c44ec6a44e3b0869b95a77037dba02d23d7961a5 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 10:12:56 -0400 Subject: [PATCH 36/42] test(e2e): disposition cross-cutting parity lanes Signed-off-by: Jesse Jaggars --- .../schema-v2-cross-cutting-dispositions.toml | 109 +++++++++++ .../gateway/schema-v2-live-results.toml | 114 ++++++++++++ .../schema-v2-step11-baseline-attestation.txt | 101 ++++++++++ ...schema-v2-step11-candidate-attestation.txt | 102 +++++++++++ ...hema_v2_cross_cutting_dispositions_test.py | 173 ++++++++++++++++++ .../gateway_schema_v2_live_results_test.py | 36 ++++ 6 files changed, 635 insertions(+) create mode 100644 e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml create mode 100644 e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt create mode 100644 e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt create mode 100644 python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py diff --git a/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml b/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml new file mode 100644 index 0000000000..813da3bc61 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Step 11 dispositions for the cross-cutting gateway capabilities that are not +# exercised by the compute-driver waves. The paired deterministic preflight is +# regression evidence only: it does not replace a candidate-owned oracle that +# drives both gateway processes and observes the same live behavior. +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_commit = "363d8540830b2ea294d43198daa2b7a283a2face" +overall_status = "platform_blocked" + +[deterministic_preflight] +status = "pass" +baseline_attestation = "e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt" +baseline_attestation_sha256 = "df3c5e1dcb073eb8db0cb28377516523b331d44fc6f5a5fefab2e92273efaf51" +candidate_attestation = "e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt" +candidate_attestation_sha256 = "2562bff73eba8042199597e814b618b12b93994d15e4ae36e902347347d95fed" +commands = [ + "cargo test -q -p openshell-server --lib", + "cargo test -q -p openshell-gateway-interceptors", + "cargo test -q -p openshell-supervisor-middleware", + "cargo test -q -p openshell-otel-test-support", + "cargo test -q -p openshell-server --test multiplex_tls_integration", + "cargo test -q -p openshell-server --test edge_tunnel_auth", +] +evidence = [ + "The frozen baseline passed 1,455 openshell-server library tests with 8 ignored; the candidate passed 1,481 with 8 ignored.", + "Both revisions passed 49 gateway-interceptor tests, 85 supervisor-middleware tests, 5 multiplex TLS integration tests, and 5 edge-tunnel authentication tests.", + "The openshell-otel-test-support crate compiled successfully on both revisions but currently defines no tests; OTLP behavior is exercised by openshell-server unit tests and still requires a collector-backed live lane.", + "Both runs used checkout-local target directories with sccache disabled after a prior ENOSPC event corrupted the shared cache.", +] + +[[capabilities]] +id = "oidc-bearer-authentication" +status = "platform_blocked" +owner = "OpenShell auth and OIDC CI lane" +lane = "linux-oidc-paired-keycloak-jwks" +blocker = "The host has no isolated paired issuer fixture that presents equivalent signed tokens to both exact-source gateways. Existing Keycloak and OIDC suites are candidate-oriented and cannot establish schema-v1/schema-v2 audience, role, scope, expiry, signature, and key-rotation parity without a shared candidate-owned oracle." + +[[capabilities]] +id = "mtls-user-authentication" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-mtls-user-principal-paired" +blocker = "The paired TLS integration tests validate handshake policy but do not drive an authenticated user RPC through both gateways or prove certificate-to-principal mapping. A live lane must use isolated client PKI, equivalent schema files, and the same authorization oracle for enabled and disabled mTLS user authentication." + +[[capabilities]] +id = "unsafe-unauthenticated-user-mode" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-auth-chain-paired-isolated" +blocker = "No paired live fixture currently proves that the trusted-development switch admits user requests while sandbox callbacks continue to require sandbox credentials. Candidate-only extension examples use this switch for setup, but that is not an authentication-boundary parity test." + +[[capabilities]] +id = "gateway-minted-sandbox-jwt" +status = "platform_blocked" +owner = "OpenShell gateway JWT CI lane" +lane = "linux-gateway-jwt-claims-paired" +blocker = "Step 10 proved authenticated callback connectivity but did not capture and compare token claims. A controlled callback and extension fixture must verify issuer, gateway identity, key ID, audience, subject, token type, and expiry semantics for both gateways; explicit zero versus omitted TTL remains the ledgered gateway-jwt-zero-sentinel-removed intentional change." + +[[capabilities]] +id = "otlp-observability" +status = "platform_blocked" +owner = "OpenShell observability CI lane" +lane = "linux-otlp-grpc-collector-paired" +blocker = "The deterministic exporter tests passed, but no retained paired collector capture proves emitted gateway and in-tree-driver spans, gateway name, service name, selected-driver resource attributes, or continued serving after collector failure. A loopback OTLP/gRPC collector lane must observe those outcomes from both exact-source processes." + +[[capabilities]] +id = "gateway-interceptor-registration" +status = "platform_blocked" +owner = "OpenShell gateway interceptor CI lane" +lane = "linux-gateway-interceptor-paired" +blocker = "The interceptor library suite passed on both revisions, while the governance-interceptor smoke test remains candidate-only. A paired service must compare Describe validation, binding selection, unary request and response mutation, failure policy, timeout and size limits, and secret exclusion against both gateway schema variants." + +[[capabilities]] +id = "supervisor-middleware-registration" +status = "platform_blocked" +owner = "OpenShell supervisor middleware CI lane" +lane = "linux-supervisor-middleware-paired" +blocker = "The middleware library suite passed on both revisions, while the content-guard smoke test remains candidate-only. A paired gateway, sandbox, and middleware fixture must compare Describe negotiation, policy distribution, guarded and unguarded traffic, failure policy, timeout, and payload limits; the field-name normalization remains the ledgered middleware-payload-name-normalized intentional change." + +[[capabilities]] +id = "provider-profile-sources" +status = "platform_blocked" +owner = "OpenShell provider profile CI lane" +lane = "linux-provider-profiles-interceptor-paired" +blocker = "Server tests cover source construction and duplicate rejection, but no shared live interceptor catalog has been queried through both gateways. The assigned lane must compare builtin, user, and interceptor source ordering, normalized profile identities, duplicate rejection, discovery output, and failure handling with isolated persistent state." + +[[capabilities]] +id = "inference-control-plane-configuration" +status = "platform_blocked" +owner = "OpenShell inference E2E lane" +lane = "linux-inference-provider-routing-paired" +blocker = "Existing inference routing tests are candidate-oriented and this host has no assigned paired provider and model-service fixture. The lane must configure providers and routes through each gateway control plane, observe the effective supervisor bundle, and compare model discovery and an inference request without reusing database state." + +[[capabilities]] +id = "credential-driver-selection-and-kek" +status = "platform_blocked" +owner = "OpenShell credential security CI lane" +lane = "linux-credential-kek-paired" +blocker = "Both server suites passed deterministic selection, KEK, and credential tests, but they are revision-local tests rather than one candidate-owned live oracle. A paired lane must start isolated gateways with independent databases and KEKs, compare default encrypted storage and invalid selections, store and resolve opaque credentials, and verify that logs and configuration do not disclose secrets." + +[[capabilities]] +id = "credential-driver-backend-tables" +status = "platform_blocked" +owner = "OpenShell credential driver E2E lane" +lane = "kubernetes-vault-uds-credential-drivers-paired" +blocker = "The host has no assigned disposable Kubernetes Secrets or Vault backend and no retained paired UDS credential-driver execution. Existing backend E2E coverage is candidate-oriented. The assigned lane must compare in-tree and remote transport validation plus opaque store and retrieve behavior using isolated namespaces, Vault state, sockets, databases, and credentials." diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index 51e5a9c7f6..a1e37c7bdf 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -196,3 +196,117 @@ evidence = [ "Each variant built its supervisor image from its own staged supervisor binary in a forced temporary Podman service and isolated store; all supervisor runtime image references were digest-pinned.", "A staged and hashed transparent CLI wrapper retained exact exec stdout bytes; the verifier tied them to the unique conformance run ID, required authenticated preflight and every lifecycle exit 0, and hashed result, launch, driver, raw-log, exec-stdout, gateway-configuration, package, and staged-artifact evidence.", ] + +# Step 11 inventories the remaining cross-cutting capabilities. Paired +# deterministic suites passed, but they are regression preflight rather than +# live parity evidence. Every row therefore remains assigned to a live lane. + +[[result]] +id = "oidc-bearer-authentication" +step = 11 +capability = "OIDC bearer token validation, role and scope authorization, and JWKS refresh" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell auth and OIDC CI lane" +lane = "linux-oidc-paired-keycloak-jwks" +blocker = "The host has no isolated paired issuer fixture that presents equivalent signed tokens to both exact-source gateways. Existing Keycloak and OIDC suites are candidate-oriented and cannot establish schema-v1/schema-v2 audience, role, scope, expiry, signature, and key-rotation parity without a shared candidate-owned oracle." + +[[result]] +id = "mtls-user-authentication" +step = 11 +capability = "mTLS client-certificate user identity and authorization" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-mtls-user-principal-paired" +blocker = "The paired TLS integration tests validate handshake policy but do not drive an authenticated user RPC through both gateways or prove certificate-to-principal mapping. A live lane must use isolated client PKI, equivalent schema files, and the same authorization oracle for enabled and disabled mTLS user authentication." + +[[result]] +id = "unsafe-unauthenticated-user-mode" +step = 11 +capability = "Explicit trusted-development unauthenticated user mode with authenticated sandbox callbacks" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-auth-chain-paired-isolated" +blocker = "No paired live fixture currently proves that the trusted-development switch admits user requests while sandbox callbacks continue to require sandbox credentials. Candidate-only extension examples use this switch for setup, but that is not an authentication-boundary parity test." + +[[result]] +id = "gateway-minted-sandbox-jwt" +step = 11 +capability = "Gateway-minted sandbox and extension JWT claims and lifetime semantics" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway JWT CI lane" +lane = "linux-gateway-jwt-claims-paired" +blocker = "Step 10 proved authenticated callback connectivity but did not capture and compare token claims. A controlled callback and extension fixture must verify issuer, gateway identity, key ID, audience, subject, token type, and expiry semantics for both gateways; explicit zero versus omitted TTL remains the ledgered gateway-jwt-zero-sentinel-removed intentional change." + +[[result]] +id = "otlp-observability" +step = 11 +capability = "Gateway and in-tree driver OTLP traces and failure isolation" +driver = "observability" +status = "platform_blocked" +owner = "OpenShell observability CI lane" +lane = "linux-otlp-grpc-collector-paired" +blocker = "The deterministic exporter tests passed, but no retained paired collector capture proves emitted gateway and in-tree-driver spans, gateway name, service name, selected-driver resource attributes, or continued serving after collector failure. A loopback OTLP/gRPC collector lane must observe those outcomes from both exact-source processes." + +[[result]] +id = "gateway-interceptor-registration" +step = 11 +capability = "Gateway interceptor registration, binding, mutation, and failure behavior" +driver = "gateway-interceptor" +status = "platform_blocked" +owner = "OpenShell gateway interceptor CI lane" +lane = "linux-gateway-interceptor-paired" +blocker = "The interceptor library suite passed on both revisions, while the governance-interceptor smoke test remains candidate-only. A paired service must compare Describe validation, binding selection, unary request and response mutation, failure policy, timeout and size limits, and secret exclusion against both gateway schema variants." + +[[result]] +id = "supervisor-middleware-registration" +step = 11 +capability = "Supervisor middleware registration, policy distribution, and enforcement" +driver = "supervisor-middleware" +status = "platform_blocked" +owner = "OpenShell supervisor middleware CI lane" +lane = "linux-supervisor-middleware-paired" +blocker = "The middleware library suite passed on both revisions, while the content-guard smoke test remains candidate-only. A paired gateway, sandbox, and middleware fixture must compare Describe negotiation, policy distribution, guarded and unguarded traffic, failure policy, timeout, and payload limits; the field-name normalization remains the ledgered middleware-payload-name-normalized intentional change." + +[[result]] +id = "provider-profile-sources" +step = 11 +capability = "Builtin, user, and interceptor provider-profile source composition" +driver = "provider-profiles" +status = "platform_blocked" +owner = "OpenShell provider profile CI lane" +lane = "linux-provider-profiles-interceptor-paired" +blocker = "Server tests cover source construction and duplicate rejection, but no shared live interceptor catalog has been queried through both gateways. The assigned lane must compare builtin, user, and interceptor source ordering, normalized profile identities, duplicate rejection, discovery output, and failure handling with isolated persistent state." + +[[result]] +id = "inference-control-plane-configuration" +step = 11 +capability = "Persisted provider and route configuration delivered to sandbox inference" +driver = "inference" +status = "platform_blocked" +owner = "OpenShell inference E2E lane" +lane = "linux-inference-provider-routing-paired" +blocker = "Existing inference routing tests are candidate-oriented and this host has no assigned paired provider and model-service fixture. The lane must configure providers and routes through each gateway control plane, observe the effective supervisor bundle, and compare model discovery and an inference request without reusing database state." + +[[result]] +id = "credential-driver-selection-and-kek" +step = 11 +capability = "Credential-driver selection, KEK handling, encrypted storage, and secret containment" +driver = "credentials" +status = "platform_blocked" +owner = "OpenShell credential security CI lane" +lane = "linux-credential-kek-paired" +blocker = "Both server suites passed deterministic selection, KEK, and credential tests, but they are revision-local tests rather than one candidate-owned live oracle. A paired lane must start isolated gateways with independent databases and KEKs, compare default encrypted storage and invalid selections, store and resolve opaque credentials, and verify that logs and configuration do not disclose secrets." + +[[result]] +id = "credential-driver-backend-tables" +step = 11 +capability = "In-tree and remote credential-driver transport and opaque credential lifecycle" +driver = "credential-driver" +status = "platform_blocked" +owner = "OpenShell credential driver E2E lane" +lane = "kubernetes-vault-uds-credential-drivers-paired" +blocker = "The host has no assigned disposable Kubernetes Secrets or Vault backend and no retained paired UDS credential-driver execution. Existing backend E2E coverage is candidate-oriented. The assigned lane must compare in-tree and remote transport validation plus opaque store and retrieve behavior using isolated namespaces, Vault state, sockets, databases, and credentials." diff --git a/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt b/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt new file mode 100644 index 0000000000..823c5dd1ba --- /dev/null +++ b/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt @@ -0,0 +1,101 @@ +schema_v2_step11_deterministic_attestation_version=1 +variant=baseline +source_commit=74960ebfaeec4673885089ed995fad902459749f +source_tree_clean=true +cargo_target_scope=checkout-local +rustc_wrapper=disabled +cargo 1.95.0 (f2d3ce0bd 2026-03-21) +rustc 1.95.0 (59807616e 2026-04-14) + +=== suite:server-lib === +command=cargo test -q -p openshell-server --lib +--- output --- + +running 1463 tests +....................................................................................... 87/1463 +....................................................................................... 174/1463 +...............................................................................i....... 261/1463 +......................i................................................................ 348/1463 +..............................................................................i........ 435/1463 +....................................................................................... 522/1463 +....................................................................................... 609/1463 +....................................................................................... 696/1463 +....................................................................................... 783/1463 +......................................................................................i 870/1463 +....................................................................................... 957/1463 +....................................................................................... 1044/1463 +....................................................................................... 1131/1463 +................................................................................i...... 1218/1463 +.........................................ii............................................ 1305/1463 +...............................i....................................................... 1392/1463 +....................................................................... +test result: ok. 1455 passed; 0 failed; 8 ignored; 0 measured; 0 filtered out; finished in 9.13s + +--- exit_status:0 --- + +=== suite:gateway-interceptors === +command=cargo test -q -p openshell-gateway-interceptors +--- output --- + +running 49 tests +................................................. +test result: ok. 49 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.32s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:supervisor-middleware === +command=cargo test -q -p openshell-supervisor-middleware +--- output --- + +running 85 tests +..................................................................................... +test result: ok. 85 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.52s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:otel-test-support === +command=cargo test -q -p openshell-otel-test-support +--- output --- + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:multiplex-tls-integration === +command=cargo test -q -p openshell-server --test multiplex_tls_integration +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.14s + +--- exit_status:0 --- + +=== suite:edge-tunnel-auth === +command=cargo test -q -p openshell-server --test edge_tunnel_auth +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + +--- exit_status:0 --- + +attestation_complete=true diff --git a/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt b/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt new file mode 100644 index 0000000000..5b2cab4905 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt @@ -0,0 +1,102 @@ +schema_v2_step11_deterministic_attestation_version=1 +variant=candidate +source_commit=363d8540830b2ea294d43198daa2b7a283a2face +source_tree_clean=true +cargo_target_scope=checkout-local +rustc_wrapper=disabled +cargo 1.95.0 (f2d3ce0bd 2026-03-21) +rustc 1.95.0 (59807616e 2026-04-14) + +=== suite:server-lib === +command=cargo test -q -p openshell-server --lib +--- output --- + +running 1489 tests +....................................................................................... 87/1489 +....................................................................................... 174/1489 +....................................................................................... 261/1489 +...........i.............................i............................................. 348/1489 +....................................................................................... 435/1489 +................i...................................................................... 522/1489 +....................................................................................... 609/1489 +....................................................................................... 696/1489 +....................................................................................... 783/1489 +....................................................................................... 870/1489 +........................i.............................................................. 957/1489 +....................................................................................... 1044/1489 +....................................................................................... 1131/1489 +....................................................................................... 1218/1489 +..................i...............................................ii................... 1305/1489 +........................................................i.............................. 1392/1489 +....................................................................................... 1479/1489 +.......... +test result: ok. 1481 passed; 0 failed; 8 ignored; 0 measured; 0 filtered out; finished in 9.12s + +--- exit_status:0 --- + +=== suite:gateway-interceptors === +command=cargo test -q -p openshell-gateway-interceptors +--- output --- + +running 49 tests +................................................. +test result: ok. 49 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.27s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:supervisor-middleware === +command=cargo test -q -p openshell-supervisor-middleware +--- output --- + +running 85 tests +..................................................................................... +test result: ok. 85 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:otel-test-support === +command=cargo test -q -p openshell-otel-test-support +--- output --- + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:multiplex-tls-integration === +command=cargo test -q -p openshell-server --test multiplex_tls_integration +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + +--- exit_status:0 --- + +=== suite:edge-tunnel-auth === +command=cargo test -q -p openshell-server --test edge_tunnel_auth +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s + +--- exit_status:0 --- + +attestation_complete=true diff --git a/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py b/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py new file mode 100644 index 0000000000..dc83417e85 --- /dev/null +++ b/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate schema-v2 Step 11 cross-cutting capability dispositions.""" + +from __future__ import annotations + +import hashlib +import re +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +DISPOSITIONS_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml" +) +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" +INTENTIONAL_CHANGES_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-intentional-changes.toml" +) + +STEP_11_CAPABILITY_IDS = { + "credential-driver-backend-tables", + "credential-driver-selection-and-kek", + "gateway-interceptor-registration", + "gateway-minted-sandbox-jwt", + "inference-control-plane-configuration", + "mtls-user-authentication", + "oidc-bearer-authentication", + "otlp-observability", + "provider-profile-sources", + "supervisor-middleware-registration", + "unsafe-unauthenticated-user-mode", +} +EXPECTED_SUITES = { + "server-lib": "cargo test -q -p openshell-server --lib", + "gateway-interceptors": "cargo test -q -p openshell-gateway-interceptors", + "supervisor-middleware": "cargo test -q -p openshell-supervisor-middleware", + "otel-test-support": "cargo test -q -p openshell-otel-test-support", + "multiplex-tls-integration": ( + "cargo test -q -p openshell-server --test multiplex_tls_integration" + ), + "edge-tunnel-auth": "cargo test -q -p openshell-server --test edge_tunnel_auth", +} +EXPECTED_CANDIDATE_COMMIT = "363d8540830b2ea294d43198daa2b7a283a2face" +EXPECTED_INTENTIONAL_CHANGES = { + "gateway-jwt-zero-sentinel-removed": "gateway-minted-sandbox-jwt", + "middleware-payload-name-normalized": "supervisor-middleware-registration", +} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def assert_full_sha(value: object, field: str) -> str: + assert isinstance(value, str), f"{field} must be a string" + assert re.fullmatch(r"[0-9a-f]{40}", value), f"{field} must be a full SHA" + return value + + +def test_step_11_dispositions_cover_exact_cross_cutting_capabilities() -> None: + manifest = load_toml(DISPOSITIONS_PATH) + + assert set(manifest) == { + "manifest_version", + "baseline_commit", + "candidate_commit", + "overall_status", + "deterministic_preflight", + "capabilities", + } + assert manifest["manifest_version"] == 1 + assert manifest["overall_status"] == "platform_blocked" + assert ( + assert_full_sha(manifest["baseline_commit"], "baseline_commit") + == load_toml(CAPABILITY_PATH)["baseline_commit"] + ) + assert ( + assert_full_sha(manifest["candidate_commit"], "candidate_commit") + == EXPECTED_CANDIDATE_COMMIT + ) + + capabilities = manifest["capabilities"] + assert {entry["id"] for entry in capabilities} == STEP_11_CAPABILITY_IDS + assert len(capabilities) == len(STEP_11_CAPABILITY_IDS) + for entry in capabilities: + assert set(entry) == {"id", "status", "owner", "lane", "blocker"} + assert entry["status"] == "platform_blocked" + assert all(entry[field].strip() for field in ("owner", "lane")) + assert len(entry["blocker"]) >= 160 + + +def test_step_11_capability_ids_exist_in_inventory() -> None: + inventory_ids = { + entry["id"] for entry in load_toml(CAPABILITY_PATH)["capabilities"] + } + assert inventory_ids >= STEP_11_CAPABILITY_IDS + + +def test_deterministic_preflight_is_not_reported_as_live_parity() -> None: + manifest = load_toml(DISPOSITIONS_PATH) + preflight = manifest["deterministic_preflight"] + + assert set(preflight) == { + "status", + "baseline_attestation", + "baseline_attestation_sha256", + "candidate_attestation", + "candidate_attestation_sha256", + "commands", + "evidence", + } + assert preflight["status"] == "pass" + assert set(preflight["commands"]) == set(EXPECTED_SUITES.values()) + assert len(preflight["commands"]) == len(EXPECTED_SUITES) + assert len(preflight["evidence"]) >= 4 + assert all(item.strip() for item in preflight["evidence"]) + + assert manifest["overall_status"] == "platform_blocked" + assert all( + entry["status"] == "platform_blocked" for entry in manifest["capabilities"] + ) + + attestations = { + "baseline": manifest["baseline_commit"], + "candidate": manifest["candidate_commit"], + } + for variant, source_commit in attestations.items(): + path_field = f"{variant}_attestation" + digest_field = f"{variant}_attestation_sha256" + path = REPO_ROOT / preflight[path_field] + assert path.is_file(), f"missing tracked Step 11 attestation: {path}" + assert re.fullmatch(r"[0-9a-f]{64}", preflight[digest_field]) + assert hashlib.sha256(path.read_bytes()).hexdigest() == preflight[digest_field] + + attestation = path.read_text(encoding="utf-8") + assert "schema_v2_step11_deterministic_attestation_version=1\n" in attestation + assert f"variant={variant}\n" in attestation + assert f"source_commit={source_commit}\n" in attestation + assert "source_tree_clean=true\n" in attestation + assert "cargo_target_scope=checkout-local\n" in attestation + assert "rustc_wrapper=disabled\n" in attestation + assert attestation.endswith("\nattestation_complete=true\n") + suite_sections = {} + for section in attestation.split("\n=== suite:")[1:]: + suite, separator, body = section.partition(" ===\n") + assert separator + assert suite not in suite_sections + suite_sections[suite] = body + assert set(suite_sections) == set(EXPECTED_SUITES) + for suite, command in EXPECTED_SUITES.items(): + section = suite_sections[suite] + assert section.startswith(f"command={command} \n--- output ---\n") + assert section.count("--- exit_status:0 ---") == 1 + assert "--- exit_status:" not in section.replace( + "--- exit_status:0 ---", "" + ) + + +def test_step_11_representation_changes_remain_in_intentional_change_ledger() -> None: + changes = { + entry["id"]: entry + for entry in load_toml(INTENTIONAL_CHANGES_PATH)["intentional_changes"] + } + + for change_id, capability_id in EXPECTED_INTENTIONAL_CHANGES.items(): + assert change_id in changes + change = changes[change_id] + assert change["parity_disposition"] == "intentional_change" + assert capability_id in change["validation_capability_ids"] diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index 591257260b..d2c0f252e2 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -19,6 +19,9 @@ COMPUTE_BOUNDARY_PATH = ( REPO_ROOT / "e2e/configs/gateway/schema-v2-compute-boundary-comparison.json" ) +CROSS_CUTTING_DISPOSITIONS_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml" +) REQUIRED_HEADER_FIELDS = { "manifest_version", @@ -51,6 +54,19 @@ "vm-launch-and-resource-configuration", } REQUIRED_STEP_10_IDS = {"compute-driver-boundary-parity"} +REQUIRED_STEP_11_IDS = { + "credential-driver-backend-tables", + "credential-driver-selection-and-kek", + "gateway-interceptor-registration", + "gateway-minted-sandbox-jwt", + "inference-control-plane-configuration", + "mtls-user-authentication", + "oidc-bearer-authentication", + "otlp-observability", + "provider-profile-sources", + "supervisor-middleware-registration", + "unsafe-unauthenticated-user-mode", +} STEP_10_CANDIDATE_COMMIT = "4a39da510e4d278a24dd60291149519c9a570b46" STEP_10_REPORT_SHA256 = ( "65541eec5f642461a88b04b5459474fd7a475adeb7071a65a53fe183caad6a01" @@ -335,6 +351,26 @@ def test_step_10_records_verified_compute_boundary_parity(tmp_path: Path) -> Non assert reproduced.read_bytes() == COMPUTE_BOUNDARY_PATH.read_bytes() +def test_step_11_records_cross_cutting_live_lane_dispositions() -> None: + results = { + result["id"]: result + for result in load_toml(RESULTS_PATH)["result"] + if result["step"] == 11 + } + dispositions = { + entry["id"]: entry + for entry in load_toml(CROSS_CUTTING_DISPOSITIONS_PATH)["capabilities"] + } + + assert set(results) == REQUIRED_STEP_11_IDS + assert set(dispositions) == REQUIRED_STEP_11_IDS + for result_id, result in results.items(): + disposition = dispositions[result_id] + assert result["status"] == "platform_blocked" + for field in ("status", "owner", "lane", "blocker"): + assert result[field] == disposition[field] + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": From ae51165a4fdc4515783ddc53990c8cd65b3b2a0f Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 11:25:04 -0400 Subject: [PATCH 37/42] fix(packaging): preflight gateway config upgrades Signed-off-by: Jesse Jaggars --- Cargo.lock | 1 + .../openshell-gateway-interceptors/src/lib.rs | 5 + .../src/plan.rs | 2 +- crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/cli.rs | 373 +++++++++++++++++- .../src/compute/driver_config.rs | 63 +-- crates/openshell-server/src/config_file.rs | 290 +++++++++++++- crates/openshell-server/src/tls.rs | 52 +-- .../src/lib.rs | 5 + deploy/deb/openshell-gateway.service | 1 + deploy/man/openshell-gateway.8.md | 29 +- docs/about/installation.mdx | 21 + docs/reference/gateway-config.mdx | 38 ++ .../schema-v2-intentional-changes.toml | 4 +- .../gateway/schema-v2-live-results.toml | 43 ++ .../schema-v2-parity-gap-dispositions.toml | 12 +- ...eway_schema_v2_intentional_changes_test.py | 8 +- .../gateway_schema_v2_live_results_test.py | 40 ++ ..._schema_v2_parity_gap_dispositions_test.py | 36 ++ python/openshell/release_formula_test.py | 35 ++ skills/debug-openshell-cluster/SKILL.md | 16 + snapcraft.yaml | 6 +- tasks/scripts/snap-gateway-wrapper.sh | 67 +++- tasks/scripts/test-packaging-assets.sh | 49 +++ tasks/scripts/test-snap-gateway-wrapper.sh | 192 +++++++++ 25 files changed, 1311 insertions(+), 78 deletions(-) create mode 100755 tasks/scripts/test-snap-gateway-wrapper.sh diff --git a/Cargo.lock b/Cargo.lock index 174c25aba8..89de4153b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4351,6 +4351,7 @@ dependencies = [ "jsonwebtoken", "k8s-openapi", "kube", + "libc", "metrics", "metrics-exporter-prometheus", "miette", diff --git a/crates/openshell-gateway-interceptors/src/lib.rs b/crates/openshell-gateway-interceptors/src/lib.rs index d5bb5df59e..8d05878dd2 100644 --- a/crates/openshell-gateway-interceptors/src/lib.rs +++ b/crates/openshell-gateway-interceptors/src/lib.rs @@ -43,6 +43,11 @@ pub enum InterceptorError { pub type Result = std::result::Result; +/// Validate static interceptor configuration without opening a transport. +pub fn validate_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { + plan::validate_interceptor_configs(configs) +} + pub(crate) type ExtensionChannel = InterceptedService; /// Return `None` when no interceptors are configured. diff --git a/crates/openshell-gateway-interceptors/src/plan.rs b/crates/openshell-gateway-interceptors/src/plan.rs index c89de6f6ee..3328348249 100644 --- a/crates/openshell-gateway-interceptors/src/plan.rs +++ b/crates/openshell-gateway-interceptors/src/plan.rs @@ -740,7 +740,7 @@ fn validate_service_config(config: &GatewayInterceptorConfig) -> Result<()> { Ok(()) } -fn validate_interceptor_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { +pub fn validate_interceptor_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { let mut names = BTreeSet::new(); for config in configs { validate_service_config(config)?; diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 2619fee5cc..bf4c3f04e1 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -36,6 +36,7 @@ k8s-openapi = { workspace = true } tokio = { workspace = true } socket2 = { workspace = true } nix = { workspace = true } +libc = "0.2" # gRPC tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index ce35612fe9..043f9970dc 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -38,9 +38,30 @@ struct Cli { enum Commands { /// Generate mTLS PKI and write Kubernetes Secrets (Helm pre-install hook). GenerateCerts(certgen::CertgenArgs), + /// Inspect gateway configuration without starting the service. + Config(ConfigArgs), } #[derive(clap::Args, Debug)] +struct ConfigArgs { + #[command(subcommand)] + command: ConfigCommand, +} + +#[derive(clap::Subcommand, Debug)] +enum ConfigCommand { + /// Validate the selected configuration without modifying it or starting the gateway. + Preflight(ConfigPreflightArgs), +} + +#[derive(clap::Args, Debug)] +struct ConfigPreflightArgs { + /// Explicit configuration path. Overrides `OPENSHELL_GATEWAY_CONFIG` and XDG discovery. + #[arg(long)] + path: Option, +} + +#[derive(clap::Args, Clone, Debug)] #[allow(clippy::struct_excessive_bools)] struct RunArgs { /// Path to a TOML configuration file (see RFC 0003). @@ -235,6 +256,9 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry match cli.command { Some(Commands::GenerateCerts(args)) => certgen::run(args).await, + Some(Commands::Config(args)) => match args.command { + ConfigCommand::Preflight(args) => run_config_preflight(args, cli.run, &matches), + }, None => Box::pin(run_from_args(cli.run, matches, compute_drivers)).await, } } @@ -636,13 +660,129 @@ fn parse_compute_driver(value: &str) -> std::result::Result { openshell_core::config::normalize_compute_driver_name(value) } +fn run_config_preflight( + args: ConfigPreflightArgs, + mut run: RunArgs, + matches: &ArgMatches, +) -> Result<()> { + let path = if let Some(path) = args.path { + Some(path) + } else { + resolve_config_path(&run)? + }; + let Some(path) = path else { + return Ok(()); + }; + let file = config_file::preflight(&path).map_err(|error| miette::miette!("{error}"))?; + merge_file_into_args(&mut run, &file.openshell.gateway, matches); + validate_preflight_semantics(&run, matches, &file) + .map_err(|_| config_file::ConfigPreflightError::invalid_current(&path)) + .map_err(|error| miette::miette!("{error}")) +} + +fn validate_preflight_semantics( + args: &RunArgs, + matches: &ArgMatches, + file: &ConfigFile, +) -> Result<()> { + let gateway = &file.openshell.gateway; + validate_grpc_rate_limit_args( + args.grpc_rate_limit_requests, + args.grpc_rate_limit_window_seconds, + )?; + GuestTlsPaths::validate_configuration(Some(gateway), args.disable_tls) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; + + let has_client_ca = args.tls_client_ca.is_some(); + let mtls_auth_enabled = resolve_mtls_auth_enabled(args, matches, Some(file), None); + if args.disable_tls && has_client_ca { + return Err(miette::miette!( + "--disable-tls and --tls-client-ca are mutually exclusive" + )); + } + if mtls_auth_enabled && args.disable_tls { + return Err(miette::miette!("mTLS user authentication requires TLS")); + } + if mtls_auth_enabled && !has_client_ca { + return Err(miette::miette!( + "mTLS user authentication requires --tls-client-ca" + )); + } + if !args.disable_tls && args.tls_cert.is_some() != args.tls_key.is_some() { + return Err(miette::miette!( + "gateway TLS requires both --tls-cert and --tls-key" + )); + } + if !args.disable_tls && has_client_ca && args.tls_cert.is_none() && args.tls_key.is_none() { + return Err(miette::miette!( + "an explicit --tls-client-ca requires --tls-cert and --tls-key" + )); + } + if !args.disable_tls + && let Some(tls) = gateway.tls.as_ref() + { + crate::tls::validate_external_cert_config( + tls.external_cert_path.as_deref(), + tls.external_key_path.as_deref(), + &tls.external_server_names, + ) + .map_err(|error| miette::miette!("{error}"))?; + } + openshell_gateway_interceptors::validate_configs(&gateway.interceptors) + .map_err(|error| miette::miette!("{error}"))?; + let mut middleware_names = std::collections::HashSet::new(); + for middleware in &file.openshell.supervisor.middleware { + let registration = openshell_core::proto::SupervisorMiddlewareService::try_from(middleware) + .map_err(|error| miette::miette!("{error}"))?; + openshell_supervisor_middleware::validate_registration_config(®istration)?; + if !middleware_names.insert(registration.name) { + return Err(miette::miette!( + "duplicate supervisor middleware registration" + )); + } + } + if args.name.trim().is_empty() { + return Err(miette::miette!("gateway name must not be empty")); + } + + let health_bind = resolve_aux_listener( + args.bind_address, + args.health_port, + matches, + "health_port", + || gateway.health_bind_address, + ); + let metrics_bind = resolve_aux_listener( + args.bind_address, + args.metrics_port, + matches, + "metrics_port", + || gateway.metrics_bind_address, + ); + if health_bind.is_some_and(|address| address.port() == args.port) + || metrics_bind.is_some_and(|address| address.port() == args.port) + || health_bind + .zip(metrics_bind) + .is_some_and(|(health, metrics)| health.port() == metrics.port()) + { + return Err(miette::miette!("gateway listener ports must be distinct")); + } + Ok(()) +} + fn resolve_config_path(args: &RunArgs) -> Result> { if let Some(path) = args.config.clone() { return Ok(Some(path)); } let default_path = defaults::default_gateway_config_path()?; - Ok(default_path.is_file().then_some(default_path)) + match std::fs::symlink_metadata(&default_path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + // Fail closed when the path exists or cannot be inspected. Returning it + // lets the shared loader produce the same safe, path-specific error used + // for an explicit configuration instead of treating it as absent. + Ok(_) | Err(_) => Ok(Some(default_path)), + } } fn apply_runtime_defaults(args: &mut RunArgs) -> Result> { @@ -1387,6 +1527,237 @@ mod tests { )); } + #[test] + fn config_preflight_subcommand_parses_without_runtime_requirements() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _db = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + + let cli = Cli::try_parse_from([ + "openshell-gateway", + "config", + "preflight", + "--path", + "/tmp/gateway.toml", + ]) + .expect("config preflight should parse without runtime arguments"); + + assert!(matches!( + cli.command, + Some(super::Commands::Config(super::ConfigArgs { + command: super::ConfigCommand::Preflight(_) + })) + )); + } + + #[test] + fn config_preflight_validates_explicit_path_without_creating_state() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let state_parent = tempfile::tempdir().unwrap(); + let state_home = state_parent.path().join("not-created"); + let config = config_home.path().join("gateway.toml"); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _state = EnvVarGuard::set("XDG_STATE_HOME", state_home.to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight( + super::ConfigPreflightArgs { path: Some(config) }, + run, + &matches, + ) + .expect("valid explicit config passes preflight"); + + assert!( + !state_home.exists(), + "preflight must not create runtime state" + ); + } + + #[test] + fn config_preflight_explicit_path_overrides_environment_selection() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = tempfile::tempdir().unwrap(); + let legacy = dir.path().join("legacy.toml"); + let current = dir.path().join("current.toml"); + std::fs::write(&legacy, "[openshell]\nversion = 1\n").unwrap(); + std::fs::write(¤t, "[openshell]\nversion = 2\n").unwrap(); + let _config_env = EnvVarGuard::set("OPENSHELL_GATEWAY_CONFIG", legacy.to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + let error = super::run_config_preflight( + super::ConfigPreflightArgs { path: None }, + run.clone(), + &matches, + ) + .expect_err("environment-selected legacy config must fail"); + assert!(error.to_string().contains("category=legacy_schema_v1")); + + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(current), + }, + run, + &matches, + ) + .expect("explicit preflight path must override environment selection"); + } + + #[test] + fn config_preflight_allows_absent_auto_discovery_but_rejects_explicit_absence() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight( + super::ConfigPreflightArgs { path: None }, + run.clone(), + &matches, + ) + .expect("absent auto-discovered config is optional"); + + let missing = config_home.path().join("missing.toml"); + let error = super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(missing.clone()), + }, + run, + &matches, + ) + .expect_err("explicit missing config must fail"); + assert!(error.to_string().contains("category=missing_path")); + assert!(error.to_string().contains(&missing.display().to_string())); + } + + #[test] + fn config_preflight_rejects_effective_semantic_errors() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let cases = [ + ( + "rate-limit", + "[openshell]\nversion = 2\n[openshell.gateway]\nname = 'secret-semantic-marker'\ngrpc_rate_limit_requests = 10\n", + ), + ( + "guest-tls", + "[openshell]\nversion = 2\n[openshell.gateway]\nguest_tls_ca = '/tls/ca.pem'\n", + ), + ( + "external-tls", + "[openshell]\nversion = 2\n[openshell.gateway.tls]\ncert_path = '/tls/server.pem'\nkey_path = '/tls/server-key.pem'\nexternal_cert_path = '/tls/external.pem'\nexternal_server_names = ['external.example']\n", + ), + ( + "interceptor", + "[openshell]\nversion = 2\n[[openshell.gateway.interceptors]]\nname = ''\ngrpc_endpoint = 'https://interceptor.example'\n", + ), + ( + "middleware", + "[openshell]\nversion = 2\n[[openshell.supervisor.middleware]]\nname = 'guard'\ngrpc_endpoint = 'http://127.0.0.1:50051'\nallow_insecure_transport = true\nmax_payload_bytes = 1024\ntimeout = 'invalid'\n", + ), + ]; + + for (name, contents) in cases { + let path = dir.path().join(format!("{name}.toml")); + std::fs::write(&path, contents).unwrap(); + let before = std::fs::read(&path).unwrap(); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let result = super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(path.clone()), + }, + run, + &matches, + ); + let Err(error) = result else { + panic!("{name}: invalid effective configuration passed preflight"); + }; + assert!(error.to_string().contains("category=malformed"), "{name}"); + assert!(error.to_string().contains("detected_version=2"), "{name}"); + assert!(!error.to_string().contains("secret-semantic-marker")); + assert!(!format!("{error:?}").contains("secret-semantic-marker")); + assert_eq!(std::fs::read(&path).unwrap(), before, "{name}"); + } + } + + #[test] + fn config_preflight_matches_effective_tls_environment_semantics() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let partial_external = dir.path().join("partial-external.toml"); + std::fs::write( + &partial_external, + "[openshell]\nversion = 2\n[openshell.gateway.tls]\ncert_path = '/tls/server.pem'\nkey_path = '/tls/server-key.pem'\nexternal_cert_path = '/tls/external.pem'\nexternal_server_names = ['external.example']\n", + ) + .unwrap(); + let disable_tls = EnvVarGuard::set("OPENSHELL_DISABLE_TLS", "true"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(partial_external), + }, + run, + &matches, + ) + .expect("inactive TLS table must not block an effective plaintext gateway"); + drop(disable_tls); + + let config = dir.path().join("client-ca-only.toml"); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); + let _disable_tls = EnvVarGuard::remove("OPENSHELL_DISABLE_TLS"); + let _client_ca = EnvVarGuard::set("OPENSHELL_TLS_CLIENT_CA", "/tls/ca.pem"); + let _cert = EnvVarGuard::remove("OPENSHELL_TLS_CERT"); + let _key = EnvVarGuard::remove("OPENSHELL_TLS_KEY"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let error = super::run_config_preflight( + super::ConfigPreflightArgs { path: Some(config) }, + run, + &matches, + ) + .expect_err("client CA without an explicit server pair must fail before cert generation"); + assert!(error.to_string().contains("category=malformed")); + } + + #[test] + fn config_preflight_allows_complete_future_generated_tls_paths() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("gateway.toml"); + std::fs::write( + &path, + "[openshell]\nversion = 2\n[openshell.gateway]\nguest_tls_ca = '/future/ca.pem'\nguest_tls_cert = '/future/client.pem'\nguest_tls_key = '/future/client-key.pem'\n", + ) + .unwrap(); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight( + super::ConfigPreflightArgs { path: Some(path) }, + run, + &matches, + ) + .expect("complete package-generated TLS paths may not exist before certificate generation"); + } + #[test] fn bare_invocation_with_no_db_url_parses_for_runtime_defaults() { // db_url is Option at the clap level so subcommand parsing diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 40ab449c66..24254ef8c7 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -28,6 +28,42 @@ impl GuestTlsPaths { } impl GuestTlsPaths { + fn configured_paths( + gateway: &config_file::GatewayFileSection, + ) -> (Option<&PathBuf>, Option<&PathBuf>, Option<&PathBuf>) { + ( + gateway.guest_tls_ca.as_ref(), + gateway.guest_tls_cert.as_ref(), + gateway.guest_tls_key.as_ref(), + ) + } + + /// Validate guest TLS relationships without reading certificate files. + pub(crate) fn validate_configuration( + gateway: Option<&config_file::GatewayFileSection>, + tls_disabled: bool, + ) -> std::result::Result<(), String> { + let configured = gateway.map(Self::configured_paths); + let provided = configured + .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); + if tls_disabled && provided { + return Err( + "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" + .to_string(), + ); + } + if let Some((ca, cert, key)) = configured + && (ca.is_some() || cert.is_some() || key.is_some()) + && (ca.is_none() || cert.is_none() || key.is_none()) + { + return Err( + "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" + .to_string(), + ); + } + Ok(()) + } + /// Resolve gateway-owned guest TLS inputs. Explicit TOML values take /// precedence over the package-managed local bundle; partial bundles are /// rejected before any driver is deserialized or constructed. @@ -36,35 +72,12 @@ impl GuestTlsPaths { local: Option<&LocalTlsPaths>, tls_disabled: bool, ) -> std::result::Result, String> { - let configured = gateway.map(|gateway| { - ( - gateway.guest_tls_ca.as_ref(), - gateway.guest_tls_cert.as_ref(), - gateway.guest_tls_key.as_ref(), - ) - }); - let provided = configured - .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); - + Self::validate_configuration(gateway, tls_disabled)?; if tls_disabled { - if provided { - return Err( - "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" - .to_string(), - ); - } return Ok(None); } - if let Some((ca, cert, key)) = configured - && (ca.is_some() || cert.is_some() || key.is_some()) - { - let (Some(ca), Some(cert), Some(key)) = (ca, cert, key) else { - return Err( - "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" - .to_string(), - ); - }; + if let Some((Some(ca), Some(cert), Some(key))) = gateway.map(Self::configured_paths) { for (field, path) in [ ("guest_tls_ca", ca), ("guest_tls_cert", cert), diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 3efd6cf7a5..e15f130e18 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -20,7 +20,8 @@ //! values. use std::collections::BTreeMap; -use std::io::Cursor; +use std::fs::OpenOptions; +use std::io::{Cursor, Read as _}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -329,6 +330,8 @@ pub enum ConfigFileError { #[source] source: std::io::Error, }, + #[error("gateway config path '{}' is not a regular file ({kind})", path.display())] + NonRegular { path: PathBuf, kind: &'static str }, #[error("failed to parse gateway config file '{}': {source}", path.display())] Parse { path: PathBuf, @@ -379,20 +382,103 @@ pub enum ConfigFileError { }, } -/// Load and validate a TOML config file. -/// -/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running -/// without a config file still uses CLI, environment, and built-in defaults. -#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] -pub fn load(path: &Path) -> Result { - let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { - path: path.to_path_buf(), +const CONFIG_MIGRATION_URL: &str = + "https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2"; + +/// Stable package-preflight failure with no configuration contents attached. +#[derive(Debug, thiserror::Error)] +#[error( + "gateway config preflight failed: path='{}' category={} detected_version={}; the file was preserved unchanged; migrate it before restarting the gateway: {CONFIG_MIGRATION_URL}", + path.display(), + category, + detected_version +)] +pub struct ConfigPreflightError { + path: PathBuf, + category: &'static str, + detected_version: String, +} + +impl ConfigPreflightError { + pub(crate) fn invalid_current(path: &Path) -> Self { + Self { + path: path.to_path_buf(), + category: "malformed", + detected_version: SCHEMA_VERSION.to_string(), + } + } + + #[cfg(test)] + fn category(&self) -> &'static str { + self.category + } + + #[cfg(test)] + fn detected_version(&self) -> &str { + &self.detected_version + } +} + +fn non_regular_kind(metadata: &std::fs::Metadata) -> &'static str { + let file_type = metadata.file_type(); + if file_type.is_symlink() { + "symlink" + } else if file_type.is_dir() { + "directory" + } else { + "non-regular" + } +} + +fn read_config_contents(path: &Path) -> Result { + let path_buf = path.to_path_buf(); + let metadata = std::fs::symlink_metadata(path).map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), source, })?; + if !metadata.file_type().is_file() { + return Err(ConfigFileError::NonRegular { + path: path_buf, + kind: non_regular_kind(&metadata), + }); + } + + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let mut file = options.open(path).map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), + source, + })?; + let opened_metadata = file.metadata().map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), + source, + })?; + if !opened_metadata.file_type().is_file() { + return Err(ConfigFileError::NonRegular { + path: path_buf, + kind: non_regular_kind(&opened_metadata), + }); + } + + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|source| ConfigFileError::Io { + path: path.to_path_buf(), + source, + })?; + Ok(contents) +} + +fn parse_and_validate(path: &Path, contents: &str) -> Result { if contents.trim().is_empty() { return Err(ConfigFileError::MissingVersion); } - let file: ConfigFile = toml::from_str(&contents).map_err(|source| ConfigFileError::Parse { + let file: ConfigFile = toml::from_str(contents).map_err(|source| ConfigFileError::Parse { path: path.to_path_buf(), source, })?; @@ -434,6 +520,97 @@ pub fn load(path: &Path) -> Result { Ok(file) } +fn detected_version(contents: &str) -> String { + let Ok(value) = contents.parse::() else { + return "unavailable".to_string(); + }; + let Some(openshell) = value.get("openshell") else { + return "missing".to_string(); + }; + let Some(openshell) = openshell.as_table() else { + return "unavailable".to_string(); + }; + openshell.get("version").map_or_else( + || "missing".to_string(), + |version| { + version + .as_integer() + .map_or_else(|| "unavailable".to_string(), |version| version.to_string()) + }, + ) +} + +fn preflight_error( + path: &Path, + detected_version: String, + source: ConfigFileError, +) -> ConfigPreflightError { + let category = match &source { + ConfigFileError::NonRegular { .. } => "non_regular_path", + ConfigFileError::Io { source, .. } if source.kind() == std::io::ErrorKind::NotFound => { + "missing_path" + } + ConfigFileError::Io { source, .. } if source.kind() == std::io::ErrorKind::InvalidData => { + "malformed" + } + ConfigFileError::Io { .. } => "unreadable_path", + ConfigFileError::MissingVersion => "missing_version", + ConfigFileError::UnsupportedVersion { version: 1 } => "legacy_schema_v1", + ConfigFileError::UnsupportedVersion { version } if *version > SCHEMA_VERSION => { + "future_version" + } + ConfigFileError::UnsupportedVersion { .. } => "unsupported_version", + ConfigFileError::Parse { .. } + | ConfigFileError::SecretInFile { .. } + | ConfigFileError::InvalidValue { .. } + | ConfigFileError::InvalidDriverTable { .. } + | ConfigFileError::MiddlewareTlsCaRead { .. } + | ConfigFileError::MiddlewareTlsCaInvalid { .. } => "malformed", + }; + ConfigPreflightError { + path: path.to_path_buf(), + category, + detected_version, + } +} + +/// Validate a gateway configuration without starting the gateway or writing state. +#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] +pub fn preflight(path: &Path) -> Result { + let contents = read_config_contents(path) + .map_err(|source| preflight_error(path, "unavailable".to_string(), source))?; + let version = detected_version(&contents); + if version == "missing" { + return Err(preflight_error( + path, + version, + ConfigFileError::MissingVersion, + )); + } + if let Ok(version_number) = version.parse::() + && version_number != SCHEMA_VERSION + { + return Err(preflight_error( + path, + version, + ConfigFileError::UnsupportedVersion { + version: version_number, + }, + )); + } + parse_and_validate(path, &contents).map_err(|source| preflight_error(path, version, source)) +} + +/// Load and validate a TOML config file. +/// +/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running +/// without a config file still uses CLI, environment, and built-in defaults. +#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] +pub fn load(path: &Path) -> Result { + let contents = read_config_contents(path)?; + parse_and_validate(path, &contents) +} + /// Return a driver's table without gateway-level inheritance. /// Driver-specific configuration belongs exclusively to /// `[openshell.drivers.]` in schema version 2. @@ -1044,6 +1221,99 @@ ssh_gateway_port = 8080 fn accepts_current_version() { let tmp = write_raw_tmp("[openshell]\nversion = 2\n"); load(tmp.path()).expect("schema version 2 must be accepted"); + preflight(tmp.path()).expect("schema version 2 must pass preflight"); + } + + #[test] + fn preflight_classifies_versions_and_malformed_files_without_disclosing_contents() { + for (contents, category, version) in [ + ("[openshell]\nversion = 1\n", "legacy_schema_v1", "1"), + ("[openshell]\n", "missing_version", "missing"), + ("[openshell]\nversion = 3\n", "future_version", "3"), + ("[openshell]\nversion = 0\n", "unsupported_version", "0"), + ("[openshell]\nversion = 'two'\n", "malformed", "unavailable"), + ( + "[openshell]\nversion = 2\nsecret-value = [", + "malformed", + "unavailable", + ), + ( + "[openshell]\nversion = 2\n[openshell.gateway]\nunknown = 'secret-value'\n", + "malformed", + "2", + ), + ] { + let tmp = write_raw_tmp(contents); + let error = preflight(tmp.path()).expect_err("preflight must reject fixture"); + assert_eq!(error.category(), category, "contents: {contents}"); + assert_eq!(error.detected_version(), version, "contents: {contents}"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains(&format!("category={category}"))); + assert!(diagnostic.contains(&format!("detected_version={version}"))); + assert!(diagnostic.contains("the file was preserved unchanged")); + assert!(diagnostic.contains(CONFIG_MIGRATION_URL)); + assert!(!diagnostic.contains("secret-value")); + assert!(!format!("{error:?}").contains("secret-value")); + assert!(std::error::Error::source(&error).is_none()); + } + } + + #[test] + fn preflight_classifies_missing_path() { + let path = Path::new("/nonexistent/openshell-preflight.toml"); + let error = preflight(path).expect_err("missing explicit path must fail"); + assert_eq!(error.category(), "missing_path"); + assert_eq!(error.detected_version(), "unavailable"); + assert!(error.to_string().contains(&path.display().to_string())); + } + + #[test] + fn preflight_rejects_directory_as_non_regular() { + let tmp = tempfile::tempdir().unwrap(); + let error = preflight(tmp.path()).expect_err("directory must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + assert_eq!(error.detected_version(), "unavailable"); + } + + #[cfg(unix)] + #[test] + fn preflight_and_runtime_reject_symlinks_without_following_them() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("target.toml"); + let link = dir.path().join("gateway.toml"); + std::fs::write(&target, "[openshell]\nversion = 2\n").unwrap(); + symlink(&target, &link).unwrap(); + + let error = preflight(&link).expect_err("symlink must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + assert!(matches!( + load(&link), + Err(ConfigFileError::NonRegular { .. }) + )); + } + + #[cfg(unix)] + #[test] + fn preflight_rejects_fifo_without_blocking() { + use nix::sys::stat::Mode; + use nix::unistd::mkfifo; + + let dir = tempfile::tempdir().unwrap(); + let fifo = dir.path().join("gateway.toml"); + mkfifo(&fifo, Mode::S_IRUSR | Mode::S_IWUSR).unwrap(); + let error = preflight(&fifo).expect_err("FIFO must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + } + + #[test] + fn preflight_classifies_non_utf8_as_malformed() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), [0xff, 0xfe]).unwrap(); + let error = preflight(tmp.path()).expect_err("non-UTF-8 config must fail"); + assert_eq!(error.category(), "malformed"); + assert_eq!(error.detected_version(), "unavailable"); } #[test] diff --git a/crates/openshell-server/src/tls.rs b/crates/openshell-server/src/tls.rs index aa627746ce..4231407c7a 100644 --- a/crates/openshell-server/src/tls.rs +++ b/crates/openshell-server/src/tls.rs @@ -329,41 +329,49 @@ fn load_certified_key(cert_path: &Path, key_path: &Path) -> Result, external_key_path: Option<&Path>, external_server_names: &[String], -) -> Result>> { +) -> Result<()> { match (external_cert_path, external_key_path) { - (None, None) => Ok(None), (Some(_), None) => Err(Error::tls( "external_cert_path is set but external_key_path is missing", )), (None, Some(_)) => Err(Error::tls( "external_key_path is set but external_cert_path is missing", )), - (Some(ext_cert_path), Some(ext_key_path)) => { - if external_server_names.is_empty() { - return Err(Error::tls( - "external certificate is configured but external_server_names is empty — \ - the external cert would never be served", - )); - } - let internal = load_certified_key(cert_path, key_path)?; - let external = load_certified_key(ext_cert_path, ext_key_path)?; - Ok(Some(Arc::new(DualCertResolver { - internal, - external, - external_names: external_server_names.to_vec(), - }))) - } + (Some(_), Some(_)) if external_server_names.is_empty() => Err(Error::tls( + "external certificate is configured but external_server_names is empty — \ + the external cert would never be served", + )), + (None, None) | (Some(_), Some(_)) => Ok(()), } } +/// Build an SNI-based cert resolver when an external certificate is configured. +/// Returns `None` when no external cert is configured (single-cert mode). +fn build_cert_resolver( + cert_path: &Path, + key_path: &Path, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], +) -> Result>> { + validate_external_cert_config(external_cert_path, external_key_path, external_server_names)?; + let (Some(ext_cert_path), Some(ext_key_path)) = (external_cert_path, external_key_path) else { + return Ok(None); + }; + let internal = load_certified_key(cert_path, key_path)?; + let external = load_certified_key(ext_cert_path, ext_key_path)?; + Ok(Some(Arc::new(DualCertResolver { + internal, + external, + external_names: external_server_names.to_vec(), + }))) +} + /// Build a `ServerConfig` from certificate, key, and optional client CA files. fn build_server_config( cert_path: &Path, diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index c3a1f6f71a..32c9b5b037 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -739,6 +739,11 @@ impl Default for MiddlewareRegistry { } } +/// Validate one external middleware registration without opening its transport. +pub fn validate_registration_config(registration: &SupervisorMiddlewareService) -> Result<()> { + validate_registration(registration).map(|_| ()) +} + fn validate_registration(registration: &SupervisorMiddlewareService) -> Result { if !is_stable_identifier(®istration.name) { return Err(miette!( diff --git a/deploy/deb/openshell-gateway.service b/deploy/deb/openshell-gateway.service index 9724235ed6..c5f766a129 100644 --- a/deploy/deb/openshell-gateway.service +++ b/deploy/deb/openshell-gateway.service @@ -8,6 +8,7 @@ Type=simple StateDirectory=openshell/gateway Environment=OPENSHELL_LOCAL_TLS_DIR=%h/.local/state/openshell/tls EnvironmentFile=-%E/openshell/gateway.env +ExecStartPre=/usr/bin/openshell-gateway config preflight ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal ExecStart=/usr/bin/openshell-gateway Restart=on-failure diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 68439ea596..1369e5ddbf 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -14,6 +14,8 @@ openshell-gateway - OpenShell gateway server daemon **openshell-gateway** \[*OPTIONS*\] +**openshell-gateway** **config preflight** [**--path** *PATH*] + # DESCRIPTION **openshell-gateway** is the control-plane server for OpenShell. It @@ -110,6 +112,28 @@ Compute driver settings such as sandbox image, callback endpoint, image pull policy, network name, VM state directory, and guest TLS material are configured in the TOML file passed with **--config**. +# CONFIGURATION PREFLIGHT + +Validate a gateway configuration before starting the daemon: + + openshell-gateway config preflight [--path PATH] + +With no path, preflight validates a nonempty OPENSHELL_GATEWAY_CONFIG. If that +variable is unset, it optionally validates an auto-discovered XDG config. The +absence of either config succeeds. An explicit missing path, legacy schema-v1 +file, invalid TOML, symlink, or nonregular file fails with a nonzero status. +Preflight merges file and environment values and applies read-only startup checks +for rate-limit, TLS, interceptor, and middleware relationships. Preflight never +changes the file and reports that failed input was preserved. + +The Debian and Ubuntu systemd user unit runs preflight before certificate +generation, while retaining its EnvironmentFile and bare ExecStart behavior. The +Snap wrapper first validates a nonempty OPENSHELL_GATEWAY_CONFIG. Otherwise it +validates the canonical SNAP_COMMON/gateway.toml path whenever it exists or is a +symlink. A broken symlink fails preflight before the gateway is started. Correct +or manually migrate an operator-owned v1 file, then run preflight again before +restarting the service. + # SYSTEMD INTEGRATION The package installs a systemd user unit at @@ -126,8 +150,9 @@ View logs: journalctl --user -u openshell-gateway journalctl --user -u openshell-gateway -f -The unit runs **openshell-gateway generate-certs** as an **ExecStartPre** -step on first start. This generates a self-signed PKI bundle for mTLS +The unit runs **openshell-gateway config preflight** and then +**openshell-gateway generate-certs** as **ExecStartPre** steps. Certificate +generation creates a self-signed PKI bundle for mTLS and sandbox JWT signing material, adding missing JWT files to older TLS-only installs when needed. The packaged unit sets **OPENSHELL_LOCAL_TLS_DIR** to *~/.local/state/openshell/tls* and uses that diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 7064fe7a56..aacdb3380d 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -155,3 +155,24 @@ Kubernetes deployments use the OpenShell Helm chart. For step-by-step installati - To register, select, and inspect gateways, refer to [Gateways](/sandboxes/manage-gateways). - To supply API keys or tokens, refer to [Manage Providers](/sandboxes/manage-providers). - To control what the agent can access, refer to [Policies](/sandboxes/policies). + +## Validate a package-managed gateway configuration + +Debian and Ubuntu packages validate the selected gateway configuration before +generating local certificates or starting the service. Snap does the same before +its gateway daemon starts. The validation never changes the file. If startup reports a legacy schema, +malformed TOML, a symlink, or a nonregular configuration path, fix or manually +migrate the operator-owned file instead of deleting it. + +Check the selected file before restarting a service: + +```shell +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +``` + +Without --path, preflight checks a nonempty OPENSHELL_GATEWAY_CONFIG or an +auto-discovered XDG config; no config is also a successful result. Debian keeps +its bare service invocation and gateway.env semantics. Snap gives a nonempty +OPENSHELL_GATEWAY_CONFIG precedence over SNAP_COMMON/gateway.toml. +See [Gateway Configuration](/reference/gateway-config#gateway-config-preflight) +for preflight details and manual schema-v1 migration steps. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 75d5d2022d..b7e56c1cbb 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -967,3 +967,41 @@ compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" ``` + +## Preflight package configuration {#gateway-config-preflight} + +Before starting a package-managed gateway, validate the selected file without +changing it: + +```shell +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +``` + +Without --path, the command validates a nonempty OPENSHELL_GATEWAY_CONFIG. +Otherwise, it validates an existing XDG gateway config when one is discovered. +When neither source selects a config, preflight succeeds. An explicit missing path, +a legacy schema-v1 file, invalid TOML, a symlink, or any nonregular file fails. +Preflight also merges the selected file with the current `OPENSHELL_*` environment +and applies read-only startup checks for rate-limit pairs, TLS relationships, +interceptor registrations, and supervisor middleware registrations. It validates +complete guest TLS path sets without requiring package-generated certificates to +exist before certificate generation. A failed preflight always preserves the +file; it never migrates, replaces, or rewrites configuration. + +Debian and Ubuntu run this preflight from the systemd user unit before local +certificate generation. The unit still loads the gateway.env environment file +and starts the gateway with no configuration arguments. Snap validates a nonempty +OPENSHELL_GATEWAY_CONFIG first; otherwise it validates and passes its canonical +SNAP_COMMON/gateway.toml only when that path exists in the filesystem. A broken +symlink is therefore rejected instead of being treated as absent. + +Package startup does not modify an operator-owned v1 file. Back it up, follow +[Migrate to schema version 2](#migrate-to-schema-version-2), then validate the +result explicitly before restarting the service: + +```shell +cp ~/.config/openshell/gateway.toml ~/.config/openshell/gateway.toml.v1.bak +$EDITOR ~/.config/openshell/gateway.toml +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +systemctl --user restart openshell-gateway +``` diff --git a/e2e/configs/gateway/schema-v2-intentional-changes.toml b/e2e/configs/gateway/schema-v2-intentional-changes.toml index 67b265f3e8..d0707b8a54 100644 --- a/e2e/configs/gateway/schema-v2-intentional-changes.toml +++ b/e2e/configs/gateway/schema-v2-intentional-changes.toml @@ -163,8 +163,8 @@ validation_capability_ids = ["vm-launch-and-resource-configuration", "vm-guest-s id = "package-default-only-auto-migration" category = "migration_policy" origin_main_contract = "Package-managed and operator-edited schema-v1 configuration may exist at upgrade time." -schema_v2_contract = "RPM and Homebrew automatically replace only recognized byte-identical package defaults; operator-edited files require explicit migration." -migration = "Automatically migrate recognized defaults and preserve every edited file for the operator to convert using the schema-v2 guide." +schema_v2_contract = "RPM and Homebrew automatically replace only recognized byte-identical package defaults; Debian and Snap preserve every legacy file and fail closed through read-only package preflight with explicit manual-migration guidance because their historical generated defaults have no safe provenance marker." +migration = "Automatically migrate recognized defaults, preserve every edited file, and require manual schema-v2 conversion when a package cannot prove that a legacy file is an untouched default." rationale = "An upgrade must not overwrite operator intent merely because the old schema no longer parses." parity_disposition = "intentional_change" validation_capability_ids = ["rpm-schema-upgrade", "homebrew-debian-and-snap-upgrades"] diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml index a1e37c7bdf..3860ac77f1 100644 --- a/e2e/configs/gateway/schema-v2-live-results.toml +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -310,3 +310,46 @@ status = "platform_blocked" owner = "OpenShell credential driver E2E lane" lane = "kubernetes-vault-uds-credential-drivers-paired" blocker = "The host has no assigned disposable Kubernetes Secrets or Vault backend and no retained paired UDS credential-driver execution. Existing backend E2E coverage is candidate-oriented. The assigned lane must compare in-tree and remote transport validation plus opaque store and retrieve behavior using isolated namespaces, Vault state, sockets, databases, and credentials." + +# Step 12 resolves deterministic package-startup blockers without claiming that +# source-tree tests substitute for installation, upgrade, or refresh evidence. + +[[result]] +id = "rpm-package-upgrade" +step = 12 +capability = "RPM schema-v1 default migration and operator-edited configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell RPM package upgrade CI lane" +lane = "fedora-rpm-prior-release-upgrade" +blocker = "Deterministic migration tests prove exact-default replacement, edited-file preservation, mode preservation, idempotence, and unsafe-path rejection, but no prior RPM was installed and upgraded on this host. The assigned lane must verify package ownership and permissions, service restart, exact and edited schema-v1 files, and rollback-safe failure behavior." + +[[result]] +id = "debian-package-upgrade" +step = 12 +capability = "Debian prior-artifact upgrade with schema-v1 configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell Debian package upgrade CI lane" +lane = "ubuntu-debian-prior-release-upgrade" +blocker = "The source-tree tests prove Debian preflight selection, source-free diagnostics, non-mutation, and ordering before certificate generation, but dpkg-deb and an installed prior Debian artifact are unavailable. The assigned lane must preserve generated and edited schema-v1 files, exercise manual conversion, and prove successful schema-v2 service restart after upgrade." + +[[result]] +id = "snap-package-refresh" +step = 12 +capability = "Snap prior-release refresh with schema-v1 configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell Snap refresh CI lane" +lane = "ubuntu-snap-prior-release-refresh" +blocker = "The source-tree tests prove Snap config precedence, preflight failure handling, and non-mutation, but this host cannot install and refresh confined Snap revisions. The assigned lane must refresh a prior Snap with generated and edited schema-v1 files, exercise manual conversion, and prove successful schema-v2 daemon startup." + +[[result]] +id = "homebrew-package-upgrade" +step = 12 +capability = "Homebrew prior-release upgrade with package-default and operator configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell macOS Homebrew package upgrade CI lane" +lane = "macos-homebrew-prior-release-upgrade" +blocker = "No Homebrew formula installation or prior-release upgrade ran on this Linux host. The assigned macOS lane must upgrade a prior formula with recognized package defaults and edited schema-v1 configuration, verify only provable defaults migrate automatically, and prove successful schema-v2 service restart." diff --git a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml index d0a3a65f3c..473e092433 100644 --- a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml +++ b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml @@ -23,14 +23,14 @@ owner_step = 6 [[gaps]] id = "debian-snap-v1-upgrade" -severity = "blocker" +severity = "none" parity_relation = "upgrade_regression" -disposition = "must_fix_before_release_gate" +disposition = "resolved" origin_main_behavior = "Debian auto-discovers a persistent schema-v1 XDG gateway.toml and Snap passes a persistent schema-v1 SNAP_COMMON gateway.toml." -candidate_behavior = "The schema-v2 gateway rejects those files, and neither package currently performs a package-specific migration or preflight." -impact = "Upgrading a working installation can leave its gateway service unable to start." -resolution = "Add package-specific preflight behavior that never overwrites edited configuration, recognizes any package-generated v1 default that is safe to migrate, and reports an actionable manual migration diagnostic for every preserved v1 file." -validation = "Upgrade real prior Debian and Snap artifacts with generated and edited v1 files; verify safe migration or deterministic operator guidance, file preservation, and successful restart after conversion." +candidate_behavior = "Debian and Snap run the gateway's source-free, read-only preflight against the same effectively selected config before certificate generation or daemon startup; rejected legacy files, schema/layout errors, and documented cross-field startup errors remain unchanged with actionable migration guidance. Selected driver-specific fields remain lazily validated at driver construction." +impact = "The package startup regression is resolved without inferring provenance or overwriting operator-owned configuration; release remains gated on real Debian upgrade and Snap refresh execution." +resolution = "Use the shared strict schema-v2 layout parser plus documented read-only effective startup checks, fail closed for explicit, unsafe, or legacy paths, preserve optional absence, and require manual migration because historical Debian and Snap generators have no safe default-provenance marker." +validation = "Deterministic Rust, shell, and Python tests cover selection precedence, stable diagnostics, content non-disclosure, semantic checks, file preservation, and package ordering. Real prior Debian upgrade and Snap refresh lanes remain platform-blocked and must verify preserved v1 files plus successful restart after manual conversion." owner_step = 12 [[gaps]] diff --git a/python/openshell/gateway_schema_v2_intentional_changes_test.py b/python/openshell/gateway_schema_v2_intentional_changes_test.py index acf1bff3d6..84e8f6968b 100644 --- a/python/openshell/gateway_schema_v2_intentional_changes_test.py +++ b/python/openshell/gateway_schema_v2_intentional_changes_test.py @@ -148,5 +148,11 @@ def test_operator_edited_package_configuration_is_never_auto_rewritten() -> None if change["id"] == "package-default-only-auto-migration" ) - assert "byte-identical package defaults" in package_policy["schema_v2_contract"] + contract = package_policy["schema_v2_contract"] + assert "byte-identical package defaults" in contract + assert "Debian and Snap preserve every legacy file" in contract + assert "fail closed through read-only package preflight" in contract + assert "manual-migration guidance" in contract + assert "no safe provenance marker" in contract assert "preserve every edited file" in package_policy["migration"] + assert "manual schema-v2 conversion" in package_policy["migration"] diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index d2c0f252e2..3cb667fa99 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -67,6 +67,12 @@ "supervisor-middleware-registration", "unsafe-unauthenticated-user-mode", } +REQUIRED_STEP_12_IDS = { + "debian-package-upgrade", + "homebrew-package-upgrade", + "rpm-package-upgrade", + "snap-package-refresh", +} STEP_10_CANDIDATE_COMMIT = "4a39da510e4d278a24dd60291149519c9a570b46" STEP_10_REPORT_SHA256 = ( "65541eec5f642461a88b04b5459474fd7a475adeb7071a65a53fe183caad6a01" @@ -371,6 +377,40 @@ def test_step_11_records_cross_cutting_live_lane_dispositions() -> None: assert result[field] == disposition[field] +def test_step_12_keeps_real_package_upgrade_lanes_blocked() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 12 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_12_IDS + assert all(result["status"] == "platform_blocked" for result in results) + by_id = {result["id"]: result for result in results} + + rpm = by_id["rpm-package-upgrade"] + assert rpm["owner"] == "OpenShell RPM package upgrade CI lane" + assert rpm["lane"] == "fedora-rpm-prior-release-upgrade" + assert "prior RPM" in rpm["blocker"] + assert "installed and upgraded" in rpm["blocker"] + + debian = by_id["debian-package-upgrade"] + assert debian["owner"] == "OpenShell Debian package upgrade CI lane" + assert debian["lane"] == "ubuntu-debian-prior-release-upgrade" + assert "installed prior Debian artifact" in debian["blocker"] + assert "source-tree tests" in debian["blocker"] + + snap = by_id["snap-package-refresh"] + assert snap["owner"] == "OpenShell Snap refresh CI lane" + assert snap["lane"] == "ubuntu-snap-prior-release-refresh" + assert "refresh confined Snap revisions" in snap["blocker"] + assert "source-tree tests" in snap["blocker"] + + homebrew = by_id["homebrew-package-upgrade"] + assert homebrew["owner"] == "OpenShell macOS Homebrew package upgrade CI lane" + assert homebrew["lane"] == "macos-homebrew-prior-release-upgrade" + assert "prior-release upgrade" in homebrew["blocker"] + assert "Linux host" in homebrew["blocker"] + + def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: for result in load_toml(RESULTS_PATH)["result"]: if result["status"] != "platform_blocked": diff --git a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py index d02c0ab82b..76c8f08bd4 100644 --- a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py +++ b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py @@ -13,6 +13,7 @@ GAP_LEDGER_PATH = ( REPO_ROOT / "e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml" ) +LIVE_RESULTS_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-live-results.toml" REQUIRED_HEADER_FIELDS = { "ledger_version", @@ -147,6 +148,41 @@ def test_step_6_gap_dispositions_are_resolved() -> None: assert all(gap["disposition"] == "resolved" for gap in step_6_gaps) +def test_step_12_product_gaps_are_resolved_without_claiming_live_package_parity() -> ( + None +): + step_12_gaps = [gap for gap in load_ledger()["gaps"] if gap["owner_step"] == 12] + assert {gap["id"] for gap in step_12_gaps} == { + "debian-snap-v1-upgrade", + "rpm-exact-default-migration", + } + + debian_snap = next( + gap for gap in step_12_gaps if gap["id"] == "debian-snap-v1-upgrade" + ) + assert debian_snap["severity"] == "none" + assert debian_snap["disposition"] == "resolved" + assert "source-free, read-only preflight" in debian_snap["candidate_behavior"] + assert "fail closed" in debian_snap["resolution"] + assert "manual migration" in debian_snap["resolution"] + assert "no safe default-provenance marker" in debian_snap["resolution"] + assert "platform-blocked" in debian_snap["validation"] + + with LIVE_RESULTS_PATH.open("rb") as results_file: + package_results = { + result["id"]: result + for result in tomllib.load(results_file)["result"] + if result["step"] == 12 + } + for result_id in ( + "debian-package-upgrade", + "homebrew-package-upgrade", + "rpm-package-upgrade", + "snap-package-refresh", + ): + assert package_results[result_id]["status"] == "platform_blocked" + + def test_legacy_environment_resolution_preserves_singular_semantics() -> None: legacy_gap = next( gap diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index aa7b56e432..b31a686980 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -214,3 +214,38 @@ def test_rpm_migration_exec_start_pre_argument_order() -> None: "/usr/share/openshell-gateway/gateway.toml.default " "/usr/share/openshell-gateway/gateway.toml.default.v1" ) in spec + + +def test_schema_v2_debian_and_snap_preflight_wiring() -> None: + repo_root = Path(__file__).resolve().parents[2] + unit = (repo_root / "deploy/deb/openshell-gateway.service").read_text( + encoding="utf-8" + ) + wrapper = (repo_root / "tasks/scripts/snap-gateway-wrapper.sh").read_text( + encoding="utf-8" + ) + package_deb = (repo_root / "tasks/scripts/package-deb.sh").read_text( + encoding="utf-8" + ) + preflight = "ExecStartPre=/usr/bin/openshell-gateway config preflight" + certs = "ExecStartPre=/usr/bin/openshell-gateway generate-certs" + assert preflight in unit + assert unit.index(preflight) < unit.index(certs) + assert "EnvironmentFile=-%E/openshell/gateway.env" in unit + assert "ExecStart=/usr/bin/openshell-gateway" in unit + assert "$src_dir/openshell-gateway.service" in package_deb + assert "$pkgroot/usr/lib/systemd/user/openshell-gateway.service" in package_deb + assert 'if [ -n "${OPENSHELL_GATEWAY_CONFIG:-}" ]; then' in wrapper + assert ( + 'elif [ -e "$CANONICAL_CONFIG_FILE" ] || [ -L "$CANONICAL_CONFIG_FILE" ]; then' + in wrapper + ) + assert wrapper.count('"${SNAP}/bin/openshell-gateway" config preflight') == 4 + assert 'config preflight "--path=$cli_config"' in wrapper + assert 'config preflight --path "$CANONICAL_CONFIG_FILE"' in wrapper + assert ( + 'exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@"' + in wrapper + ) + assert wrapper.count('exec "${SNAP}/bin/openshell-gateway" "$@"') == 3 + assert '[ -f "$CANONICAL_CONFIG_FILE" ]' not in wrapper diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index e7918e6ab0..2961896d62 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -741,3 +741,19 @@ When handing results back to the user, include: - Service exposure status. - Sandbox workload status. - The exact command that failed and the shortest fix. + +## Package Configuration Preflight + +For a Debian, Ubuntu, or Snap gateway that stops before certificate generation or +daemon startup, validate the selected configuration without starting the service: + + openshell-gateway config preflight [--path PATH] + +Without a path, preflight validates a nonempty OPENSHELL_GATEWAY_CONFIG or an +auto-discovered XDG config; no config succeeds. An explicit missing path, legacy +schema-v1 file, malformed TOML, symlink, or nonregular file fails before the +gateway ExecStart. It also applies read-only effective-config checks for rate +limits, TLS, interceptors, and supervisor middleware. Preflight preserves every +failed file. Do not advise users to +delete or rewrite it automatically; back it up and follow the manual schema-v2 +migration in the Gateway Configuration reference. diff --git a/snapcraft.yaml b/snapcraft.yaml index f567c63c47..35ec3b1d0c 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -87,9 +87,9 @@ apps: # Operators must manually restart the service after a refresh if needed. refresh-mode: endure # The wrapper sets OPENSHELL_DISABLE_TLS=true and OPENSHELL_DB_URL to - # use $SNAP_COMMON/gateway.db. If $SNAP_COMMON/gateway.toml exists it is - # passed to the gateway as --config, allowing operators to override - # settings without rebuilding the snap. + # use $SNAP_COMMON/gateway.db. Before startup it validates the selected + # operator-provided config without creating or rewriting it. A nonempty + # OPENSHELL_GATEWAY_CONFIG takes precedence over gateway.toml. environment: XDG_DATA_HOME: "$SNAP_COMMON" XDG_RUNTIME_DIR: "$SNAP_COMMON" diff --git a/tasks/scripts/snap-gateway-wrapper.sh b/tasks/scripts/snap-gateway-wrapper.sh index 14d2054f79..ffc03966ab 100755 --- a/tasks/scripts/snap-gateway-wrapper.sh +++ b/tasks/scripts/snap-gateway-wrapper.sh @@ -5,8 +5,8 @@ # Snap wrapper for openshell-gateway. Sets snap-specific defaults: # - OPENSHELL_DB_URL -> sqlite:$SNAP_COMMON/gateway.db (overridable) # - OPENSHELL_DISABLE_TLS -> true -# If $SNAP_COMMON/gateway.toml exists, passes it as --config so operators -# can override settings without rebuilding the snap. +# It validates, but never creates or rewrites, an operator-provided config +# before starting the gateway. set -eu @@ -14,8 +14,65 @@ CANONICAL_CONFIG_FILE="${SNAP_COMMON}/gateway.toml" export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${SNAP_COMMON}/gateway.db?mode=rwc}" export OPENSHELL_DISABLE_TLS="${OPENSHELL_DISABLE_TLS:-true}" -if [ -z "${OPENSHELL_GATEWAY_CONFIG:-}" ] && [ -f "$CANONICAL_CONFIG_FILE" ]; then - exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@" +# Mirror clap's CLI-over-environment precedence so preflight always inspects +# the same file the daemon will load. Reject ambiguous duplicate selectors +# before either command runs. +cli_config="" +config_seen=false +expect_config_path=false +options_done=false +for argument in "$@"; do + if [ "$options_done" = true ]; then + continue + fi + if [ "$expect_config_path" = true ]; then + case "$argument" in + -*) + echo "openshell-gateway: --config requires a nonempty path" >&2 + exit 2 + ;; + esac + if [ "$config_seen" = true ]; then + echo "openshell-gateway: duplicate --config option" >&2 + exit 2 + fi + cli_config=$argument + config_seen=true + expect_config_path=false + continue + fi + case "$argument" in + --) + options_done=true + ;; + --config) + expect_config_path=true + ;; + --config=*) + if [ "$config_seen" = true ]; then + echo "openshell-gateway: duplicate --config option" >&2 + exit 2 + fi + cli_config=${argument#--config=} + config_seen=true + ;; + esac +done +if [ "$expect_config_path" = true ] || { [ "$config_seen" = true ] && [ -z "$cli_config" ]; }; then + echo "openshell-gateway: --config requires a nonempty path" >&2 + exit 2 fi -exec "${SNAP}/bin/openshell-gateway" "$@" +if [ "$config_seen" = true ]; then + "${SNAP}/bin/openshell-gateway" config preflight "--path=$cli_config" + exec "${SNAP}/bin/openshell-gateway" "$@" +elif [ -n "${OPENSHELL_GATEWAY_CONFIG:-}" ]; then + "${SNAP}/bin/openshell-gateway" config preflight + exec "${SNAP}/bin/openshell-gateway" "$@" +elif [ -e "$CANONICAL_CONFIG_FILE" ] || [ -L "$CANONICAL_CONFIG_FILE" ]; then + "${SNAP}/bin/openshell-gateway" config preflight --path "$CANONICAL_CONFIG_FILE" + exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@" +else + "${SNAP}/bin/openshell-gateway" config preflight + exec "${SNAP}/bin/openshell-gateway" "$@" +fi diff --git a/tasks/scripts/test-packaging-assets.sh b/tasks/scripts/test-packaging-assets.sh index a03f60559a..48c58687dc 100755 --- a/tasks/scripts/test-packaging-assets.sh +++ b/tasks/scripts/test-packaging-assets.sh @@ -59,4 +59,53 @@ assert_contains \ 'ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal' assert_not_contains "$spec" '%%S/openshell/tls' +# Schema-v2 package startup wiring. +snap_wrapper="${ROOT}/tasks/scripts/snap-gateway-wrapper.sh" +package_deb="${ROOT}/tasks/scripts/package-deb.sh" +assert_file_exists "$snap_wrapper" +assert_file_exists "$package_deb" +assert_contains "$service" "ExecStartPre=/usr/bin/openshell-gateway config preflight" +assert_contains "$package_deb" "\$src_dir/openshell-gateway.service" +assert_contains "$package_deb" "\$pkgroot/usr/lib/systemd/user/openshell-gateway.service" +assert_contains "$snap_wrapper" "if [ -n \"\${OPENSHELL_GATEWAY_CONFIG:-}\" ]; then" +assert_contains \ + "$snap_wrapper" \ + "elif [ -e \"\$CANONICAL_CONFIG_FILE\" ] || [ -L \"\$CANONICAL_CONFIG_FILE\" ]; then" +assert_contains "$snap_wrapper" "config preflight --path \"\$CANONICAL_CONFIG_FILE\"" +assert_not_contains "$snap_wrapper" "[ -f \"\$CANONICAL_CONFIG_FILE\" ]" +bash "$ROOT/tasks/scripts/test-snap-gateway-wrapper.sh" "$snap_wrapper" +if ! awk '/config preflight/ { seen = 1 } /generate-certs/ { exit !seen }' "$service"; then + echo "FAIL: Debian preflight must precede certificate generation" >&2 + exit 1 +fi + +# Build a throwaway package when Debian tooling is available to prove the +# staged unit comes from deploy/deb/. Other hosts retain the static source-to- +# destination assertion above; the real Debian upgrade lane remains required. +if command -v dpkg-deb >/dev/null 2>&1; then + package_work=$(mktemp -d "${TMPDIR:-/tmp}/openshell-package-assets.XXXXXX") + trap 'rm -rf "$package_work"' EXIT + mkdir -p "$package_work/bin" "$package_work/output" + for binary in openshell openshell-gateway openshell-driver-vm; do + printf '#!/bin/sh\nexit 0\n' >"$package_work/bin/$binary" + chmod +x "$package_work/bin/$binary" + done + OPENSHELL_CLI_BINARY="$package_work/bin/openshell" \ + OPENSHELL_GATEWAY_BINARY="$package_work/bin/openshell-gateway" \ + OPENSHELL_DRIVER_VM_BINARY="$package_work/bin/openshell-driver-vm" \ + OPENSHELL_DEB_VERSION=0.0.0 \ + OPENSHELL_DEB_ARCH=amd64 \ + OPENSHELL_OUTPUT_DIR="$package_work/output" \ + "$package_deb" >/dev/null + dpkg-deb --fsys-tarfile "$package_work/output/openshell_0.0.0_amd64.deb" \ + | tar -xOf - ./usr/lib/systemd/user/openshell-gateway.service \ + >"$package_work/staged.service" + if ! cmp -s "$service" "$package_work/staged.service"; then + echo "FAIL: package-deb did not stage the current Debian service" >&2 + exit 1 + fi +else + echo "SKIP: dpkg-deb unavailable; Debian artifact staging requires its assigned lane" +fi + echo "packaging asset tests passed" diff --git a/tasks/scripts/test-snap-gateway-wrapper.sh b/tasks/scripts/test-snap-gateway-wrapper.sh new file mode 100755 index 0000000000..e6286b00a4 --- /dev/null +++ b/tasks/scripts/test-snap-gateway-wrapper.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +wrapper_input=${1:?Usage: test-snap-gateway-wrapper.sh } +wrapper_dir=$(cd "$(dirname "$wrapper_input")" && pwd) +wrapper="${wrapper_dir}/$(basename "$wrapper_input")" +work=$(mktemp -d "${TMPDIR:-/tmp}/openshell snap wrapper.XXXXXX") +trap 'rm -rf "$work"' EXIT + +snap="$work/snap" +common="$work/common" +log="$work/calls" +expected="$work/expected" +mkdir -p "$snap/bin" "$common" + +cat >"$snap/bin/openshell-gateway" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >>"$FAKE_GATEWAY_LOG" +printf 'env:%s|%s|%s\n' \ + "${OPENSHELL_GATEWAY_CONFIG:-}" \ + "${OPENSHELL_DB_URL:-}" \ + "${OPENSHELL_DISABLE_TLS:-}" >>"$FAKE_GATEWAY_LOG" +if [ "${1:-}" = config ] && [ "${2:-}" = preflight ] && [ "${FAKE_PREFLIGHT_FAIL:-}" = 1 ]; then + exit 42 +fi +EOF +chmod +x "$snap/bin/openshell-gateway" + +run_wrapper() { + local config=$1 + local fail=${2:-} + if [ "$config" = unset ]; then + env -u OPENSHELL_GATEWAY_CONFIG \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL="$fail" \ + "$wrapper" --trace + else + env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$config" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL="$fail" \ + "$wrapper" --trace + fi +} + +assert_log() { + printf '%s\n' "$1" >"$expected" + if ! cmp -s "$expected" "$log"; then + echo "FAIL: unexpected call sequence" >&2 + diff -u "$expected" "$log" >&2 + exit 1 + fi +} + +override="$work/override.toml" +printf 'operator override\n' >"$override" +cp "$override" "$work/override-before" +: >"$log" +run_wrapper "$override" +assert_log "config preflight +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--trace +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/override-before" "$override" + +cli_config="$work/cli.toml" +printf 'CLI override\n' >"$cli_config" +cp "$cli_config" "$work/cli-before" +: >"$log" +env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" --trace --config "$cli_config" +assert_log "config preflight --path=$cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--trace --config $cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/cli-before" "$cli_config" + +: >"$log" +if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL=1 \ + "$wrapper" --config="$cli_config"; then + echo "FAIL: CLI-selected config preflight failure reached gateway start" >&2 + exit 1 +fi +assert_log "config preflight --path=$cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/cli-before" "$cli_config" + +for invalid_selector in terminator nested-config; do + : >"$log" + if [ "$invalid_selector" = terminator ]; then + invalid_args=(--config --) + else + invalid_args=(--config "--config=$cli_config") + fi + if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" "${invalid_args[@]}"; then + echo "FAIL: invalid $invalid_selector selector reached gateway execution" >&2 + exit 1 + fi + if [ -s "$log" ]; then + echo "FAIL: invalid $invalid_selector selector reached preflight" >&2 + exit 1 + fi +done + +: >"$log" +env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" --config=--dash-leading +assert_log "config preflight --path=--dash-leading +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--config=--dash-leading +env:$override|sqlite:$common/gateway.db?mode=rwc|true" + +canonical="$common/gateway.toml" +printf 'valid schema-v2\n' >"$canonical" +cp "$canonical" "$work/canonical-before" +: >"$log" +run_wrapper unset +assert_log "config preflight --path $canonical +env:|sqlite:$common/gateway.db?mode=rwc|true +--config $canonical --trace +env:|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/canonical-before" "$canonical" + +rm "$canonical" +: >"$log" +run_wrapper unset +assert_log "config preflight +env:|sqlite:$common/gateway.db?mode=rwc|true +--trace +env:|sqlite:$common/gateway.db?mode=rwc|true" + +assert_preflight_failure() { + local name=$1 + : >"$log" + if run_wrapper unset 1; then + echo "FAIL: $name reached gateway start" >&2 + exit 1 + fi + assert_log "config preflight --path $canonical +env:|sqlite:$common/gateway.db?mode=rwc|true" +} + +printf 'legacy version = 1\n' >"$canonical" +cp "$canonical" "$work/legacy-before" +assert_preflight_failure legacy +cmp -s "$work/legacy-before" "$canonical" + +printf 'not valid TOML = [\n' >"$canonical" +cp "$canonical" "$work/malformed-before" +assert_preflight_failure malformed +cmp -s "$work/malformed-before" "$canonical" + +rm "$canonical" +ln -s "$work/missing-target" "$canonical" +readlink "$canonical" >"$work/link-before" +assert_preflight_failure broken-symlink +readlink "$canonical" >"$work/link-after" +cmp -s "$work/link-before" "$work/link-after" + +rm "$canonical" +mkdir "$canonical" +printf 'nonregular marker\n' >"$canonical/marker" +cp "$canonical/marker" "$work/marker-before" +assert_preflight_failure nonregular +cmp -s "$work/marker-before" "$canonical/marker" + +echo "Snap gateway wrapper tests passed" From 5486551a265af43dc09dc6888d97cff8ec46b0c7 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 15:18:04 -0400 Subject: [PATCH 38/42] fix(config): preserve rebase integration guarantees Signed-off-by: Jesse Jaggars --- architecture/compute-runtimes.md | 6 +- crates/openshell-core/src/container_paths.rs | 8 + crates/openshell-driver-docker/src/lib.rs | 10 +- crates/openshell-driver-vm/README.md | 1 + crates/openshell-driver-vm/src/driver.rs | 139 ++++++++++++------ crates/openshell-driver-vm/src/main.rs | 88 ++++------- crates/openshell-gateway/src/vm.rs | 48 +++++- .../src/compute/driver_config.rs | 9 +- crates/openshell-server/src/config_file.rs | 21 +-- docs/reference/gateway-config.mdx | 16 +- .../gateway_schema_v2_live_results_test.py | 35 +++-- 11 files changed, 237 insertions(+), 144 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 0df620df2b..2b573f2413 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -300,7 +300,11 @@ explicitly. The Helm chart independently uses `Unconfined` for Kubernetes. Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, and the explicit cleartext-Basic-auth acknowledgement without allowing -workload environment to override them. Local containers project provider SPIFFE +workload environment to override them. Podman and VM can also project an +operator CA bundle for an HTTPS or TLS-intercepting proxy. The VM driver validates and +stages its credential and CA bundle under fixed guest paths, then forwards +those paths through the protected supervisor argument file rather than the +guest environment. Local containers project provider SPIFFE through a dedicated host UNIX-socket parent mount. A VM cannot safely expose that host socket: it accepts only a separately operated, concrete TCP listener when `provider_spiffe_allow_guest_tcp = true` explicitly acknowledges guest diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index 9cf3022014..63511c13ff 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -74,6 +74,13 @@ pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest" /// secrets, so this is the same delivery the per-sandbox JWT already uses. pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; +/// Guest path for the corporate proxy CA bundle staged by the VM driver. +/// +/// The bundle is operator-owned but not secret. The driver validates it and +/// writes it into each sandbox overlay with mode `0644`, then passes only this +/// guest path to the supervisor. +pub const VM_GUEST_PROXY_CA_PATH: &str = "/opt/openshell/tls/proxy-ca.pem"; + /// Guest path for the driver-authored supervisor argument list in VM sandboxes. /// /// Podman and Kubernetes build the supervisor's command line directly; the VM @@ -120,6 +127,7 @@ mod tests { VM_GUEST_TLS_KEY_PATH, VM_GUEST_SANDBOX_TOKEN_PATH, VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, + VM_GUEST_PROXY_CA_PATH, VM_GUEST_INIT_DROPIN_DIR, VM_GUEST_INIT_DROPIN_MANIFEST, VM_GUEST_SUPERVISOR_ARGS_PATH, diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 5c8c02f73d..91138f6e6c 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -566,9 +566,7 @@ impl DockerComputeDriver { docker_config.grpc_endpoint = gateway_callback_endpoint( GatewayCallbackTopology::Docker, gateway_port, - docker_config.guest_tls_ca.is_some() - || docker_config.guest_tls_cert.is_some() - || docker_config.guest_tls_key.is_some(), + docker_guest_tls_configured(&docker_config), ); } let grpc_endpoint = docker_container_openshell_endpoint( @@ -4108,6 +4106,12 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { + docker_config.guest_tls_ca.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some() +} + pub(crate) fn docker_guest_tls_paths( docker_config: &DockerComputeConfig, ) -> CoreResult> { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 7a57893692..a83a1df103 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -158,6 +158,7 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`, | `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | | `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | | `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP CONNECT targets. | +| `proxy_ca_bundle` | unset | Gateway-host PEM CA bundle trusted for the corporate proxy and TLS-intercepted server certificates. The driver validates it and stages it at a fixed non-secret guest path in the protected overlay. Requires `https_proxy`. | | `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor through a protected per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 677e44d6c3..d8a29dce9d 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -172,6 +172,8 @@ const GUEST_INIT_DROPIN_MANIFEST: &str = /// Guest path of the root-only corporate proxy credential staged by the driver. const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; +/// Guest path of the corporate proxy CA bundle staged by the driver. +const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROXY_CA_PATH; /// Guest path of the driver-authored supervisor argument list. /// /// The counterpart of [`GUEST_INIT_DROPIN_MANIFEST`] for the supervisor's own @@ -258,6 +260,9 @@ pub struct VmDriverConfig { /// Corporate forward proxy settings delivered to the guest init script. #[serde(flatten)] pub upstream_proxy: UpstreamProxyConfig, + /// Gateway-host PEM CA bundle staged into the guest overlay for the + /// corporate proxy and TLS-intercepted server certificates. + pub proxy_ca_bundle: Option, /// Guest-reachable SPIFFE Workload API TCP endpoint. A VM cannot safely /// project a host UNIX socket; this must be a deliberately exposed TCP /// listener and requires `provider_spiffe_allow_guest_tcp`. @@ -322,6 +327,10 @@ impl std::fmt::Debug for VmDriverConfig { "proxy_connect_by_hostname", &self.upstream_proxy.proxy_connect_by_hostname, ) + .field( + "proxy_ca_bundle_configured", + &self.proxy_ca_bundle.is_some(), + ) .field( "provider_spiffe_workload_api_tcp_endpoint_configured", &self.provider_spiffe_workload_api_tcp_endpoint.is_some(), @@ -355,6 +364,7 @@ impl Default for VmDriverConfig { guest_tls_cert: None, guest_tls_key: None, upstream_proxy: UpstreamProxyConfig::default(), + proxy_ca_bundle: None, provider_spiffe_workload_api_tcp_endpoint: None, provider_spiffe_allow_guest_tcp: false, gpu_enabled: false, @@ -379,6 +389,14 @@ impl VmDriverConfig { pub fn validate_runtime_security_config(&self) -> Result<(), String> { self.upstream_proxy.validate()?; + if let Some(path) = self.proxy_ca_bundle.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_ca_bundle must not be empty when set".to_string()); + } + if self.upstream_proxy.https_proxy.is_none() { + return Err("proxy_ca_bundle is set but no https_proxy is configured".to_string()); + } + } if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( endpoint, @@ -4940,33 +4958,6 @@ fn build_guest_environment( GUEST_TLS_KEY_PATH.to_string(), ); } - if let Some(url) = config.upstream_proxy.https_proxy.as_ref() { - environment.insert("OPENSHELL_VM_UPSTREAM_PROXY".to_string(), url.clone()); - } - if let Some(no_proxy) = config.upstream_proxy.no_proxy.as_ref() { - environment.insert( - "OPENSHELL_VM_UPSTREAM_NO_PROXY".to_string(), - no_proxy.clone(), - ); - } - if config.upstream_proxy.proxy_auth_file.is_some() { - environment.insert( - "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE".to_string(), - GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string(), - ); - } - if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { - environment.insert( - "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE".to_string(), - "true".to_string(), - ); - } - if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { - environment.insert( - "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME".to_string(), - "true".to_string(), - ); - } if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { environment.insert( openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), @@ -5824,6 +5815,11 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { args.push("--upstream-proxy-connect-by-hostname".to_string()); } + if config.proxy_ca_bundle.is_some() { + args.push("--upstream-proxy-ca-bundle".to_string()); + // The guest path, never the gateway-host path the operator configured. + args.push(GUEST_PROXY_CA_PATH.to_string()); + } args } @@ -5866,7 +5862,10 @@ async fn read_sandbox_proxy_credential(path: &Path) -> Result { let path_owned = path.to_path_buf(); let display_path = path.display().to_string(); let raw = tokio::task::spawn_blocking(move || { - openshell_core::driver_utils::read_upstream_proxy_credential_file(&path_owned) + let path = path_owned + .to_str() + .ok_or_else(|| "proxy_auth_file path is not valid UTF-8".to_string())?; + openshell_core::driver_utils::read_upstream_proxy_credential_file(path) }) .await .map_err(|err| Status::internal(format!("proxy_auth_file read task failed: {err}")))? @@ -5878,11 +5877,37 @@ async fn read_sandbox_proxy_credential(path: &Path) -> Result { Ok(credential.to_string()) } +/// Read and validate the corporate proxy CA bundle from the gateway host. +/// +/// The validation rejects symlinks and non-regular files, bounds the read, +/// requires PEM certificate markers, and never includes file contents in an +/// error. +async fn read_sandbox_proxy_ca_bundle(path: &Path) -> Result, Status> { + let path_owned = path.to_path_buf(); + let display_path = path.display().to_string(); + tokio::task::spawn_blocking(move || { + let path = path_owned + .to_str() + .ok_or_else(|| "proxy_ca_bundle path is not valid UTF-8".to_string())?; + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(path, "proxy_ca_bundle") + .map(String::into_bytes) + }) + .await + .map_err(|err| Status::internal(format!("proxy_ca_bundle read task failed: {err}")))? + .map_err(|err| { + Status::invalid_argument(format!( + "proxy_ca_bundle '{display_path}' could not be read: {err}" + )) + }) +} + /// Stage the corporate upstream-proxy configuration into the guest overlay. /// -/// Writes two files into the overlay upperdir the driver owns: +/// Writes three files into the overlay upperdir the driver owns: /// /// * the credential at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], mode `0600`; +/// * the CA bundle at [`GUEST_PROXY_CA_PATH`], mode `0644` (a CA certificate +/// is not secret); /// * the supervisor argument list at [`GUEST_SUPERVISOR_ARGS_PATH`], mode /// `0644`. /// @@ -5917,6 +5942,16 @@ async fn inject_guest_upstream_proxy( set_rootfs_image_file_mode(overlay_disk, &credential_path, 0o600) .map_err(|err| Status::internal(format!("set VM guest proxy credential mode: {err}")))?; + let ca_bundle = match config.proxy_ca_bundle.as_deref() { + Some(path) => read_sandbox_proxy_ca_bundle(path).await?, + None => Vec::new(), + }; + let ca_path = overlay_upper_path(GUEST_PROXY_CA_PATH); + write_rootfs_image_file(overlay_disk, &ca_path, &ca_bundle) + .map_err(|err| Status::internal(format!("write VM guest proxy CA bundle: {err}")))?; + set_rootfs_image_file_mode(overlay_disk, &ca_path, 0o644) + .map_err(|err| Status::internal(format!("set VM guest proxy CA bundle mode: {err}")))?; + let args = upstream_proxy_cli_args(config); validate_guest_supervisor_args(&args).map_err(Status::failed_precondition)?; let guest_path = overlay_upper_path(GUEST_SUPERVISOR_ARGS_PATH); @@ -8804,7 +8839,7 @@ mod tests { } #[test] - fn build_guest_environment_projects_operator_proxy_and_spiffe_endpoint() { + fn build_guest_environment_projects_spiffe_endpoint_without_operator_proxy() { let config = VmDriverConfig { upstream_proxy: UpstreamProxyConfig { https_proxy: Some("https://proxy.example:8443".to_string()), @@ -8823,18 +8858,13 @@ mod tests { ..Default::default() }; let env = build_guest_environment(&sandbox, &config, None); - assert!( - env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY=https://proxy.example:8443".to_string()) - ); - assert!(env.contains(&"OPENSHELL_VM_UPSTREAM_NO_PROXY=.svc".to_string())); - assert!(env.contains(&"OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME=true".to_string())); - assert!(env.contains(&format!( - "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE={GUEST_UPSTREAM_PROXY_AUTH_PATH}" - ))); assert!(env.contains( &"OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=tcp:192.0.2.10:8081".to_string() )); - assert!(!env.iter().any(|value| value.contains("user:pass"))); + assert!( + !env.iter().any(|value| value.contains("UPSTREAM_PROXY")), + "operator proxy settings must use protected guest argument staging: {env:?}" + ); } #[test] @@ -9837,7 +9867,11 @@ mod tests { // credential removable with the sandbox (remove_sandbox_state_dir // deletes the whole directory) and unforgeable by the guest image // (the upperdir shadows the read-only image layer). - for guest_path in [GUEST_UPSTREAM_PROXY_AUTH_PATH, GUEST_SUPERVISOR_ARGS_PATH] { + for guest_path in [ + GUEST_UPSTREAM_PROXY_AUTH_PATH, + GUEST_PROXY_CA_PATH, + GUEST_SUPERVISOR_ARGS_PATH, + ] { assert!( guest_path.starts_with("/opt/openshell/"), "{guest_path} must be under the reserved guest control root" @@ -9860,19 +9894,25 @@ mod tests { #[test] fn upstream_proxy_args_pass_guest_paths_not_host_paths() { - let config = proxy_config( + let mut config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), ); + config.proxy_ca_bundle = Some(PathBuf::from("/etc/openshell/tls/corp-ca.pem")); let args = upstream_proxy_cli_args(&config); - // The credential lives at a fixed guest path; the gateway-host path - // the operator configured must never reach the guest argv. + // The credential and CA live at fixed guest paths; the gateway-host + // paths the operator configured must never reach the guest argv. let auth = args .iter() .position(|arg| arg == "--upstream-proxy-auth-file") .map(|i| args[i + 1].as_str()); assert_eq!(auth, Some(GUEST_UPSTREAM_PROXY_AUTH_PATH)); + let ca = args + .iter() + .position(|arg| arg == "--upstream-proxy-ca-bundle") + .map(|i| args[i + 1].as_str()); + assert_eq!(ca, Some(GUEST_PROXY_CA_PATH)); assert!( !args .iter() @@ -9959,6 +9999,15 @@ mod tests { config .validate_runtime_security_config() .expect("a lone proxy URL is a complete configuration"); + + let config = VmDriverConfig { + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/corp-ca.pem")), + ..Default::default() + }; + let err = config + .validate_runtime_security_config() + .expect_err("a CA bundle without a proxy URL must fail closed"); + assert!(err.contains("proxy_ca_bundle"), "{err}"); } #[test] @@ -10079,7 +10128,7 @@ mod tests { std::fs::write(&path, "proxyuser:proxypass\n").unwrap(); assert_eq!( - read_sandbox_proxy_credential(path.to_str().unwrap()) + read_sandbox_proxy_credential(&path) .await .expect("a well-formed credential is accepted"), "proxyuser:proxypass" @@ -10087,7 +10136,7 @@ mod tests { // Rejected here rather than inside every sandbox's supervisor. std::fs::write(&path, "no-separator\n").unwrap(); - let err = read_sandbox_proxy_credential(path.to_str().unwrap()) + let err = read_sandbox_proxy_credential(&path) .await .expect_err("a malformed credential must fail closed"); assert_eq!(err.code(), Code::InvalidArgument); diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index b421780668..4636f76138 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -142,6 +142,11 @@ struct Args { )] upstream_proxy_connect_by_hostname: bool, + /// Gateway-host PEM CA bundle trusted for the corporate proxy and for + /// server certificates re-signed by a TLS-intercepting proxy. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_CA_BUNDLE")] + upstream_proxy_ca_bundle: Option, + /// Guest-reachable SPIFFE Workload API endpoint (`tcp:IP:port`). #[arg( long = "provider-spiffe-workload-api-tcp-endpoint", @@ -184,30 +189,6 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] sandbox_gid: Option, - // Corporate forward proxy for sandbox egress. Operator-owned: these reach - // the guest supervisor on its argv, which the sandbox image and the - // user-supplied environment cannot influence. - #[arg(long, env = "OPENSHELL_VM_HTTPS_PROXY")] - https_proxy: Option, - - #[arg(long, env = "OPENSHELL_VM_NO_PROXY")] - no_proxy: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_FILE")] - proxy_auth_file: Option, - - // Value-taking rather than a presence flag so an explicit `false` in - // `[openshell.drivers.vm]` survives the gateway -> driver hop and still - // trips the "acknowledgement without a credential" check. - #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_ALLOW_INSECURE")] - proxy_auth_allow_insecure: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_CONNECT_BY_HOSTNAME")] - proxy_connect_by_hostname: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_CA_BUNDLE")] - proxy_ca_bundle: Option, - #[arg(long, hide = true)] vm_backend: Option, @@ -298,6 +279,7 @@ async fn main() -> Result<()> { proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname.then_some(true), }, + proxy_ca_bundle: args.upstream_proxy_ca_bundle.clone(), provider_spiffe_workload_api_tcp_endpoint: args .provider_spiffe_workload_api_tcp_endpoint .clone(), @@ -307,12 +289,6 @@ async fn main() -> Result<()> { gpu_vcpus: args.gpu_vcpus, sandbox_uid: args.sandbox_uid, sandbox_gid: args.sandbox_gid, - https_proxy: args.https_proxy.clone(), - no_proxy: args.no_proxy.clone(), - proxy_auth_file: args.proxy_auth_file.clone(), - proxy_auth_allow_insecure: args.proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.proxy_connect_by_hostname, - proxy_ca_bundle: args.proxy_ca_bundle.clone(), }) .await .map_err(|err| miette::miette!("{err}"))?; @@ -688,57 +664,47 @@ mod tests { fn corporate_proxy_flags_parse_into_driver_settings() { let args = Args::parse_from([ "openshell-driver-vm", - "--openshell-endpoint", - "https://host.openshell.internal:17670", - "--https-proxy", + "--upstream-proxy", "http://proxy.corp.com:8080", - "--no-proxy", + "--upstream-no-proxy", "10.0.0.0/8,.svc.cluster.local", - "--proxy-auth-file", + "--upstream-proxy-auth-file", "/etc/openshell/secrets/proxy-auth", - "--proxy-auth-allow-insecure", - "true", - "--proxy-connect-by-hostname", - "false", - "--proxy-ca-bundle", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--upstream-proxy-ca-bundle", "/etc/openshell/tls/proxy-ca.pem", ]); assert_eq!( - args.https_proxy.as_deref(), + args.upstream_proxy.as_deref(), Some("http://proxy.corp.com:8080") ); assert_eq!( - args.no_proxy.as_deref(), + args.upstream_no_proxy.as_deref(), Some("10.0.0.0/8,.svc.cluster.local") ); assert_eq!( - args.proxy_auth_file.as_deref(), - Some("/etc/openshell/secrets/proxy-auth") + args.upstream_proxy_auth_file.as_deref(), + Some(PathBuf::from("/etc/openshell/secrets/proxy-auth").as_path()) ); - assert_eq!(args.proxy_auth_allow_insecure, Some(true)); - // Value-taking rather than a presence flag, so the gateway can - // forward an explicit `false` from `[openshell.drivers.vm]`. - assert_eq!(args.proxy_connect_by_hostname, Some(false)); + assert!(args.upstream_proxy_auth_allow_insecure); + assert!(args.upstream_proxy_connect_by_hostname); assert_eq!( - args.proxy_ca_bundle.as_deref(), - Some("/etc/openshell/tls/proxy-ca.pem") + args.upstream_proxy_ca_bundle.as_deref(), + Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem").as_path()) ); } #[test] fn corporate_proxy_settings_default_to_unset() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--openshell-endpoint", - "https://host.openshell.internal:17670", - ]); - assert!(args.https_proxy.is_none()); - assert!(args.no_proxy.is_none()); - assert!(args.proxy_auth_file.is_none()); - assert!(args.proxy_auth_allow_insecure.is_none()); - assert!(args.proxy_connect_by_hostname.is_none()); - assert!(args.proxy_ca_bundle.is_none()); + let args = Args::parse_from(["openshell-driver-vm"]); + assert!(args.upstream_proxy.is_none()); + assert!(args.upstream_no_proxy.is_none()); + assert!(args.upstream_proxy_auth_file.is_none()); + assert!(!args.upstream_proxy_auth_allow_insecure); + assert!(!args.upstream_proxy_connect_by_hostname); + assert!(args.upstream_proxy_ca_bundle.is_none()); } #[test] diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index c26220e12d..99de85d6c2 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -111,6 +111,10 @@ pub struct VmComputeConfig { #[serde(flatten)] pub upstream_proxy: UpstreamProxyConfig, + /// Path on the gateway host to a PEM CA bundle trusted for the corporate + /// proxy and for server certificates re-signed by a TLS-intercepting proxy. + pub proxy_ca_bundle: Option, + /// Explicit guest-reachable SPIFFE Workload API TCP listener. VM guests /// cannot receive a host UNIX socket, so this requires acknowledgement. pub provider_spiffe_workload_api_tcp_endpoint: Option, @@ -152,6 +156,21 @@ impl VmComputeConfig { 4096 } + fn validate_proxy_config(&self) -> Result<()> { + self.upstream_proxy.validate().map_err(Error::config)?; + if let Some(path) = self.proxy_ca_bundle.as_ref() { + if path.as_os_str().is_empty() { + return Err(Error::config("proxy_ca_bundle must not be empty when set")); + } + if self.upstream_proxy.https_proxy.is_none() { + return Err(Error::config( + "proxy_ca_bundle is set but no https_proxy is configured", + )); + } + } + Ok(()) + } + #[must_use] fn default_driver_search_dirs(home: Option) -> Vec { let mut dirs = Vec::new(); @@ -183,6 +202,7 @@ impl Default for VmComputeConfig { guest_tls_cert: None, guest_tls_key: None, upstream_proxy: UpstreamProxyConfig::default(), + proxy_ca_bundle: None, provider_spiffe_workload_api_tcp_endpoint: None, provider_spiffe_allow_guest_tcp: false, } @@ -483,7 +503,7 @@ pub async fn spawn( } validate_vm_sandbox_identity(vm_config)?; - vm_config.upstream_proxy.validate().map_err(Error::config)?; + vm_config.validate_proxy_config()?; if let Some(endpoint) = vm_config .provider_spiffe_workload_api_tcp_endpoint .as_deref() @@ -600,6 +620,9 @@ fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConf if proxy.proxy_connect_by_hostname == Some(true) { command.arg("--upstream-proxy-connect-by-hostname"); } + if let Some(path) = config.proxy_ca_bundle.as_ref() { + command.arg("--upstream-proxy-ca-bundle").arg(path); + } if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { command .arg("--provider-spiffe-workload-api-tcp-endpoint") @@ -745,13 +768,14 @@ mod tests { append_vm_proxy_and_spiffe_args( &mut command, &VmComputeConfig { - upstream_proxy: openshell_core::UpstreamProxyConfig { + upstream_proxy: UpstreamProxyConfig { https_proxy: Some("http://proxy.corp.com:8080".to_string()), no_proxy: Some("10.0.0.0/8".to_string()), proxy_auth_file: Some(PathBuf::from("/etc/openshell/secrets/proxy-auth")), proxy_auth_allow_insecure: Some(true), proxy_connect_by_hostname: Some(true), }, + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem")), provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), provider_spiffe_allow_guest_tcp: true, ..VmComputeConfig::default() @@ -774,6 +798,8 @@ mod tests { "/etc/openshell/secrets/proxy-auth", "--upstream-proxy-auth-allow-insecure", "--upstream-proxy-connect-by-hostname", + "--upstream-proxy-ca-bundle", + "/etc/openshell/tls/proxy-ca.pem", "--provider-spiffe-workload-api-tcp-endpoint", "tcp:192.0.2.10:8081", "--provider-spiffe-allow-guest-tcp", @@ -790,7 +816,7 @@ mod tests { #[test] fn invalid_corporate_proxy_config_is_rejected_before_the_driver_starts() { - let err = openshell_core::UpstreamProxyConfig { + let err = UpstreamProxyConfig { https_proxy: Some("socks5://proxy.corp.com:1080".to_string()), ..Default::default() } @@ -798,11 +824,21 @@ mod tests { .expect_err("only http:// and https:// proxies are supported"); assert!(err.contains("https_proxy"), "{err}"); - openshell_core::UpstreamProxyConfig { - https_proxy: Some("http://proxy.corp.com:8080".to_string()), + VmComputeConfig { + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem")), ..Default::default() } - .validate() + .validate_proxy_config() + .expect_err("a CA bundle without a proxy URL must fail closed"); + + VmComputeConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + ..Default::default() + }, + ..Default::default() + } + .validate_proxy_config() .expect("a lone proxy URL is a complete configuration"); } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 24254ef8c7..0c892dc373 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -361,9 +361,12 @@ socket_path = "/run/openshell/kyma.sock" ); let file: config_file::ConfigFile = toml::from_str(&source).expect("valid TOML"); - let local_error = - driver_config_from_context::(test_context(Some(&file)), "kyma") - .expect_err("local driver TLS field must be rejected"); + let local_error = driver_config_from_context::( + test_context(Some(&file)), + "kyma", + &[], + ) + .expect_err("local driver TLS field must be rejected"); assert!(local_error.to_string().contains(field)); assert!(local_error.to_string().contains("[openshell.gateway]")); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index e15f130e18..189b4b5b4e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -745,10 +745,6 @@ credential_drivers = ["kubernetes-secrets"] grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 policy_validation_failure_mode = "retain_last_valid" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -client_tls_secret_name = "openshell-sandbox-tls" -service_account_name = "openshell-sandbox" [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" @@ -774,10 +770,6 @@ namespace = "agents" let file = load(tmp.path()).expect("valid file parses"); let gw = &file.openshell.gateway; assert_eq!(gw.log_level.as_deref(), Some("info")); - assert_eq!( - gw.default_image.as_deref(), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); assert_eq!( @@ -791,7 +783,18 @@ namespace = "agents" Some(&["kubernetes-secrets".to_string()][..]) ); assert!(gw.default_credential_driver.is_none()); - assert!(file.openshell.drivers.contains_key("kubernetes")); + let kubernetes = file + .openshell + .drivers + .get("kubernetes") + .and_then(toml::Value::as_table) + .expect("kubernetes driver table"); + assert_eq!( + kubernetes + .get("default_image") + .and_then(toml::Value::as_str), + Some("ghcr.io/nvidia/openshell/sandbox:latest") + ); assert!( file.openshell .credential_drivers diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index b7e56c1cbb..de1433561a 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -928,11 +928,14 @@ overlay_disk_mib = 4096 # address routable from the guest's masqueraded egress. # # Because a microVM has no bind mounts or container secrets, the driver stages -# the credential into the per-sandbox overlay disk, root-only inside the guest, -# and removes it with the sandbox. The credential is therefore at rest in that -# overlay image on the gateway host — the same delivery the per-sandbox gateway -# token already uses, and a difference from the Podman secret model worth noting -# when choosing where to keep proxy credentials. +# the credential and optional CA bundle into the per-sandbox overlay disk and +# removes them with the sandbox. The credential is root-only; the non-secret CA +# bundle is mode 0644. Both are referenced from the protected supervisor +# argument file by fixed guest paths, never their gateway-host paths. The +# credential is therefore at rest in that overlay image on the gateway host — +# the same delivery the per-sandbox gateway token already uses, and a difference +# from the Podman secret model worth noting when choosing where to keep proxy +# credentials. # https_proxy = "http://host.openshell.internal:8080" # no_proxy = "10.0.0.0/8,.internal.example" # proxy_auth_file = "/etc/openshell/secrets/proxy-auth" @@ -940,6 +943,9 @@ overlay_disk_mib = 4096 # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs; see the Podman section above. # proxy_connect_by_hostname = true +# Gateway-host PEM bundle trusted for an https:// proxy and for server +# certificates re-signed by a TLS-intercepting proxy. Requires https_proxy. +# proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" # VM guests cannot mount a host Workload API Unix socket. Configure only a # separately operated guest-reachable TCP listener and explicitly acknowledge # the exposure; host-only sockets are never exposed automatically. diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py index 3cb667fa99..3cb52c5d62 100644 --- a/python/openshell/gateway_schema_v2_live_results_test.py +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -73,7 +73,17 @@ "rpm-package-upgrade", "snap-package-refresh", } -STEP_10_CANDIDATE_COMMIT = "4a39da510e4d278a24dd60291149519c9a570b46" +EXPECTED_EXECUTED_CANDIDATE_COMMITS = { + "portable-lifecycle-podman": "a3860084d019ed2ac979e3eaa1ddf085a96b773c", + "gateway-wide-process-options": "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea", + "gateway-tls-client-auth-policy": "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea", + "podman-driver-option-parity": "e09070d4ba0b30f0ac278fbb9938c1520ecff696", + "kubernetes-core-option-parity": "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd", + "compute-driver-boundary-parity": "4a39da510e4d278a24dd60291149519c9a570b46", +} +STEP_10_CANDIDATE_COMMIT = EXPECTED_EXECUTED_CANDIDATE_COMMITS[ + "compute-driver-boundary-parity" +] STEP_10_REPORT_SHA256 = ( "65541eec5f642461a88b04b5459474fd7a475adeb7071a65a53fe183caad6a01" ) @@ -153,25 +163,28 @@ def test_step_5_covers_every_in_tree_compute_driver() -> None: def test_executed_results_pin_commits_and_evidence() -> None: manifest = load_toml(RESULTS_PATH) - for result in manifest["result"]: - if result["status"] not in {"pass", "intentional_change"}: - continue + executed = { + result["id"]: result + for result in manifest["result"] + if result["status"] in {"pass", "intentional_change"} + } + assert set(executed) == set(EXPECTED_EXECUTED_CANDIDATE_COMMITS) - assert set(result) >= PASS_FIELDS, result["id"] + for result_id, result in executed.items(): + assert set(result) >= PASS_FIELDS, result_id assert result["validated_baseline_commit"] == manifest["baseline_commit"] candidate = assert_full_sha( result["validated_candidate_commit"], - f"{result['id']}.validated_candidate_commit", + f"{result_id}.validated_candidate_commit", ) + assert candidate == EXPECTED_EXECUTED_CANDIDATE_COMMITS[result_id] assert isinstance(result["evidence"], list) and result["evidence"] assert all( isinstance(item, str) and item.strip() for item in result["evidence"] ) - subprocess.run( - ["git", "merge-base", "--is-ancestor", candidate, "HEAD"], - cwd=REPO_ROOT, - check=True, - ) + # These immutable SHAs bind the original live-validation artifacts. + # A later history-preserving source rebase can intentionally make those + # exact execution commits non-ancestors without changing the evidence. def test_step_6_records_gateway_option_and_tls_dispositions() -> None: From 6e8b2b41bd0cf5b09f8da0e51863aeb9fff6029b Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 15:18:06 -0400 Subject: [PATCH 39/42] test(ci): isolate temporary git signing config Signed-off-by: Jesse Jaggars --- tasks/scripts/codex_security_range_test.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tasks/scripts/codex_security_range_test.py b/tasks/scripts/codex_security_range_test.py index dd77fe7693..abf5026cdc 100644 --- a/tasks/scripts/codex_security_range_test.py +++ b/tasks/scripts/codex_security_range_test.py @@ -18,7 +18,24 @@ def _git(repo: Path, *args: str) -> str: - return subprocess.check_output(["git", *args], cwd=repo).decode("utf-8").strip() + # Temporary fixture repositories must not inherit developer-wide signing + # requirements: these tests intentionally create disposable commits and + # lightweight tags without prompting for a private key. + return ( + subprocess.check_output( + [ + "git", + "-c", + "commit.gpgsign=false", + "-c", + "tag.gpgsign=false", + *args, + ], + cwd=repo, + ) + .decode("utf-8") + .strip() + ) def _commit(repo: Path, name: str) -> None: From 18e17c101c64d69e51048fd5234104665dd800dd Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 17:44:43 -0400 Subject: [PATCH 40/42] fix(config): update remaining schema v2 consumers Signed-off-by: Jesse Jaggars --- .github/workflows/branch-checks.yml | 26 ++++--------------- .../tasks/development-gateway.yml | 5 ++-- .../openshell/gateway_config_fixture_test.py | 23 ++++++++++++++++ skills/debug-openshell-cluster/SKILL.md | 12 +++++---- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 8148618d0f..2d3b2dc2aa 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -231,6 +231,11 @@ jobs: - name: Test run: mise run test:python + - name: Test local gateway configuration helpers + run: | + bash tasks/scripts/test-gateway-pull-policy.sh + bash tasks/scripts/test-gateway-config.sh + go: name: Go SDK needs: pr_metadata @@ -300,24 +305,3 @@ jobs: run: | OPENSHELL_NPM_VERSION="$(uv run python tasks/scripts/release.py get-version --npm)" \ mise run sdk:ts:publish - - gateway-config: - name: Gateway configuration fixtures - needs: pr_metadata - if: needs.pr_metadata.outputs.should_run == 'true' - runs-on: linux-amd64-cpu8 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Install tools - run: mise install --locked - - - name: Test local gateway configuration helpers - run: | - bash tasks/scripts/test-gateway-pull-policy.sh - bash tasks/scripts/test-gateway-config.sh diff --git a/nix/test-guest/provisioners/roles/gateway-rootless-podman/tasks/development-gateway.yml b/nix/test-guest/provisioners/roles/gateway-rootless-podman/tasks/development-gateway.yml index 4c89402dda..20698dbdb8 100644 --- a/nix/test-guest/provisioners/roles/gateway-rootless-podman/tasks/development-gateway.yml +++ b/nix/test-guest/provisioners/roles/gateway-rootless-podman/tasks/development-gateway.yml @@ -26,12 +26,12 @@ mode: "0600" content: | [openshell] - version = 1 + version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" - compute_drivers = ["podman"] + compute_driver = "podman" disable_tls = true [openshell.gateway.auth] @@ -42,7 +42,6 @@ public_key_path = "{{ openshell_gateway_state_root }}/pki/jwt/public.pem" kid_path = "{{ openshell_gateway_state_root }}/pki/jwt/kid" gateway_id = "openshell-test-guest" - ttl_secs = 0 [openshell.drivers.podman] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" diff --git a/python/openshell/gateway_config_fixture_test.py b/python/openshell/gateway_config_fixture_test.py index 92f96ebfa6..e2276841dc 100644 --- a/python/openshell/gateway_config_fixture_test.py +++ b/python/openshell/gateway_config_fixture_test.py @@ -9,9 +9,14 @@ from pathlib import Path import pytest +import yaml REPO_ROOT = Path(__file__).resolve().parents[2] FIXTURE_DIR = REPO_ROOT / "e2e/configs/gateway" +TEST_GUEST_ROLE_PATH = ( + REPO_ROOT + / "nix/test-guest/provisioners/roles/gateway-rootless-podman/tasks/development-gateway.yml" +) @pytest.mark.parametrize( @@ -30,6 +35,24 @@ def test_e2e_gateway_fixtures_use_schema_v2_scalar_driver( assert "sandbox_namespace" not in gateway +def test_rootless_podman_test_guest_producer_uses_schema_v2() -> None: + tasks = yaml.safe_load(TEST_GUEST_ROLE_PATH.read_text(encoding="utf-8")) + producer = next( + task + for task in tasks + if task.get("name") == "Write rootless Podman gateway configuration" + ) + content = producer["ansible.builtin.copy"]["content"] + config = tomllib.loads(content) + gateway = config["openshell"]["gateway"] + + assert config["openshell"]["version"] == 2 + assert gateway["compute_driver"] == "podman" + assert "compute_drivers" not in gateway + assert "ttl_secs" not in gateway["gateway_jwt"] + assert "podman" in config["openshell"]["drivers"] + + def test_docker_e2e_gateway_fixture_uses_canonical_policy_and_label() -> None: config = tomllib.loads((FIXTURE_DIR / "docker.toml").read_text(encoding="utf-8")) docker = config["openshell"]["drivers"]["docker"] diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 2961896d62..8dcc01c2db 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -92,11 +92,13 @@ journalctl -u openshell-gateway --no-pager --lines=200 Gateway configuration requires `[openshell] version = 2`, a singular `compute_driver` selector, and driver-owned settings under -`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers`, -`--drivers`, and `OPENSHELL_DRIVERS` selectors rather than silently migrating -them. Homebrew and RPM package startup migrates only exact package-generated v1 -defaults. If an upgraded package still reports an unsupported version, inspect -the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must +`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers` and +`--drivers` selectors rather than silently migrating them. One valid, nonempty +`OPENSHELL_DRIVERS` value remains a deprecated environment-only alias when the +canonical selector is absent; the gateway selects that driver with a warning. +Empty, invalid, comma-delimited, or conflicting values fail startup. Homebrew +and RPM package startup migrates only exact package-generated v1 defaults. If +an upgraded package still reports an unsupported version, inspect the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must follow the published schema-v2 migration steps and must not be overwritten. Guest TLS CA, certificate, and key paths are the exception to driver ownership: configure the complete bundle under `[openshell.gateway]`, and the gateway From 6c91583c8d8c9f41c0492ad6004a0c6282fcdb92 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 4 Sep 2026 18:13:54 -0400 Subject: [PATCH 41/42] fix(ci): provide e2fs tools to VM tests Signed-off-by: Jesse Jaggars --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index e361fd1597..78f294fe5c 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,8 @@ cargo-nextest # Assemble Debian artifacts on macOS and Linux. dpkg + # Build and inspect ext4 images in VM driver tests. + e2fsprogs git # Required to find packages. pkg-config From e5ff58b3ecfd3c1fd74e8e6ce5dee102e5408243 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Sat, 5 Sep 2026 10:24:25 -0400 Subject: [PATCH 42/42] fix(config): align preflight with gateway startup Signed-off-by: Jesse Jaggars --- architecture/compute-runtimes.md | 8 +- crates/openshell-driver-docker/src/lib.rs | 44 +- .../openshell-driver-kubernetes/src/config.rs | 10 + .../openshell-driver-kubernetes/src/driver.rs | 17 +- crates/openshell-driver-podman/src/config.rs | 24 ++ crates/openshell-driver-podman/src/driver.rs | 29 +- crates/openshell-gateway/src/lib.rs | 127 ++++-- crates/openshell-gateway/src/vm.rs | 47 ++- crates/openshell-server/src/cli.rs | 390 ++++++++++++++++-- crates/openshell-server/src/lib.rs | 145 ++++++- .../openshell/templates/_gateway-workload.tpl | 6 +- .../openshell/tests/gateway_config_test.yaml | 24 ++ deploy/man/openshell-gateway.8.md | 24 +- docs/about/installation.mdx | 12 +- docs/reference/gateway-config.mdx | 37 +- python/openshell/release_formula_test.py | 4 +- skills/debug-openshell-cluster/SKILL.md | 33 +- tasks/scripts/snap-gateway-wrapper.sh | 8 +- tasks/scripts/test-packaging-assets.sh | 2 +- tasks/scripts/test-snap-gateway-wrapper.sh | 44 +- 20 files changed, 835 insertions(+), 200 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 2b573f2413..7ef7626353 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -127,9 +127,11 @@ defines the available implementation set, while the runtime consumes a generic registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry -to `run_cli_with_compute_drivers`; factories receive only the selected -`[openshell.drivers.]` table and return either an in-process driver or a -gateway-managed remote endpoint. The server constructs the common runtime +to `run_cli_with_compute_drivers`. Factories expose the same side-effect-free +configuration validation to package preflight and runtime startup, receive only +the selected `[openshell.drivers.]` table, and return either an in-process +driver or a gateway-managed remote endpoint when built. Preflight never builds a +driver or connects to its transport. The server constructs the common runtime adapter and snapshots `GetCapabilities` for either result. A configured UDS endpoint still takes precedence over a compiled registration with the same name. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 91138f6e6c..c4cb42ca54 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -191,6 +191,34 @@ pub struct DockerComputeConfig { pub app_armor_profile: Option, } +impl DockerComputeConfig { + /// Validate startup configuration without connecting to Docker. + pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { + if let Some(socket_path) = self.socket_path.as_deref() + && socket_path.to_str().is_none() + { + return Err(Error::config(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + ))); + } + validate_sandbox_pids_limit(self.sandbox_pids_limit)?; + validate_image_pull_policy(self.image_pull_policy)?; + self.upstream_proxy.validate().map_err(Error::config)?; + if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } + parse_optional_host_gateway_ip(&self.host_gateway_ip)?; + if gateway_bind_address.port() == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } + Ok(()) + } +} + impl Default for DockerComputeConfig { fn default() -> Self { Self { @@ -508,6 +536,7 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { + docker_config.validate_configuration(gateway_bind_address)?; let socket_path = docker_config .socket_path .clone() @@ -536,24 +565,9 @@ impl DockerComputeDriver { .is_some_and(|dirs| !dirs.is_empty()); let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - validate_image_pull_policy(docker_config.image_pull_policy)?; - docker_config - .upstream_proxy - .validate() - .map_err(Error::config)?; validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; - if let Some(socket) = docker_config.provider_spiffe_workload_api_socket.as_deref() { - openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) - .map_err(Error::config)?; - } validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } let network_name = docker_network_name(docker_config); let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 56b553c351..d8a7b063c7 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -401,6 +401,16 @@ impl Default for KubernetesComputeConfig { } impl KubernetesComputeConfig { + /// Validate startup configuration without connecting to Kubernetes. + pub fn validate_configuration(&self) -> Result<(), String> { + self.validate_workspace_mode()?; + self.validate_provider_spiffe_workload_api_socket_path()?; + self.validate_sandbox_identity_config()?; + self.validate_proxy_uid()?; + self.validate_image_pull_policies()?; + self.validate_upstream_proxy_config() + } + /// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS, /// MAX_SA_TOKEN_TTL_SECS]` range used by the projected-volume spec. /// Invalid (≤0) values fall back to the default 3600. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 4ee23b26b8..8c403ca194 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -497,22 +497,7 @@ impl KubernetesComputeDriver { shutdown_rx: tokio::sync::watch::Receiver, ) -> Result { config - .validate_workspace_mode() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_provider_spiffe_workload_api_socket_path() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_sandbox_identity_config() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_proxy_uid() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_image_pull_policies() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_upstream_proxy_config() + .validate_configuration() .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index ba6881800a..9f139cde94 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -221,6 +221,30 @@ pub fn parse_id_map_entry( } impl PodmanComputeConfig { + /// Validate and normalize startup configuration without connecting to Podman. + pub fn validate_configuration(&mut self) -> Result<(), crate::client::PodmanApiError> { + self.validate_tls_config()?; + self.validate_runtime_limits()?; + self.validate_host_gateway_ip()?; + self.validate_proxy_config()?; + self.validate_app_armor_profile()?; + if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { + let raw = socket.to_str().ok_or_else(|| { + crate::client::PodmanApiError::InvalidInput( + "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), + ) + })?; + // Preserve pass-through support for an explicitly configured + // container-reachable Workload API TCP endpoint. + if !raw.starts_with("tcp:") { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(crate::client::PodmanApiError::InvalidInput)?; + } + } + self.canonicalize_userns()?; + self.validate_userns_mappings() + } + /// Returns `true` when all three TLS paths are configured. #[must_use] pub fn tls_enabled(&self) -> bool { diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index e14bf2d9d6..d96cbf2a98 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -367,31 +367,10 @@ impl PodmanComputeDriver { } } - // Validate TLS configuration before connecting. Partial configs - // (e.g. CA set but cert/key missing) are rejected early so operators - // get a clear error instead of a silent fallback to plaintext HTTP. - config.validate_tls_config()?; - config.validate_runtime_limits()?; - config.validate_host_gateway_ip()?; - config.validate_proxy_config()?; - config.validate_app_armor_profile()?; - if let Some(socket) = config.provider_spiffe_workload_api_socket.as_deref() { - let raw = socket.to_str().ok_or_else(|| { - PodmanApiError::InvalidInput( - "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), - ) - })?; - // Preserve Podman's established pass-through support for an - // explicitly configured container-reachable Workload API TCP - // endpoint. The Workload API client validates its endpoint grammar - // when it connects. - if !raw.starts_with("tcp:") { - openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) - .map_err(PodmanApiError::InvalidInput)?; - } - } - config.canonicalize_userns()?; - config.validate_userns_mappings()?; + // Validate and normalize configuration before connecting. Partial TLS + // and invalid resource, proxy, SPIFFE, AppArmor, or userns settings + // fail before the runtime is contacted. + config.validate_configuration()?; let client = PodmanClient::new(socket_path); diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index dc97fc9ffe..2f222c6cb8 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -70,6 +70,13 @@ struct UnsupportedWindowsFactory { #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for UnsupportedWindowsFactory { + fn validate_config( + &self, + _context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Err(unsupported_windows_compute_driver(self.name)) + } + async fn build( &self, _context: openshell_server::ComputeDriverBuildContext<'_>, @@ -90,6 +97,14 @@ struct MxcFactory; #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for MxcFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let _: openshell_driver_mxc::MxcComputeConfig = context.driver_config()?; + Ok(()) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, @@ -161,18 +176,20 @@ struct KubernetesFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for KubernetesFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + kubernetes_config(context)? + .validate_configuration() + .map_err(openshell_core::Error::config) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = - context.driver_config()?; - if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { - config.workspace_default_storage_size = size; - } - if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { - config.workspace_storage_class = storage_class; - } + let config = kubernetes_config(context.config_context())?; let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( config, context.shutdown_receiver(), @@ -186,6 +203,21 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn kubernetes_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = + context.driver_config()?; + if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { + config.workspace_default_storage_size = size; + } + if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { + config.workspace_storage_class = storage_class; + } + Ok(config) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct DockerFactory; @@ -193,6 +225,14 @@ struct DockerFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for DockerFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + config.validate_configuration(context.gateway_bind_address()) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, @@ -226,22 +266,21 @@ struct PodmanFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for PodmanFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + podman_config(context)? + .validate_configuration() + .map_err(|error| openshell_core::Error::config(error.to_string())) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + let mut config = podman_config(context.config_context())?; require_guest_tls_for_local_driver(&context, "podman")?; - config.gateway_port = context.gateway_port(); - if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - config.socket_path = Some(path.into()); - } - if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { - config.host_gateway_ip = ip; - } - if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { - config.userns = Some(mode); - } apply_guest_tls( &mut config.guest_tls_ca, &mut config.guest_tls_cert, @@ -258,6 +297,24 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn podman_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + config.gateway_port = context.gateway_port(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { + config.socket_path = Some(path.into()); + } + if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { + config.host_gateway_ip = ip; + } + if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { + config.userns = Some(mode); + } + Ok(config) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct VmFactory; @@ -265,15 +322,28 @@ struct VmFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for VmFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let mut config = vm_config(context)?; + if config.grpc_endpoint.trim().is_empty() { + let scheme = if context.gateway_tls_enabled() { + "https" + } else { + "http" + }; + config.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port()); + } + config.validate_configuration() + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: vm::VmComputeConfig = context.driver_config()?; + let mut config = vm_config(context.config_context())?; require_guest_tls_for_local_driver(&context, "vm")?; - if config.state_dir.as_os_str().is_empty() { - config.state_dir = vm::VmComputeConfig::default_state_dir(); - } if config.grpc_endpoint.trim().is_empty() && (!context.gateway_tls_enabled() || context.guest_tls_paths().is_some()) { @@ -303,6 +373,17 @@ impl openshell_server::ComputeDriverFactory for VmFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn vm_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: vm::VmComputeConfig = context.driver_config()?; + if config.state_dir.as_os_str().is_empty() { + config.state_dir = vm::VmComputeConfig::default_state_dir(); + } + Ok(config) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn require_guest_tls_for_local_driver( context: &openshell_server::ComputeDriverBuildContext<'_>, diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 99de85d6c2..45b6b025b2 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -156,6 +156,30 @@ impl VmComputeConfig { 4096 } + /// Validate startup configuration without resolving binaries, creating + /// state directories, spawning a process, or connecting a socket. + pub fn validate_configuration(&self) -> Result<()> { + if self.grpc_endpoint.trim().is_empty() { + return Err(Error::config( + "grpc_endpoint is required when using the vm compute driver", + )); + } + validate_vm_sandbox_identity(self)?; + self.validate_proxy_config()?; + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + ) + .map_err(Error::config)?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err(Error::config( + "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", + )); + } + Ok(()) + } + fn validate_proxy_config(&self) -> Result<()> { self.upstream_proxy.validate().map_err(Error::config)?; if let Some(path) = self.proxy_ca_bundle.as_ref() { @@ -496,28 +520,7 @@ pub async fn spawn( vm_config: &VmComputeConfig, otlp_config: Option<&OtlpConfig>, ) -> Result { - if vm_config.grpc_endpoint.trim().is_empty() { - return Err(Error::config( - "grpc_endpoint is required when using the vm compute driver", - )); - } - - validate_vm_sandbox_identity(vm_config)?; - vm_config.validate_proxy_config()?; - if let Some(endpoint) = vm_config - .provider_spiffe_workload_api_tcp_endpoint - .as_deref() - { - openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( - endpoint, - vm_config.provider_spiffe_allow_guest_tcp, - ) - .map_err(Error::config)?; - } else if vm_config.provider_spiffe_allow_guest_tcp { - return Err(Error::config( - "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", - )); - } + vm_config.validate_configuration()?; let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 043f9970dc..26743ca61a 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -7,6 +7,8 @@ use clap::parser::ValueSource; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; use openshell_core::config::{DEFAULT_GATEWAY_NAME, DEFAULT_SERVER_PORT}; +use std::collections::BTreeMap; +use std::ffi::OsString; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use tracing::{error, info, warn}; @@ -54,11 +56,15 @@ enum ConfigCommand { Preflight(ConfigPreflightArgs), } -#[derive(clap::Args, Debug)] +#[derive(clap::Args, Debug, Default)] struct ConfigPreflightArgs { /// Explicit configuration path. Overrides `OPENSHELL_GATEWAY_CONFIG` and XDG discovery. - #[arg(long)] + #[arg(long, conflicts_with = "gateway_args")] path: Option, + + /// Gateway daemon arguments to replay after `--`. + #[arg(last = true, allow_hyphen_values = true, value_name = "GATEWAY_ARGS")] + gateway_args: Vec, } #[derive(clap::Args, Clone, Debug)] @@ -257,7 +263,9 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry match cli.command { Some(Commands::GenerateCerts(args)) => certgen::run(args).await, Some(Commands::Config(args)) => match args.command { - ConfigCommand::Preflight(args) => run_config_preflight(args, cli.run, &matches), + ConfigCommand::Preflight(args) => { + run_config_preflight_with_drivers(args, cli.run, &matches, &compute_drivers) + } }, None => Box::pin(run_from_args(cli.run, matches, compute_drivers)).await, } @@ -660,30 +668,127 @@ fn parse_compute_driver(value: &str) -> std::result::Result { openshell_core::config::normalize_compute_driver_name(value) } +#[cfg(test)] fn run_config_preflight( args: ConfigPreflightArgs, + run: RunArgs, + matches: &ArgMatches, +) -> Result<()> { + run_config_preflight_with_drivers(args, run, matches, &ComputeDriverRegistry::new()) +} + +fn run_config_preflight_with_drivers( + args: ConfigPreflightArgs, + run: RunArgs, + matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, +) -> Result<()> { + if args.gateway_args.is_empty() { + return run_effective_config_preflight(args.path, run, matches, compute_drivers); + } + + let replay_matches = match command().try_get_matches_from( + std::iter::once(OsString::from("openshell-gateway")).chain(args.gateway_args), + ) { + Ok(matches) => matches, + Err(error) + if matches!( + error.kind(), + clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion + ) => + { + return Ok(()); + } + Err(error) => return Err(miette::miette!("{error}")), + }; + let replay = + Cli::from_arg_matches(&replay_matches).map_err(|error| miette::miette!("{error}"))?; + if replay.command.is_some() { + // A valid non-daemon action does not consume gateway startup + // configuration. Let the immediately following invocation perform it. + return Ok(()); + } + run_effective_config_preflight(None, replay.run, &replay_matches, compute_drivers) +} + +fn run_effective_config_preflight( + path_override: Option, mut run: RunArgs, matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, ) -> Result<()> { - let path = if let Some(path) = args.path { - Some(path) + let path = if path_override.is_some() { + path_override } else { resolve_config_path(&run)? }; - let Some(path) = path else { - return Ok(()); - }; - let file = config_file::preflight(&path).map_err(|error| miette::miette!("{error}"))?; - merge_file_into_args(&mut run, &file.openshell.gateway, matches); - validate_preflight_semantics(&run, matches, &file) - .map_err(|_| config_file::ConfigPreflightError::invalid_current(&path)) - .map_err(|error| miette::miette!("{error}")) + let file = path + .as_ref() + .map(|path| config_file::preflight(path).map_err(|error| miette::miette!("{error}"))) + .transpose()?; + if let Some(file) = file.as_ref() { + merge_file_into_args(&mut run, &file.openshell.gateway, matches); + } + + let validation = (|| { + // These argument relationships are shared with daemon startup and + // remain transport-free. In particular, the deprecated selector must + // fail here exactly when the immediately following daemon invocation + // would fail. + resolve_legacy_driver_selector_env(&mut run)?; + normalize_compute_driver_socket_args(&mut run)?; + + let selection = run + .compute_driver + .as_deref() + .map(|driver| compute_drivers.select(Some(driver))) + .transpose() + .map_err(|error| miette::miette!("{error}"))?; + let selected_registration = selection + .as_ref() + .and_then(|selection| compute_drivers.get(selection.name())); + let empty_file = ConfigFile::default(); + let semantic_file = file.as_ref().unwrap_or(&empty_file); + validate_preflight_semantics(&run, matches, semantic_file, selected_registration)?; + + if let Some(selection) = selection.as_ref() { + let mut endpoint_overrides = BTreeMap::new(); + if let Some(socket) = run.compute_driver_socket.clone() { + endpoint_overrides.insert(selection.name().to_string(), socket); + } + crate::validate_compute_driver_config( + compute_drivers, + selection.name(), + run.name.trim(), + SocketAddr::new(run.bind_address, run.port), + &run.log_level, + crate::compute::driver_config::DriverStartupContext { + file: file.as_ref(), + guest_tls: None, + gateway_port: run.port, + gateway_tls_enabled: !run.disable_tls, + endpoint_overrides: &endpoint_overrides, + }, + )?; + } + Ok(()) + })(); + + match (validation, path.as_ref()) { + (Ok(()), _) => Ok(()), + (Err(_), Some(path)) => Err(miette::miette!( + "{}", + config_file::ConfigPreflightError::invalid_current(path) + )), + (Err(error), None) => Err(error), + } } fn validate_preflight_semantics( args: &RunArgs, matches: &ArgMatches, file: &ConfigFile, + selected_registration: Option<&crate::ComputeDriverRegistration>, ) -> Result<()> { let gateway = &file.openshell.gateway; validate_grpc_rate_limit_args( @@ -694,7 +799,8 @@ fn validate_preflight_semantics( .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; let has_client_ca = args.tls_client_ca.is_some(); - let mtls_auth_enabled = resolve_mtls_auth_enabled(args, matches, Some(file), None); + let mtls_auth_enabled = + resolve_mtls_auth_enabled(args, matches, Some(file), selected_registration); if args.disable_tls && has_client_ca { return Err(miette::miette!( "--disable-tls and --tls-client-ca are mutually exclusive" @@ -708,6 +814,13 @@ fn validate_preflight_semantics( "mTLS user authentication requires --tls-client-ca" )); } + if mtls_auth_enabled + && selected_registration.is_some_and(|registration| !registration.supports_mtls_user_auth()) + { + return Err(miette::miette!( + "mTLS user authentication is not supported with the selected compute driver" + )); + } if !args.disable_tls && args.tls_cert.is_some() != args.tls_key.is_some() { return Err(miette::miette!( "gateway TLS requires both --tls-cert and --tls-key" @@ -1047,6 +1160,13 @@ mod tests { #[async_trait::async_trait] impl crate::ComputeDriverFactory for TestFactory { + fn validate_config( + &self, + _context: crate::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Ok(()) + } + async fn build( &self, _context: crate::ComputeDriverBuildContext<'_>, @@ -1055,6 +1175,28 @@ mod tests { } } + #[derive(Clone, Copy)] + struct RejectingValidationFactory; + + #[async_trait::async_trait] + impl crate::ComputeDriverFactory for RejectingValidationFactory { + fn validate_config( + &self, + _context: crate::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Err(openshell_core::Error::config( + "selected driver validation hook invoked", + )) + } + + async fn build( + &self, + _context: crate::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("preflight must not build the selected driver") + } + } + fn test_registry(name: &str, singleplayer: bool, mtls: bool) -> crate::ComputeDriverRegistry { let mut registration = crate::ComputeDriverRegistration::new(name, 100, None, TestFactory).unwrap(); @@ -1552,6 +1694,189 @@ mod tests { )); } + #[test] + fn config_preflight_path_and_daemon_replay_are_mutually_exclusive() { + let error = command() + .try_get_matches_from([ + "openshell-gateway", + "config", + "preflight", + "--path", + "/tmp/gateway.toml", + "--", + "--disable-tls", + ]) + .expect_err("manual path and daemon replay must be mutually exclusive"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + #[test] + fn config_preflight_replay_validates_effective_daemon_flags() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _requests = EnvVarGuard::remove("OPENSHELL_GRPC_RATE_LIMIT_REQUESTS"); + let _window = EnvVarGuard::remove("OPENSHELL_GRPC_RATE_LIMIT_WINDOW_SECONDS"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + let error = super::run_config_preflight( + super::ConfigPreflightArgs { + gateway_args: ["--grpc-rate-limit-requests", "10"] + .map(std::ffi::OsString::from) + .to_vec(), + ..Default::default() + }, + run.clone(), + &matches, + ) + .expect_err("unpaired replayed rate limit must fail preflight"); + assert!(error.to_string().contains("requires both")); + + super::run_config_preflight( + super::ConfigPreflightArgs { + gateway_args: [ + "--grpc-rate-limit-requests", + "10", + "--grpc-rate-limit-window-seconds", + "60", + ] + .map(std::ffi::OsString::from) + .to_vec(), + ..Default::default() + }, + run, + &matches, + ) + .expect("paired replayed rate limit must pass preflight"); + } + + #[test] + fn config_preflight_matches_driver_selector_and_registry_semantics() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman,docker"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let registry = test_registry("podman", true, true); + + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("plural legacy selector must fail preflight as it fails startup"); + assert!(error.to_string().contains("exactly one non-empty")); + } + + #[test] + fn config_preflight_validates_selected_driver_without_building_it() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "local", + "--disable-tls", + ]); + let mut registry = crate::ComputeDriverRegistry::new(); + registry + .install( + crate::ComputeDriverRegistration::new( + "local", + 100, + None, + RejectingValidationFactory, + ) + .unwrap(), + ) + .unwrap(); + + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("selected driver validation hook must run"); + assert!(error.to_string().contains("validation hook invoked")); + } + + #[test] + fn config_preflight_applies_selected_driver_mtls_capability() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "shared", + "--tls-cert", + "/tls/server.pem", + "--tls-key", + "/tls/server-key.pem", + "--tls-client-ca", + "/tls/ca.pem", + "--enable-mtls-auth", + "true", + ]); + let registry = test_registry("shared", false, false); + + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("selected shared driver must reject mTLS user authentication"); + assert!(error.to_string().contains("not supported")); + } + + #[test] + fn config_preflight_validates_explicit_remote_driver_endpoint() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "remote", + "--disable-tls", + ]); + + let error = + super::run_config_preflight(super::ConfigPreflightArgs::default(), run, &matches) + .expect_err("remote driver without socket_path must fail preflight"); + assert!(error.to_string().contains("requires socket_path")); + } + #[test] fn config_preflight_validates_explicit_path_without_creating_state() { let _lock = ENV_LOCK @@ -1567,7 +1892,10 @@ mod tests { let (run, matches) = parse_with_args(&["openshell-gateway"]); super::run_config_preflight( - super::ConfigPreflightArgs { path: Some(config) }, + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, run, &matches, ) @@ -1593,7 +1921,7 @@ mod tests { let (run, matches) = parse_with_args(&["openshell-gateway"]); let error = super::run_config_preflight( - super::ConfigPreflightArgs { path: None }, + super::ConfigPreflightArgs::default(), run.clone(), &matches, ) @@ -1603,6 +1931,7 @@ mod tests { super::run_config_preflight( super::ConfigPreflightArgs { path: Some(current), + ..Default::default() }, run, &matches, @@ -1621,17 +1950,14 @@ mod tests { EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); let (run, matches) = parse_with_args(&["openshell-gateway"]); - super::run_config_preflight( - super::ConfigPreflightArgs { path: None }, - run.clone(), - &matches, - ) - .expect("absent auto-discovered config is optional"); + super::run_config_preflight(super::ConfigPreflightArgs::default(), run.clone(), &matches) + .expect("absent auto-discovered config is optional"); let missing = config_home.path().join("missing.toml"); let error = super::run_config_preflight( super::ConfigPreflightArgs { path: Some(missing.clone()), + ..Default::default() }, run, &matches, @@ -1649,6 +1975,10 @@ mod tests { let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); let dir = tempfile::tempdir().unwrap(); let cases = [ + ( + "driver-selector", + "[openshell]\nversion = 2\n[openshell.gateway]\ncompute_driver = 'secret-driver-marker'\ndisable_tls = true\n", + ), ( "rate-limit", "[openshell]\nversion = 2\n[openshell.gateway]\nname = 'secret-semantic-marker'\ngrpc_rate_limit_requests = 10\n", @@ -1679,6 +2009,7 @@ mod tests { let result = super::run_config_preflight( super::ConfigPreflightArgs { path: Some(path.clone()), + ..Default::default() }, run, &matches, @@ -1688,8 +2019,8 @@ mod tests { }; assert!(error.to_string().contains("category=malformed"), "{name}"); assert!(error.to_string().contains("detected_version=2"), "{name}"); - assert!(!error.to_string().contains("secret-semantic-marker")); - assert!(!format!("{error:?}").contains("secret-semantic-marker")); + assert!(!error.to_string().contains("secret-")); + assert!(!format!("{error:?}").contains("secret-")); assert_eq!(std::fs::read(&path).unwrap(), before, "{name}"); } } @@ -1712,6 +2043,7 @@ mod tests { super::run_config_preflight( super::ConfigPreflightArgs { path: Some(partial_external), + ..Default::default() }, run, &matches, @@ -1727,7 +2059,10 @@ mod tests { let _key = EnvVarGuard::remove("OPENSHELL_TLS_KEY"); let (run, matches) = parse_with_args(&["openshell-gateway"]); let error = super::run_config_preflight( - super::ConfigPreflightArgs { path: Some(config) }, + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, run, &matches, ) @@ -1751,7 +2086,10 @@ mod tests { let (run, matches) = parse_with_args(&["openshell-gateway"]); super::run_config_preflight( - super::ConfigPreflightArgs { path: Some(path) }, + super::ConfigPreflightArgs { + path: Some(path), + ..Default::default() + }, run, &matches, ) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 2a2bc1a4a6..a0ce5691b0 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1060,6 +1060,10 @@ pub enum ComputeDriverInstance { /// Factory for a compute driver linked into a gateway binary. #[async_trait::async_trait] pub trait ComputeDriverFactory: Send + Sync { + /// Validate selected-driver configuration without starting a driver, + /// connecting a transport, or modifying runtime state. + fn validate_config(&self, context: ComputeDriverConfigContext<'_>) -> Result<()>; + async fn build(&self, context: ComputeDriverBuildContext<'_>) -> Result; } @@ -1285,20 +1289,25 @@ impl ComputeDriverRegistry { } } -pub struct ComputeDriverBuildContext<'a> { - driver_name: String, +/// Read-only inputs available while validating a selected compute driver. +/// +/// This context deliberately exposes no shutdown handle, runtime store, or +/// transport client. Implementations must remain deterministic and must not +/// start processes, connect sockets, or modify state. +#[derive(Clone, Copy)] +pub struct ComputeDriverConfigContext<'a> { + driver_name: &'a str, gateway_name: &'a str, gateway_bind_address: SocketAddr, gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, - shutdown_rx: watch::Receiver, inherited_config_keys: &'static [&'static str], } -impl ComputeDriverBuildContext<'_> { +impl ComputeDriverConfigContext<'_> { #[must_use] pub fn driver_name(&self) -> &str { - &self.driver_name + self.driver_name } #[must_use] @@ -1326,10 +1335,65 @@ impl ComputeDriverBuildContext<'_> { self.driver_startup.gateway_tls_enabled } + /// Deserialize the selected driver's merged TOML table. + pub fn driver_config(&self) -> Result + where + T: Default + serde::de::DeserializeOwned, + { + compute::driver_config::driver_config_from_context( + self.driver_startup, + self.driver_name, + self.inherited_config_keys, + ) + } +} + +pub struct ComputeDriverBuildContext<'a> { + config: ComputeDriverConfigContext<'a>, + shutdown_rx: watch::Receiver, +} + +impl ComputeDriverBuildContext<'_> { + #[must_use] + pub fn config_context(&self) -> ComputeDriverConfigContext<'_> { + self.config + } + + #[must_use] + pub fn driver_name(&self) -> &str { + self.config.driver_name() + } + + #[must_use] + pub fn gateway_name(&self) -> &str { + self.config.gateway_name() + } + + #[must_use] + pub fn gateway_bind_address(&self) -> SocketAddr { + self.config.gateway_bind_address() + } + + #[must_use] + pub fn gateway_log_level(&self) -> &str { + self.config.gateway_log_level() + } + + #[must_use] + pub fn gateway_port(&self) -> u16 { + self.config.gateway_port() + } + + #[must_use] + pub fn gateway_tls_enabled(&self) -> bool { + self.config.gateway_tls_enabled() + } + /// Gateway client credentials that a local driver may mount into guests. #[must_use] pub fn guest_tls_paths(&self) -> Option<(&Path, &Path, &Path)> { - self.driver_startup + self.config + .driver_startup .guest_tls .map(compute::driver_config::GuestTlsPaths::as_paths) } @@ -1339,11 +1403,7 @@ impl ComputeDriverBuildContext<'_> { where T: Default + serde::de::DeserializeOwned, { - compute::driver_config::driver_config_from_context( - self.driver_startup, - &self.driver_name, - self.inherited_config_keys, - ) + self.config.driver_config() } #[must_use] @@ -1353,7 +1413,8 @@ impl ComputeDriverBuildContext<'_> { #[must_use] pub fn otlp_config(&self) -> Option<&config_file::OtlpConfig> { - self.driver_startup + self.config + .driver_startup .file .and_then(|file| file.openshell.gateway.otlp.as_ref()) } @@ -1372,7 +1433,14 @@ async fn build_compute_runtime( supervisor_sessions: Arc, shutdown_rx: watch::Receiver, ) -> Result { - let driver = resolve_configured_compute_driver(registry, selection.name(), driver_startup)?; + let driver = validate_compute_driver_config( + registry, + selection.name(), + &config.name, + config.bind_address, + &config.log_level, + driver_startup, + )?; let telemetry_compute_driver = driver.telemetry_compute_driver(registry); info!(driver = %driver.name(), "Using compute driver"); if config @@ -1389,13 +1457,15 @@ async fn build_compute_runtime( let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { let build_context = ComputeDriverBuildContext { - driver_name: registration.name.clone(), - gateway_name: &config.name, - gateway_bind_address: config.bind_address, - gateway_log_level: &config.log_level, - driver_startup, + config: ComputeDriverConfigContext { + driver_name: ®istration.name, + gateway_name: &config.name, + gateway_bind_address: config.bind_address, + gateway_log_level: &config.log_level, + driver_startup, + inherited_config_keys: registration.inherited_config_keys, + }, shutdown_rx, - inherited_config_keys: registration.inherited_config_keys, }; let instance = registration.factory.build(build_context).await?; match instance { @@ -1502,6 +1572,36 @@ fn configured_compute_driver( resolve_configured_compute_driver(registry, selection.name(), driver_startup) } +#[allow(clippy::too_many_arguments)] +fn validate_compute_driver_config( + registry: &ComputeDriverRegistry, + driver_name: &str, + gateway_name: &str, + gateway_bind_address: SocketAddr, + gateway_log_level: &str, + driver_startup: compute::driver_config::DriverStartupContext<'_>, +) -> Result { + let driver = resolve_configured_compute_driver(registry, driver_name, driver_startup)?; + match &driver { + ConfiguredComputeDriver::Registered(registration) => { + registration + .factory + .validate_config(ComputeDriverConfigContext { + driver_name: ®istration.name, + gateway_name, + gateway_bind_address, + gateway_log_level, + driver_startup, + inherited_config_keys: registration.inherited_config_keys, + })?; + } + ConfiguredComputeDriver::Remote { name } => { + compute::driver_config::remote_driver_config_from_context(driver_startup, name)?; + } + } + Ok(driver) +} + fn resolve_configured_compute_driver( registry: &ComputeDriverRegistry, driver_name: &str, @@ -1783,6 +1883,13 @@ mod tests { #[async_trait::async_trait] impl super::ComputeDriverFactory for TestComputeDriverFactory { + fn validate_config( + &self, + _context: super::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Ok(()) + } + async fn build( &self, _context: super::ComputeDriverBuildContext<'_>, diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 54d9e1ff99..08e4afbe06 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -87,8 +87,12 @@ spec: - name: openshell-data mountPath: /var/openshell {{- end }} + # ConfigMap directory mounts expose keys through atomic-writer symlinks, + # while the gateway intentionally rejects symlinked configuration. + # The checksum annotation above rolls pods when this subPath changes. - name: gateway-config - mountPath: /etc/openshell + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml readOnly: true - name: sandbox-jwt mountPath: /etc/openshell-jwt diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 2878a80f61..9805af8254 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -45,6 +45,23 @@ tests: - exists: path: spec.template.metadata.annotations["checksum/gateway-config"] + - it: mounts gateway.toml as a regular read-only subPath file + template: templates/statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-config + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: gateway-config + configMap: + name: openshell-config + - it: renders a StatefulSet by default template: templates/statefulset.yaml asserts: @@ -646,6 +663,13 @@ tests: - equal: path: spec.template.spec.containers[0].name value: openshell-gateway + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-config + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml + readOnly: true - notContains: path: spec.template.spec.containers[0].volumeMounts content: diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 1369e5ddbf..970c6e1863 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -14,7 +14,7 @@ openshell-gateway - OpenShell gateway server daemon **openshell-gateway** \[*OPTIONS*\] -**openshell-gateway** **config preflight** [**--path** *PATH*] +**openshell-gateway** **config preflight** [**--path** *PATH* | **--** *GATEWAY_ARGS*...] # DESCRIPTION @@ -116,23 +116,29 @@ configured in the TOML file passed with **--config**. Validate a gateway configuration before starting the daemon: - openshell-gateway config preflight [--path PATH] + openshell-gateway config preflight [--path PATH | -- GATEWAY_ARGS...] With no path, preflight validates a nonempty OPENSHELL_GATEWAY_CONFIG. If that variable is unset, it optionally validates an auto-discovered XDG config. The absence of either config succeeds. An explicit missing path, legacy schema-v1 file, invalid TOML, symlink, or nonregular file fails with a nonzero status. Preflight merges file and environment values and applies read-only startup checks -for rate-limit, TLS, interceptor, and middleware relationships. Preflight never -changes the file and reports that failed input was preserved. +for selector and socket normalization, registered compute-driver configuration, +rate limits, TLS and mTLS, interceptors, and middleware. It does not construct a +compute driver or connect to a transport. Preflight never changes the file and +reports that failed input was preserved. + +Arguments after **--** replace **--path** mode and are parsed as the exact gateway +daemon invocation. Package wrappers use this form so command-line overrides are +validated before the same arguments reach startup. The Debian and Ubuntu systemd user unit runs preflight before certificate generation, while retaining its EnvironmentFile and bare ExecStart behavior. The -Snap wrapper first validates a nonempty OPENSHELL_GATEWAY_CONFIG. Otherwise it -validates the canonical SNAP_COMMON/gateway.toml path whenever it exists or is a -symlink. A broken symlink fails preflight before the gateway is started. Correct -or manually migrate an operator-owned v1 file, then run preflight again before -restarting the service. +Snap wrapper replays its effective daemon arguments through preflight. It first +uses a nonempty OPENSHELL_GATEWAY_CONFIG. Otherwise it passes the canonical +SNAP_COMMON/gateway.toml path whenever it exists or is a symlink. A broken symlink +fails preflight before the gateway is started. Correct or manually migrate an +operator-owned v1 file, then run preflight again before restarting the service. # SYSTEMD INTEGRATION diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index aacdb3380d..7bacb55653 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -170,9 +170,13 @@ Check the selected file before restarting a service: openshell-gateway config preflight --path ~/.config/openshell/gateway.toml ``` -Without --path, preflight checks a nonempty OPENSHELL_GATEWAY_CONFIG or an -auto-discovered XDG config; no config is also a successful result. Debian keeps -its bare service invocation and gateway.env semantics. Snap gives a nonempty -OPENSHELL_GATEWAY_CONFIG precedence over SNAP_COMMON/gateway.toml. +Without `--path`, preflight checks a nonempty `OPENSHELL_GATEWAY_CONFIG` or an +auto-discovered XDG config; no config is also a successful result. It applies +read-only startup validation to the effective selector, registered compute driver, +driver configuration, socket, rate limits, TLS, interceptors, and middleware. +Wrappers can pass daemon arguments after `--` to validate the exact startup +invocation. Debian keeps its bare service invocation and `gateway.env` semantics. +Snap replays its effective daemon arguments through preflight and gives a nonempty +`OPENSHELL_GATEWAY_CONFIG` precedence over `SNAP_COMMON/gateway.toml`. See [Gateway Configuration](/reference/gateway-config#gateway-config-preflight) for preflight details and manual schema-v1 migration steps. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index de1433561a..95411764de 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -983,22 +983,33 @@ changing it: openshell-gateway config preflight --path ~/.config/openshell/gateway.toml ``` -Without --path, the command validates a nonempty OPENSHELL_GATEWAY_CONFIG. +Without `--path`, the command validates a nonempty `OPENSHELL_GATEWAY_CONFIG`. Otherwise, it validates an existing XDG gateway config when one is discovered. When neither source selects a config, preflight succeeds. An explicit missing path, a legacy schema-v1 file, invalid TOML, a symlink, or any nonregular file fails. -Preflight also merges the selected file with the current `OPENSHELL_*` environment -and applies read-only startup checks for rate-limit pairs, TLS relationships, -interceptor registrations, and supervisor middleware registrations. It validates -complete guest TLS path sets without requiring package-generated certificates to -exist before certificate generation. A failed preflight always preserves the -file; it never migrates, replaces, or rewrites configuration. - -Debian and Ubuntu run this preflight from the systemd user unit before local -certificate generation. The unit still loads the gateway.env environment file -and starts the gateway with no configuration arguments. Snap validates a nonempty -OPENSHELL_GATEWAY_CONFIG first; otherwise it validates and passes its canonical -SNAP_COMMON/gateway.toml only when that path exists in the filesystem. A broken +Preflight merges the selected file with the current `OPENSHELL_*` environment and +applies the daemon's read-only startup checks. These checks include selector and +socket normalization, registered-driver selection and configuration, rate-limit +pairs, TLS and mTLS relationships, interceptor registrations, and supervisor +middleware registrations. It validates complete guest TLS path sets without +requiring package-generated certificates to exist before certificate generation. +It does not construct a compute driver or connect to a transport. A failed +preflight always preserves the file; it never migrates, replaces, or rewrites +configuration. + +To validate the exact daemon arguments that a wrapper will pass, place them after +`--` instead of using `--path`: + +```shell +openshell-gateway config preflight -- --config /etc/openshell/gateway.toml --grpc-rate-limit-requests 100 --grpc-rate-limit-window-seconds 60 +``` + +Debian and Ubuntu run preflight from the systemd user unit before local certificate +generation. The unit still loads the `gateway.env` environment file and starts the +gateway with no configuration arguments. Snap replays the exact effective daemon +arguments through preflight. It gives a nonempty `OPENSHELL_GATEWAY_CONFIG` +precedence; otherwise it validates and passes its canonical +`SNAP_COMMON/gateway.toml` only when that path exists in the filesystem. A broken symlink is therefore rejected instead of being treated as absent. Package startup does not modify an operator-owned v1 file. Back it up, follow diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index b31a686980..9a4948116f 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -241,8 +241,8 @@ def test_schema_v2_debian_and_snap_preflight_wiring() -> None: in wrapper ) assert wrapper.count('"${SNAP}/bin/openshell-gateway" config preflight') == 4 - assert 'config preflight "--path=$cli_config"' in wrapper - assert 'config preflight --path "$CANONICAL_CONFIG_FILE"' in wrapper + assert 'config preflight -- "$@"' in wrapper + assert 'config preflight -- --config "$CANONICAL_CONFIG_FILE" "$@"' in wrapper assert ( 'exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@"' in wrapper diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8dcc01c2db..60390ec9a5 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -297,6 +297,20 @@ release. Look for failed installs, unexpected values, missing namespace, wrong image tag, TLS settings that do not match the registered endpoint, and scheduling failures. +The chart mounts the `gateway.toml` ConfigMap key directly at +`/etc/openshell/gateway.toml` as a read-only `subPath` file. This avoids the +atomic-writer symlink exposed by a ConfigMap directory mount because the gateway +rejects symlinked configuration. A checksum pod-template annotation rolls the +workload when the ConfigMap changes. If config preflight reports a symlink or +nonregular path, inspect the rendered mount and confirm the workload rolled to +the current chart revision: + +```bash +kubectl -n openshell get deployment,statefulset -o yaml | rg -n 'gateway-config|mountPath|subPath|checksum/gateway-config' +kubectl -n openshell rollout status /openshell +kubectl -n openshell logs -c openshell-gateway --tail=200 +``` + `server.telemetryEnabled` renders `OPENSHELL_TELEMETRY_ENABLED` on the gateway pod, and the gateway propagates the effective value to sandbox supervisors. @@ -749,13 +763,16 @@ When handing results back to the user, include: For a Debian, Ubuntu, or Snap gateway that stops before certificate generation or daemon startup, validate the selected configuration without starting the service: - openshell-gateway config preflight [--path PATH] +```shell +openshell-gateway config preflight [--path PATH | -- GATEWAY_ARGS...] +``` -Without a path, preflight validates a nonempty OPENSHELL_GATEWAY_CONFIG or an +Without a path, preflight validates a nonempty `OPENSHELL_GATEWAY_CONFIG` or an auto-discovered XDG config; no config succeeds. An explicit missing path, legacy -schema-v1 file, malformed TOML, symlink, or nonregular file fails before the -gateway ExecStart. It also applies read-only effective-config checks for rate -limits, TLS, interceptors, and supervisor middleware. Preflight preserves every -failed file. Do not advise users to -delete or rewrite it automatically; back it up and follow the manual schema-v2 -migration in the Gateway Configuration reference. +schema-v1 file, malformed TOML, symlink, or nonregular file fails before gateway +startup. It also applies read-only effective-config checks for driver selection +and configuration, sockets, rate limits, TLS, interceptors, and supervisor +middleware. Arguments after `--` validate the effective daemon invocation, +including its command-line overrides. Preflight preserves every failed file. Do +not advise users to delete or rewrite it automatically; back it up and follow the +manual schema-v2 migration in the Gateway Configuration reference. diff --git a/tasks/scripts/snap-gateway-wrapper.sh b/tasks/scripts/snap-gateway-wrapper.sh index ffc03966ab..83eebcdcb4 100755 --- a/tasks/scripts/snap-gateway-wrapper.sh +++ b/tasks/scripts/snap-gateway-wrapper.sh @@ -64,15 +64,15 @@ if [ "$expect_config_path" = true ] || { [ "$config_seen" = true ] && [ -z "$cli fi if [ "$config_seen" = true ]; then - "${SNAP}/bin/openshell-gateway" config preflight "--path=$cli_config" + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" exec "${SNAP}/bin/openshell-gateway" "$@" elif [ -n "${OPENSHELL_GATEWAY_CONFIG:-}" ]; then - "${SNAP}/bin/openshell-gateway" config preflight + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" exec "${SNAP}/bin/openshell-gateway" "$@" elif [ -e "$CANONICAL_CONFIG_FILE" ] || [ -L "$CANONICAL_CONFIG_FILE" ]; then - "${SNAP}/bin/openshell-gateway" config preflight --path "$CANONICAL_CONFIG_FILE" + "${SNAP}/bin/openshell-gateway" config preflight -- --config "$CANONICAL_CONFIG_FILE" "$@" exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@" else - "${SNAP}/bin/openshell-gateway" config preflight + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" exec "${SNAP}/bin/openshell-gateway" "$@" fi diff --git a/tasks/scripts/test-packaging-assets.sh b/tasks/scripts/test-packaging-assets.sh index 48c58687dc..ac7b2b190a 100755 --- a/tasks/scripts/test-packaging-assets.sh +++ b/tasks/scripts/test-packaging-assets.sh @@ -71,7 +71,7 @@ assert_contains "$snap_wrapper" "if [ -n \"\${OPENSHELL_GATEWAY_CONFIG:-}\" ]; t assert_contains \ "$snap_wrapper" \ "elif [ -e \"\$CANONICAL_CONFIG_FILE\" ] || [ -L \"\$CANONICAL_CONFIG_FILE\" ]; then" -assert_contains "$snap_wrapper" "config preflight --path \"\$CANONICAL_CONFIG_FILE\"" +assert_contains "$snap_wrapper" "config preflight -- --config \"\$CANONICAL_CONFIG_FILE\" \"\$@\"" assert_not_contains "$snap_wrapper" "[ -f \"\$CANONICAL_CONFIG_FILE\" ]" bash "$ROOT/tasks/scripts/test-snap-gateway-wrapper.sh" "$snap_wrapper" if ! awk '/config preflight/ { seen = 1 } /generate-certs/ { exit !seen }' "$service"; then diff --git a/tasks/scripts/test-snap-gateway-wrapper.sh b/tasks/scripts/test-snap-gateway-wrapper.sh index e6286b00a4..733ec796b5 100755 --- a/tasks/scripts/test-snap-gateway-wrapper.sh +++ b/tasks/scripts/test-snap-gateway-wrapper.sh @@ -23,8 +23,20 @@ printf 'env:%s|%s|%s\n' \ "${OPENSHELL_GATEWAY_CONFIG:-}" \ "${OPENSHELL_DB_URL:-}" \ "${OPENSHELL_DISABLE_TLS:-}" >>"$FAKE_GATEWAY_LOG" -if [ "${1:-}" = config ] && [ "${2:-}" = preflight ] && [ "${FAKE_PREFLIGHT_FAIL:-}" = 1 ]; then - exit 42 +if [ "${1:-}" = config ] && [ "${2:-}" = preflight ]; then + if [ "${FAKE_PREFLIGHT_FAIL:-}" = 1 ]; then + exit 42 + fi + if [ "${FAKE_REJECT_UNPAIRED_RATE:-}" = 1 ]; then + case " $* " in + *" --grpc-rate-limit-requests "*) + case " $* " in + *" --grpc-rate-limit-window-seconds "*) ;; + *) exit 43 ;; + esac + ;; + esac + fi fi EOF chmod +x "$snap/bin/openshell-gateway" @@ -64,7 +76,7 @@ printf 'operator override\n' >"$override" cp "$override" "$work/override-before" : >"$log" run_wrapper "$override" -assert_log "config preflight +assert_log "config preflight -- --trace env:$override|sqlite:$common/gateway.db?mode=rwc|true --trace env:$override|sqlite:$common/gateway.db?mode=rwc|true" @@ -80,7 +92,7 @@ env \ OPENSHELL_GATEWAY_CONFIG="$override" \ FAKE_GATEWAY_LOG="$log" \ "$wrapper" --trace --config "$cli_config" -assert_log "config preflight --path=$cli_config +assert_log "config preflight -- --trace --config $cli_config env:$override|sqlite:$common/gateway.db?mode=rwc|true --trace --config $cli_config env:$override|sqlite:$common/gateway.db?mode=rwc|true" @@ -97,10 +109,24 @@ if env \ echo "FAIL: CLI-selected config preflight failure reached gateway start" >&2 exit 1 fi -assert_log "config preflight --path=$cli_config +assert_log "config preflight -- --config=$cli_config env:$override|sqlite:$common/gateway.db?mode=rwc|true" cmp -s "$work/cli-before" "$cli_config" +: >"$log" +if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_REJECT_UNPAIRED_RATE=1 \ + "$wrapper" --grpc-rate-limit-requests 10; then + echo "FAIL: invalid daemon overrides reached gateway start" >&2 + exit 1 +fi +assert_log "config preflight -- --grpc-rate-limit-requests 10 +env:$override|sqlite:$common/gateway.db?mode=rwc|true" + for invalid_selector in terminator nested-config; do : >"$log" if [ "$invalid_selector" = terminator ]; then @@ -130,7 +156,7 @@ env \ OPENSHELL_GATEWAY_CONFIG="$override" \ FAKE_GATEWAY_LOG="$log" \ "$wrapper" --config=--dash-leading -assert_log "config preflight --path=--dash-leading +assert_log "config preflight -- --config=--dash-leading env:$override|sqlite:$common/gateway.db?mode=rwc|true --config=--dash-leading env:$override|sqlite:$common/gateway.db?mode=rwc|true" @@ -140,7 +166,7 @@ printf 'valid schema-v2\n' >"$canonical" cp "$canonical" "$work/canonical-before" : >"$log" run_wrapper unset -assert_log "config preflight --path $canonical +assert_log "config preflight -- --config $canonical --trace env:|sqlite:$common/gateway.db?mode=rwc|true --config $canonical --trace env:|sqlite:$common/gateway.db?mode=rwc|true" @@ -149,7 +175,7 @@ cmp -s "$work/canonical-before" "$canonical" rm "$canonical" : >"$log" run_wrapper unset -assert_log "config preflight +assert_log "config preflight -- --trace env:|sqlite:$common/gateway.db?mode=rwc|true --trace env:|sqlite:$common/gateway.db?mode=rwc|true" @@ -161,7 +187,7 @@ assert_preflight_failure() { echo "FAIL: $name reached gateway start" >&2 exit 1 fi - assert_log "config preflight --path $canonical + assert_log "config preflight -- --config $canonical --trace env:|sqlite:$common/gateway.db?mode=rwc|true" }