diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 2d0fdfe366..c710f289f4 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -26,6 +26,10 @@ does not contribute to product usage metrics. `install.sh` defaults to the *latest tagged* release — the canary is therefore checking that the most recent public release still installs, not the just-published `dev` build. The `kubernetes` job is the exception: it pins to `0.0.0-dev` chart + `:dev` images. +The host-package jobs exercise fresh installs, not upgrades from a persisted +schema-v1 gateway config. Validate Homebrew and RPM exact-default migration with +the release-tooling and package lifecycle tests before relying on the canary. + The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK validation lives in the `TypeScript SDK` branch check, including a publish dry-run. The tagged release workflow publishes the package to GitHub Packages; @@ -131,7 +135,7 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i | Symptom | Likely cause | Where to look | |---|---|---| | `macos`/`ubuntu`/`fedora` job fails on `install.sh` | Latest tagged release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. | -| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_DRIVERS` env in the "Ensure …" step. | +| `macos`/`ubuntu`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_COMPUTE_DRIVER` env in the "Ensure …" step. | | `ubuntu-snap` fails after interface connection | The gateway did not recover after Docker became available, or did not become reachable within the 30-second bound. | Failure diagnostics dump Snap service/connection/change state, gateway and snapd journals, Snap logs, and port 17670 listeners. | | `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. | | `kubernetes` job fails on `kubectl wait` | Gateway pod stuck `CrashLoopBackOff` or `ImagePullBackOff`. | Diagnostics dump; check `:dev` image existence at `ghcr.io/nvidia/openshell/gateway`. | diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 5d4a98c61a..c3f0a55bdd 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -233,6 +233,11 @@ jobs: - name: Test run: mise run test:python + - name: Test local gateway configuration helpers + run: | + bash tasks/scripts/test-gateway-pull-policy.sh + bash tasks/scripts/test-gateway-config.sh + go: name: Go SDK needs: pr_metadata diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index ee4074142e..98470584de 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Ensure VM driver run: | - launchctl setenv OPENSHELL_DRIVERS vm + launchctl setenv OPENSHELL_COMPUTE_DRIVER vm launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED" - name: Install and check status @@ -53,7 +53,7 @@ jobs: fi sudo systemctl start docker || sudo service docker start mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" docker info @@ -145,7 +145,7 @@ jobs: bash -s <<'EOF' set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_COMPUTE_DRIVER=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" podman info curl -LsSf "${INSTALL_SH_URL}" | sh @@ -301,7 +301,7 @@ jobs: run: | set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=docker\n' > "${HOME}/.config/openshell/gateway.env" + printf 'OPENSHELL_COMPUTE_DRIVER=docker\n' > "${HOME}/.config/openshell/gateway.env" curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh - name: Register kind gateway and check status diff --git a/Cargo.lock b/Cargo.lock index c713ace9ce..df78c58ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4139,11 +4139,13 @@ dependencies = [ "openshell-driver-mxc", "openshell-driver-podman", "openshell-otel", + "openshell-policy", "openshell-server", "rustix 1.1.4", "serde", "tempfile", "tokio", + "toml", "tonic", "tower 0.5.3", "tracing", @@ -4356,6 +4358,7 @@ dependencies = [ "jsonwebtoken", "k8s-openapi", "kube", + "libc", "metrics", "metrics-exporter-prometheus", "miette", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..0329668023 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -127,11 +127,16 @@ defines the available implementation set, while the runtime consumes a generic registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry -to `run_cli_with_compute_drivers`; factories receive merged driver config and -return either an in-process driver or a gateway-managed remote endpoint. The -server constructs the common runtime adapter and snapshots `GetCapabilities` -for either result. A configured UDS endpoint still takes precedence over a -compiled registration with the same name. +to `run_cli_with_compute_drivers`. Factories expose the same side-effect-free +configuration validation to package preflight and runtime startup, receive only +the selected `[openshell.drivers.]` table, and return either an in-process +driver or a gateway-managed remote endpoint when built. If a selected config file +omits the selector, preflight validates each configured table that belongs to an +auto-detectable registration. It does not run runtime detection probes because +those probes can connect local sockets or launch bounded discovery commands. +Preflight never builds a driver or connects to its transport. The server constructs the common runtime +adapter and snapshots `GetCapabilities` for either result. A configured UDS +endpoint still takes precedence over a compiled registration with the same name. The `openshell-gateway` composition crate groups first-party registrations behind the `in-tree-compute-drivers` feature. `openshell-server` has no compute @@ -251,7 +256,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | -| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_driver = ""` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--compute-driver ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -288,10 +293,27 @@ pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise they implement and validate the same complete contract. The capability marker is driver-owned supervisor input and is removed from workload environments. -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Kubernetes, Docker, and Podman share one AppArmor configuration model: +`RuntimeDefault`, `Unconfined`, or `Localhost/`. Each driver translates +that model to its native API and rejects an explicitly requested confined +profile when its backend reports AppArmor unavailable. Docker keeps its +historical explicit `Unconfined` default. Podman sends no override when the +field is omitted, preserving the runtime-selected profile; development paths +that require the supervisor's namespace mount setup opt into `Unconfined` +explicitly. The Helm chart independently uses `Unconfined` for Kubernetes. + +Corporate proxy settings are driver-owned supervisor inputs. Docker, Podman, +and VM propagate `https_proxy`, `no_proxy`, an optional root-only auth file, +and the explicit cleartext-Basic-auth acknowledgement without allowing +workload environment to override them. Podman and VM can also project an +operator CA bundle for an HTTPS or TLS-intercepting proxy. The VM driver validates and +stages its credential and CA bundle under fixed guest paths, then forwards +those paths through the protected supervisor argument file rather than the +guest environment. Local containers project provider SPIFFE +through a dedicated host UNIX-socket parent mount. A VM cannot safely expose +that host socket: it accepts only a separately operated, concrete TCP listener +when `provider_spiffe_allow_guest_tcp = true` explicitly acknowledges guest +access. Host-only sockets are never implicitly forwarded to VM guests. The Kubernetes deployment packaging has two ownership boundaries. The gateway chart owns the gateway workload, configuration, Services, PKI, and @@ -309,10 +331,14 @@ can request a specific number of GPUs or the driver-specific default behaviour. For all in-tree drivers, this is equivalent to selecting a single GPU. VM runtime state paths are derived only from driver-validated sandbox IDs -matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a -private `run/` directory plus Unix peer UID/PID checks. Standalone -unauthenticated TCP mode is disabled unless explicitly enabled for local -development. +matching `[A-Za-z0-9._-]{1,128}`. Each writable overlay records its effective +sandbox UID/GID so later rootfs cache changes cannot rewrite persisted file +ownership. Unmarked pre-migration overlays recover identity from concrete +overlay or prepared-rootfs state, an explicit operator override, or the current +image account. The driver never assumes `10001:10001`; it preserves that legacy +identity only when persisted state reports it. The gateway-owned VM driver +socket uses a private `run/` directory plus Unix peer UID/PID checks. Standalone unauthenticated TCP +mode is disabled unless explicitly enabled for local development. Runtime-specific implementation notes belong in the driver crate README: diff --git a/architecture/gateway.md b/architecture/gateway.md index 769f57f6a0..b203ab9b7d 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -38,6 +38,20 @@ immediately without a grace period. Finalization is persisted separately from the exit result; the gateway deletes an ephemeral sandbox only after the finalized supervisor session disconnects. +## Configuration Boundary + +The gateway accepts exactly schema version 2. Missing, legacy, and future +versions fail before runtime construction, and driver settings belong only to +`[openshell.drivers.]`. The process does not migrate legacy files. +Package lifecycle code may replace an exact package-generated v1 default, but +it preserves edited configurations for explicit operator migration. + +Gateway listener TLS and sandbox callback TLS are separate inputs. A selected +local Docker, Podman, or VM driver requires a complete guest bundle whenever +the gateway listener uses TLS; package-managed local TLS can supply that bundle. +Kubernetes instead projects guest credentials through its configured Secret. +The gateway validates this requirement before constructing the selected driver. + ## Protocol and Auth The gateway listens on one service port and multiplexes gRPC and HTTP traffic. @@ -246,10 +260,10 @@ controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing deployments. Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. -The config default is -`gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM -gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other -shared deployments should set a positive TTL. +Omitting `gateway_jwt.ttl_secs` selects non-expiring tokens for local +single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0`. +Kubernetes and other shared deployments should set a positive TTL. Explicit +zero is rejected. Gateway JWT signing-key rotation is currently an offline operator action. The runtime loads one active signing key and one matching public verification key @@ -694,9 +708,10 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ``` The TOML file is opt-in via `--config ` / `OPENSHELL_GATEWAY_CONFIG`. -Driver implementation settings live in the TOML driver tables. See -`docs/reference/gateway-config.mdx` for worked per-driver examples and RFC -0003 for the full schema. +Driver implementation settings live exclusively in TOML driver tables. The +selector is the singular `[openshell.gateway] compute_driver`; legacy +`compute_drivers` lists are rejected. See `docs/reference/gateway-config.mdx` +for worked per-driver examples and RFC 0003 for the full schema. Each installation has an operator-assigned gateway name. Configure it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. @@ -710,26 +725,20 @@ aliases, network names, and the sandbox JWT issuer. `database_url` is env-only and rejected when present in the file (`OPENSHELL_DB_URL` / `--db-url`). -### Driver inheritance - -`[openshell.gateway]` carries a small set of values (`sandbox_namespace`, -`default_image`, -`supervisor_image`, `guest_tls_ca/cert/key`, `client_tls_secret_name`, -`host_gateway_ip`, `enable_user_namespaces`) that are inherited into each -driver's `[openshell.drivers.]` table when the driver-specific table -does not override them. The allowlist is per-driver so a gateway-wide -default cannot land in a driver that does not understand it (e.g. -`client_tls_secret_name` is K8s-only). +### Driver ownership -`image_pull_policy` is intentionally **not** inheritable: Kubernetes uses -`Always | IfNotPresent | Never` (passed verbatim to the K8s API) while -Podman uses the lowercase enum `always | missing | never | newer`. No -value means the same thing in both, so the key lives only under each -driver's own table. +`[openshell.gateway]` contains gateway process settings only. Each selected +driver reads its own configuration exclusively from +`[openshell.drivers.]`; values are never inherited from gateway scope. +Kubernetes owns `namespace`, `default_image`, `supervisor_image`, +`client_tls_secret_name`, `service_account_name`, `host_gateway_ip`, +`enable_user_namespaces`, and `sa_token_ttl_secs`. Docker uses +`sandbox_label` instead of the legacy `sandbox_namespace` name. Podman and VM +likewise own their image, endpoint, and runtime settings in their tables. -Driver-specific values that are not part of the inheritance allowlist -(e.g. Podman `socket_path`, VM `vcpus`) only come from the driver's own -table. +`image_pull_policy` uses the shared canonical vocabulary +`always | if_not_present | never | newer`. Drivers translate it to their runtime +APIs; `newer` is supported only by Podman and rejected by Docker and Kubernetes. ### OTLP export @@ -803,7 +812,10 @@ system entry instead of pretending to delete package-manager owned state. - Gateway TLS and client certificate distribution are deployment concerns owned by the operator or packaging layer. - Compute runtimes own the mechanics of starting workloads and injecting - callback configuration. + callback configuration. Local Docker, Podman, and VM callback endpoints can + be derived from their fixed host aliases. Kubernetes requires an explicit + endpoint from deployment topology; Helm renders it from the gateway Service + name and namespace rather than inferring it from sandbox placement. - Docker-backed local gateways use Docker's `host-gateway` callback alias on macOS and Docker Desktop-style runtimes. They request IPv4 loopback callback reachability and add a listener only when the primary does not cover it. diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3507011120..f28c8368e3 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -6,7 +6,9 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; +use std::fmt; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -29,6 +31,18 @@ pub const DEFAULT_GATEWAY_NAME: &str = "openshell"; /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; +/// Default cgroup PID limit for local container sandboxes. +pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; + +/// Typed default cgroup PID limit for local container sandboxes. +#[must_use] +pub fn default_sandbox_pids_limit() -> Option { + NonZeroI64::new(DEFAULT_SANDBOX_PIDS_LIMIT) +} + +/// Default Docker bridge network name for local sandboxes. +pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; + /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; @@ -107,11 +121,6 @@ pub fn resolve_supervisor_image_tag(candidates: &[&str]) -> String { /// CDI device identifier for requesting all NVIDIA GPUs. pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; -/// Default maximum number of processes (PIDs) allowed inside a sandbox container. -/// -/// Compute drivers may override this through backend configuration. -pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; - /// Normalize a configured compute driver name. /// /// Built-in driver names and custom remote driver names share the same @@ -194,12 +203,9 @@ pub struct Config { /// Database URL for persistence. pub database_url: String, - /// Compute drivers configured for the gateway. - /// - /// The config shape allows multiple drivers so the gateway can evolve - /// toward multi-backend routing. Current releases require exactly one - /// configured driver. - pub compute_drivers: Vec, + /// Explicit compute driver configured for the gateway. + /// `None` enables runtime auto-detection. + pub compute_driver: Option, /// Operator-provided endpoints for named remote compute drivers. /// @@ -503,6 +509,225 @@ const fn default_jwks_ttl_secs() -> u64 { 3600 } +/// Canonical policy controlling when a driver pulls a sandbox image. +/// +/// Backends translate this shared vocabulary to their runtime API. `newer` is +/// supported only by Podman; other backends reject it during configuration. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImagePullPolicy { + /// Always pull, even if a local image is available. + Always, + /// Pull only when a local image is unavailable. + #[default] + IfNotPresent, + /// Never pull; fail when a local image is unavailable. + Never, + /// Pull only when the registry image is newer than the local copy. + Newer, +} + +impl ImagePullPolicy { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Always => "always", + Self::IfNotPresent => "if_not_present", + Self::Never => "never", + Self::Newer => "newer", + } + } +} + +impl fmt::Display for ImagePullPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ImagePullPolicy { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "always" => Ok(Self::Always), + "if_not_present" => Ok(Self::IfNotPresent), + "never" => Ok(Self::Never), + "newer" => Ok(Self::Newer), + other => Err(format!( + "invalid image pull policy '{other}'; expected one of: always, if_not_present, never, newer" + )), + } + } +} + +/// Canonical `AppArmor` confinement requested for a sandbox container. +/// +/// Drivers translate this model to their runtime API. An omitted value leaves +/// the runtime default unchanged; `Unconfined` is explicit because the +/// supervisor needs mount operations that the default Docker/Podman profile +/// commonly denies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AppArmorProfile { + RuntimeDefault, + Unconfined, + Localhost(String), +} + +impl AppArmorProfile { + #[must_use] + pub const fn kubernetes_type(&self) -> &'static str { + match self { + Self::RuntimeDefault => "RuntimeDefault", + Self::Unconfined => "Unconfined", + Self::Localhost(_) => "Localhost", + } + } + + #[must_use] + pub fn localhost_profile(&self) -> Option<&str> { + match self { + Self::Localhost(profile) => Some(profile), + Self::RuntimeDefault | Self::Unconfined => None, + } + } + + /// Translate to the OCI `apparmor=` security option. + /// + /// `RuntimeDefault` deliberately returns `None`: omitting an OCI option + /// asks Docker/Podman to apply their runtime default profile. + #[must_use] + pub fn oci_security_opt(&self) -> Option { + match self { + Self::RuntimeDefault => None, + Self::Unconfined => Some("apparmor=unconfined".to_string()), + Self::Localhost(profile) => Some(format!("apparmor={profile}")), + } + } +} + +impl fmt::Display for AppArmorProfile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RuntimeDefault => f.write_str("RuntimeDefault"), + Self::Unconfined => f.write_str("Unconfined"), + Self::Localhost(profile) => write!(f, "Localhost/{profile}"), + } + } +} + +impl FromStr for AppArmorProfile { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "RuntimeDefault" => Ok(Self::RuntimeDefault), + "Unconfined" => Ok(Self::Unconfined), + other => match other.strip_prefix("Localhost/") { + Some("") => Err( + "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" + .to_string(), + ), + Some(profile) if !profile.contains(char::is_whitespace) => { + Ok(Self::Localhost(profile.to_string())) + } + Some(_) => { + Err("invalid AppArmor localhost profile; whitespace is not allowed".to_string()) + } + None => Err(format!( + "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" + )), + }, + } + } +} + +impl Serialize for AppArmorProfile { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for AppArmorProfile { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::from_str(&value).map_err(serde::de::Error::custom) + } +} + +/// Common local-driver corporate forward-proxy settings. +/// +/// This type is `flatten`ed by local compute-driver tables, preserving the +/// established TOML field names while keeping their safety contract shared. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct UpstreamProxyConfig { + pub https_proxy: Option, + pub no_proxy: Option, + pub proxy_auth_file: Option, + pub proxy_auth_allow_insecure: Option, + pub proxy_connect_by_hostname: Option, +} + +impl UpstreamProxyConfig { + /// Validate relationships that are independent of the container backend. + /// Credential contents are intentionally not read here and are never put + /// in an error message; drivers validate and stage them per sandbox. + pub fn validate(&self) -> Result<(), String> { + use crate::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; + + let proxy_secure = if let Some(url) = self.https_proxy.as_deref() { + parse_upstream_proxy_url(url) + .map_err(|err| match err { + UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials; supply them with proxy_auth_file so they are not stored in configuration or runtime metadata".to_string(), + err => format!("https_proxy {err}"), + })? + .secure + } else { + false + }; + + if self + .no_proxy + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + return Err("no_proxy must not be empty when set; omit it instead".to_string()); + } + if self.no_proxy.is_some() && self.https_proxy.is_none() { + return Err("no_proxy is set but no https_proxy is configured".to_string()); + } + if let Some(path) = self.proxy_auth_file.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_auth_file must not be empty when set".to_string()); + } + if self.https_proxy.is_none() { + return Err("proxy_auth_file is set but no https_proxy is configured".to_string()); + } + if !proxy_secure && self.proxy_auth_allow_insecure != Some(true) { + return Err("proxy_auth_file sends a cleartext Basic credential to an http:// proxy; set proxy_auth_allow_insecure = true to acknowledge that exposure".to_string()); + } + } else if self.proxy_auth_allow_insecure.is_some() { + return Err( + "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), + ); + } + if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { + return Err( + "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), + ); + } + Ok(()) + } +} + /// Gateway-minted sandbox JWT configuration. /// /// Points the gateway at the Ed25519 signing key (produced by `certgen`) @@ -522,18 +747,21 @@ pub struct GatewayJwtConfig { /// `openshell`. #[serde(default = "default_gateway_id")] pub gateway_id: String, - /// Token lifetime in seconds. A value of 0 disables expiration and is - /// intended only for local single-player deployments. - #[serde(default = "default_sandbox_token_ttl_secs")] - pub ttl_secs: u64, + /// Token lifetime in seconds. Omit the field for a non-expiring token. + /// Explicit zero is invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl_secs: Option, } -fn default_gateway_id() -> String { - "openshell".to_string() +impl GatewayJwtConfig { + /// Effective token lifetime. `None` represents a non-expiring token. + pub fn sandbox_token_ttl(&self) -> Option { + self.ttl_secs.map(|ttl| Duration::from_secs(ttl.get())) + } } -const fn default_sandbox_token_ttl_secs() -> u64 { - 0 +fn default_gateway_id() -> String { + "openshell".to_string() } fn default_roles_claim() -> String { @@ -569,7 +797,7 @@ impl Config { mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, database_url: String::new(), - compute_drivers: vec![], + compute_driver: None, compute_driver_endpoints: BTreeMap::new(), credential_drivers: Vec::new(), default_credential_driver: None, @@ -620,17 +848,10 @@ impl Config { self } - /// Create a new configuration with the configured compute drivers. + /// Create a new configuration with an explicit compute driver. #[must_use] - pub fn with_compute_drivers(mut self, drivers: I) -> Self - where - I: IntoIterator, - D: ToString, - { - self.compute_drivers = drivers - .into_iter() - .map(|driver| driver.to_string()) - .collect(); + pub fn with_compute_driver(mut self, driver: impl ToString) -> Self { + self.compute_driver = Some(driver.to_string()); self } @@ -825,10 +1046,10 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ - Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, + AppArmorProfile, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, - normalize_compute_driver_name, + GatewayProviderProfileSourceConfig, ImagePullPolicy, PolicyValidationFailureMode, + UpstreamProxyConfig, default_sandbox_pids_limit, normalize_compute_driver_name, }; use std::net::SocketAddr; use std::time::Duration; @@ -922,7 +1143,242 @@ mod tests { })) .expect("gateway JWT config should deserialize with default ttl"); - assert_eq!(cfg.ttl_secs, 0); + assert_eq!(cfg.ttl_secs, None); + assert_eq!(cfg.sandbox_token_ttl(), None); + + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert!(serialized.get("ttl_secs").is_none()); + } + + #[test] + fn gateway_jwt_positive_ttl_serializes_and_has_effective_duration() { + let cfg: GatewayJwtConfig = serde_json::from_value(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 3600 + })) + .expect("gateway JWT config should deserialize with positive ttl"); + + assert_eq!(cfg.sandbox_token_ttl(), Some(Duration::from_secs(3600))); + let serialized = serde_json::to_value(&cfg).expect("gateway JWT config serializes"); + assert_eq!(serialized["ttl_secs"], 3600); + } + + #[test] + fn gateway_jwt_ttl_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "signing_key_path": "/tmp/signing.pem", + "public_key_path": "/tmp/public.pem", + "kid_path": "/tmp/kid", + "ttl_secs": 0 + })) + .expect_err("zero TTL must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); + } + + #[test] + fn image_pull_policy_uses_canonical_vocabulary() { + for (value, expected) in [ + ("always", ImagePullPolicy::Always), + ("if_not_present", ImagePullPolicy::IfNotPresent), + ("never", ImagePullPolicy::Never), + ("newer", ImagePullPolicy::Newer), + ] { + assert_eq!(value.parse::(), Ok(expected)); + assert_eq!(expected.to_string(), value); + assert_eq!(serde_json::to_value(expected).unwrap(), value); + } + assert!("missing".parse::().is_err()); + assert!("IfNotPresent".parse::().is_err()); + } + + #[test] + fn app_armor_profiles_round_trip_and_translate_for_each_backend() { + for (value, expected, kubernetes_type, localhost_profile, oci_security_opt) in [ + ( + "RuntimeDefault", + AppArmorProfile::RuntimeDefault, + "RuntimeDefault", + None, + None, + ), + ( + "Unconfined", + AppArmorProfile::Unconfined, + "Unconfined", + None, + Some("apparmor=unconfined"), + ), + ( + "Localhost/openshell-supervisor", + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "Localhost", + Some("openshell-supervisor"), + Some("apparmor=openshell-supervisor"), + ), + ] { + let parsed = value.parse::().expect("valid profile"); + assert_eq!(parsed, expected); + assert_eq!(parsed.to_string(), value); + assert_eq!(parsed.kubernetes_type(), kubernetes_type); + assert_eq!(parsed.localhost_profile(), localhost_profile); + assert_eq!(parsed.oci_security_opt().as_deref(), oci_security_opt); + + let json = serde_json::to_value(&parsed).expect("profile serializes"); + assert_eq!(json, value); + assert_eq!( + serde_json::from_value::(json).expect("profile deserializes"), + parsed + ); + } + + for invalid in [ + "Localhost/", + "Localhost/openshell profile", + "runtimeDefault", + "unconfined", + "Unknown", + ] { + assert!( + invalid.parse::().is_err(), + "{invalid} should be rejected" + ); + } + } + + #[test] + fn upstream_proxy_validation_enforces_cross_field_contract() { + let auth_file = Some("/run/secrets/proxy-auth".into()); + let cases = [ + ("default", UpstreamProxyConfig::default(), None), + ( + "https auth", + UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + proxy_auth_file: auth_file.clone(), + ..Default::default() + }, + None, + ), + ( + "acknowledged http auth", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_file: auth_file.clone(), + proxy_auth_allow_insecure: Some(true), + ..Default::default() + }, + None, + ), + ( + "unacknowledged http auth", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_file: auth_file.clone(), + ..Default::default() + }, + Some("proxy_auth_allow_insecure"), + ), + ( + "no_proxy without proxy", + UpstreamProxyConfig { + no_proxy: Some("localhost".to_string()), + ..Default::default() + }, + Some("no_proxy"), + ), + ( + "blank no_proxy", + UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(" ".to_string()), + ..Default::default() + }, + Some("no_proxy"), + ), + ( + "auth file without proxy", + UpstreamProxyConfig { + proxy_auth_file: auth_file, + ..Default::default() + }, + Some("proxy_auth_file"), + ), + ( + "ack without auth file", + UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + proxy_auth_allow_insecure: Some(true), + ..Default::default() + }, + Some("proxy_auth_allow_insecure"), + ), + ( + "hostname mode without proxy", + UpstreamProxyConfig { + proxy_connect_by_hostname: Some(true), + ..Default::default() + }, + Some("proxy_connect_by_hostname"), + ), + ( + "empty proxy", + UpstreamProxyConfig { + https_proxy: Some(String::new()), + ..Default::default() + }, + Some("https_proxy"), + ), + ( + "inline credentials", + UpstreamProxyConfig { + https_proxy: Some( + "https://secret-user:secret-password@proxy.example:8443".to_string(), + ), + ..Default::default() + }, + Some("must not embed credentials"), + ), + ]; + + for (name, config, expected_error) in cases { + match expected_error { + None => config + .validate() + .unwrap_or_else(|error| panic!("{name}: {error}")), + Some(expected) => { + let error = match config.validate() { + Ok(()) => panic!("{name} should fail validation"), + Err(error) => error, + }; + assert!(error.contains(expected), "{name}: {error}"); + assert!(!error.contains("secret-user"), "{name}: {error}"); + assert!(!error.contains("secret-password"), "{name}: {error}"); + } + } + } + } + + #[test] + fn config_defaults_and_builder_use_singular_compute_driver() { + let config = Config::new(None); + assert_eq!(config.compute_driver, None); + assert_eq!( + Config::new(None) + .with_compute_driver("podman") + .compute_driver + .as_deref(), + Some("podman") + ); + } + + #[test] + fn typed_sandbox_pids_default_matches_positive_constant() { + assert_eq!( + default_sandbox_pids_limit().map(std::num::NonZeroI64::get), + Some(super::DEFAULT_SANDBOX_PIDS_LIMIT) + ); } #[test] diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index e26ea53f7a..63511c13ff 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -74,12 +74,11 @@ pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest" /// secrets, so this is the same delivery the per-sandbox JWT already uses. pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy"; -/// Guest path for the corporate proxy CA bundle in VM sandboxes. +/// Guest path for the corporate proxy CA bundle staged by the VM driver. /// -/// A CA certificate is not secret, so unlike the credential this is staged -/// world-readable. The supervisor trusts it for the handshake with an -/// `https://` proxy and for server certificates re-signed by a -/// TLS-intercepting proxy. +/// The bundle is operator-owned but not secret. The driver validates it and +/// writes it into each sandbox overlay with mode `0644`, then passes only this +/// guest path to the supervisor. pub const VM_GUEST_PROXY_CA_PATH: &str = "/opt/openshell/tls/proxy-ca.pem"; /// Guest path for the driver-authored supervisor argument list in VM sandboxes. @@ -127,10 +126,10 @@ mod tests { VM_GUEST_TLS_CERT_PATH, VM_GUEST_TLS_KEY_PATH, VM_GUEST_SANDBOX_TOKEN_PATH, - VM_GUEST_INIT_DROPIN_DIR, - VM_GUEST_INIT_DROPIN_MANIFEST, VM_GUEST_UPSTREAM_PROXY_AUTH_PATH, VM_GUEST_PROXY_CA_PATH, + VM_GUEST_INIT_DROPIN_DIR, + VM_GUEST_INIT_DROPIN_MANIFEST, VM_GUEST_SUPERVISOR_ARGS_PATH, VM_UMOCI_PATH, VM_SANDBOX_OWNER_NORMALIZED_MARKER, diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 74871751da..7182f811d5 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -7,6 +7,58 @@ use std::path::{Path, PathBuf}; use crate::proto::compute::v1::DriverSandbox; +/// Built-in sandbox network topologies used to derive a callback endpoint +/// when an operator does not configure a per-driver `grpc_endpoint` override. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GatewayCallbackTopology { + /// A Docker container reaches the host through Docker's gateway alias. + Docker, + /// A Podman container reaches the host through Podman's gateway alias. + Podman, + /// A libkrun guest reaches the host through gvproxy's gateway alias. + Vm, +} + +/// Build the endpoint a sandbox uses to call its gateway for a known topology. +/// +/// The result is deliberately derived by the gateway rather than baked into +/// individual driver defaults. A configured `grpc_endpoint` remains an +/// operator override for remote or non-standard deployments. +#[must_use] +pub fn gateway_callback_endpoint( + topology: GatewayCallbackTopology, + gateway_port: u16, + gateway_tls_enabled: bool, +) -> String { + let scheme = if gateway_tls_enabled { "https" } else { "http" }; + let host = match topology { + GatewayCallbackTopology::Docker | GatewayCallbackTopology::Vm => "host.openshell.internal", + GatewayCallbackTopology::Podman => "host.containers.internal", + }; + format!("{scheme}://{host}:{gateway_port}") +} + +#[cfg(test)] +mod callback_endpoint_tests { + use super::{GatewayCallbackTopology, gateway_callback_endpoint}; + + #[test] + fn derives_endpoint_for_each_builtin_topology() { + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Docker, 17670, false), + "http://host.openshell.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Podman, 17670, true), + "https://host.containers.internal:17670" + ); + assert_eq!( + gateway_callback_endpoint(GatewayCallbackTopology::Vm, 17670, true), + "https://host.openshell.internal:17670" + ); + } +} + // --------------------------------------------------------------------------- // Sandbox container/pod label keys (openshell.ai/ namespace) // --------------------------------------------------------------------------- @@ -663,6 +715,68 @@ pub fn validate_upstream_proxy_settings( /// Container-side directory where the provider SPIFFE Workload API socket is mounted. pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workload-api"; +/// Validate a host UNIX socket selected for provider SPIFFE projection. +/// +/// Local container drivers bind-mount the socket's dedicated parent directory, +/// not a broad host root. TCP endpoints are deliberately rejected here: a +/// container projection must be a filesystem socket, while VM guest TCP +/// exposure has its own explicit acknowledgement contract. +pub fn validate_provider_spiffe_unix_socket(path: &Path) -> Result<(), String> { + let raw = path + .to_str() + .ok_or_else(|| "provider_spiffe_workload_api_socket must be valid UTF-8".to_string())?; + if raw.trim() != raw || raw.is_empty() { + return Err("provider_spiffe_workload_api_socket must not be empty or contain surrounding whitespace".to_string()); + } + if raw.starts_with("tcp:") || raw.starts_with("unix:") { + return Err("provider_spiffe_workload_api_socket must be an absolute host UNIX socket path, not a URI".to_string()); + } + if !path.is_absolute() || path.parent().is_none_or(|parent| parent == Path::new("/")) { + return Err("provider_spiffe_workload_api_socket must be an absolute UNIX socket path below a dedicated parent directory".to_string()); + } + Ok(()) +} + +/// Return the guest/container path for a projected provider SPIFFE socket. +pub fn projected_provider_spiffe_socket_path(path: &Path) -> Result { + validate_provider_spiffe_unix_socket(path)?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .ok_or_else(|| "provider_spiffe_workload_api_socket must name a socket file".to_string())?; + Ok(format!( + "{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}/{file_name}" + )) +} + +/// Validate an explicitly operator-acknowledged guest-reachable SPIFFE TCP endpoint. +/// +/// The `tcp:` spelling is the SPIFFE Workload API endpoint grammar accepted by +/// the client. The address must be concrete; wildcard and host-only UNIX +/// sockets are never silently exposed to VM guests. +pub fn validate_guest_spiffe_tcp_endpoint( + endpoint: &str, + acknowledged: bool, +) -> Result<(), String> { + if endpoint.trim() != endpoint || endpoint.is_empty() { + return Err("provider_spiffe_workload_api_tcp_endpoint must not be empty or contain surrounding whitespace".to_string()); + } + if !acknowledged { + return Err("provider_spiffe_workload_api_tcp_endpoint exposes a Workload API to VM guests; set provider_spiffe_allow_guest_tcp = true only after explicitly acknowledging that exposure".to_string()); + } + let address = endpoint.strip_prefix("tcp:").ok_or_else(|| { + "provider_spiffe_workload_api_tcp_endpoint must use tcp:host:port (for example tcp:192.0.2.10:8081)".to_string() + })?; + let address: std::net::SocketAddr = address.parse().map_err(|_| { + "provider_spiffe_workload_api_tcp_endpoint must use a concrete IP address and non-zero port".to_string() + })?; + if address.ip().is_unspecified() || address.port() == 0 { + return Err("provider_spiffe_workload_api_tcp_endpoint must not use an unspecified address or port 0".to_string()); + } + Ok(()) +} + /// Return the XDG state path for a driver's sandbox JWT token file. /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. @@ -1101,6 +1215,28 @@ mod tests { } #[cfg(unix)] + #[test] + fn projected_spiffe_socket_requires_dedicated_absolute_unix_path() { + assert_eq!( + projected_provider_spiffe_socket_path(Path::new("/run/spire/agent.sock")).unwrap(), + "/spiffe-workload-api/agent.sock" + ); + for path in ["relative.sock", "/agent.sock", "tcp:127.0.0.1:8081"] { + assert!( + validate_provider_spiffe_unix_socket(Path::new(path)).is_err(), + "{path}" + ); + } + } + + #[test] + fn guest_spiffe_tcp_requires_acknowledgement_and_concrete_endpoint() { + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", true).is_ok()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:192.0.2.10:8081", false).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("tcp:0.0.0.0:8081", true).is_err()); + assert!(validate_guest_spiffe_tcp_endpoint("unix:/run/spire/agent.sock", true).is_err()); + } + #[test] fn credential_file_rejects_fifo_without_hanging() { // A FIFO with no writer would block a blocking open() forever. The diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 85195f2040..32d43b1c10 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -53,10 +53,11 @@ pub mod time; pub mod transport_errors; pub use config::{ - Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, - GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, - GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, - PolicyValidationFailureMode, TlsConfig, + AppArmorProfile, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, + GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, + GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, + ImagePullPolicy, MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + UpstreamProxyConfig, }; pub use dynamic_string_allowlist::DynamicStringAllowlist; pub use error::{ComputeDriverError, Error, Result}; diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..10e085ecf0 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -104,7 +104,7 @@ contract: | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | +| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. `[openshell.drivers.docker].sandbox_pids_limit` defaults to `2048`; explicit `0` is invalid. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | | `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | @@ -187,6 +187,25 @@ mounted into the container and exposed with: HTTP endpoints reject TLS material because the supervisor would not use it. +## Corporate proxy, SPIFFE, and AppArmor + +`https_proxy`, `no_proxy`, and `proxy_auth_file` in +`[openshell.drivers.docker]` are operator-owned supervisor settings. Docker +passes the proxy URL and bypass list on the supervisor command line and mounts +an optional `user:pass` auth file read-only at a root-only path. Credentials +never appear in container environment or Docker labels. An auth file used with +an `http://` proxy requires `proxy_auth_allow_insecure = true`; an `https://` +proxy protects the Basic-auth header in its TLS session. + +Set `provider_spiffe_workload_api_socket` to an absolute host UNIX socket to +project its dedicated parent directory into the supervisor and set +`OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to the guest path. TCP URIs are +rejected for this projection. `app_armor_profile` uses the shared +`RuntimeDefault`, `Unconfined`, or `Localhost/` vocabulary. Docker +uses explicit `Unconfined` by default because the supervisor's namespace mount +setup is incompatible with `docker-default`; requested confined profiles fail +at startup if Docker does not report AppArmor support. + ## Environment Ownership The driver merges template environment and sandbox spec environment first, then diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index df5aee4750..4d9051228b 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -21,13 +21,14 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ - CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, - LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + CONDITION_EXITED, CONDITION_RUNTIME_RESTART, GatewayCallbackTopology, LABEL_MANAGED_BY, + LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, + LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -55,7 +56,9 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Error, Result as CoreResult}; +use openshell_core::{ + AppArmorProfile, Error, ImagePullPolicy, Result as CoreResult, UpstreamProxyConfig, +}; use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -80,6 +83,10 @@ const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = + openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; +const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = + openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; @@ -120,10 +127,10 @@ pub struct DockerComputeConfig { pub default_image: String, /// Image pull policy for sandbox images. - pub image_pull_policy: String, + pub image_pull_policy: ImagePullPolicy, - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, + /// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes. + pub sandbox_label: String, /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, @@ -156,13 +163,61 @@ pub struct DockerComputeConfig { /// Container cgroup PID limit for Docker-managed sandboxes. /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] pub enable_bind_mounts: bool, + + /// Corporate forward-proxy settings supplied to the supervisor on argv. + /// The flattened fields retain the common `https_proxy`, `no_proxy`, and + /// `proxy_auth_*` gateway TOML contract. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Host UNIX socket to project into sandbox supervisors for provider + /// SPIFFE token exchange. + pub provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` confinement requested for sandbox containers. The explicit + /// default preserves the prior supervisor-compatible Docker behavior. + #[serde(skip_serializing_if = "Option::is_none")] + pub app_armor_profile: Option, +} + +impl DockerComputeConfig { + /// Validate startup configuration without connecting to Docker. + pub fn validate_configuration(&self, gateway_bind_address: SocketAddr) -> CoreResult<()> { + if let Some(socket_path) = self.socket_path.as_deref() + && socket_path.to_str().is_none() + { + return Err(Error::config(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + ))); + } + validate_sandbox_pids_limit(self.sandbox_pids_limit)?; + validate_image_pull_policy(self.image_pull_policy)?; + self.upstream_proxy.validate().map_err(Error::config)?; + if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(Error::config)?; + } + parse_optional_host_gateway_ip(&self.host_gateway_ip)?; + if gateway_bind_address.port() == 0 { + return Err(Error::config( + "docker compute driver requires a fixed non-zero gateway bind port", + )); + } + Ok(()) + } } impl Default for DockerComputeConfig { @@ -170,8 +225,8 @@ impl Default for DockerComputeConfig { Self { socket_path: None, default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + image_pull_policy: ImagePullPolicy::default(), + sandbox_label: "default".to_string(), grpc_endpoint: String::new(), supervisor_bin: None, supervisor_image: None, @@ -181,8 +236,11 @@ impl Default for DockerComputeConfig { network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } } @@ -197,8 +255,8 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { default_image: String, - image_pull_policy: String, - sandbox_namespace: String, + image_pull_policy: ImagePullPolicy, + sandbox_label: String, grpc_endpoint: String, network_name: String, gateway_route: DockerGatewayRoute, @@ -210,8 +268,11 @@ struct DockerDriverRuntimeConfig { guest_tls: Option, daemon_version: String, gpu: DockerGpuRuntimeCapabilities, - sandbox_pids_limit: i64, + sandbox_pids_limit: Option, enable_bind_mounts: bool, + upstream_proxy: UpstreamProxyConfig, + provider_spiffe_workload_api_socket: Option, + app_armor_profile: Option, } #[derive(Debug, Clone, Copy)] @@ -481,6 +542,7 @@ impl DockerComputeDriver { gateway_log_level: &str, docker_config: &DockerComputeConfig, ) -> CoreResult { + docker_config.validate_configuration(gateway_bind_address)?; let socket_path = docker_config .socket_path .clone() @@ -513,13 +575,9 @@ impl DockerComputeDriver { cdi_supported, wsl_all_gpu_fallback_enabled, }; - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; + validate_docker_proxy_auth_file(&docker_config.upstream_proxy)?; + validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; let gateway_port = gateway_bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } let network_name = docker_network_name(docker_config); let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; @@ -529,13 +587,11 @@ impl DockerComputeDriver { docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); + docker_config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Docker, + gateway_port, + docker_guest_tls_configured(&docker_config), + ); } let grpc_endpoint = docker_container_openshell_endpoint( &docker_config.grpc_endpoint, @@ -550,8 +606,8 @@ impl DockerComputeDriver { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), + image_pull_policy: docker_config.image_pull_policy, + sandbox_label: docker_config.sandbox_label.clone(), grpc_endpoint, network_name, gateway_route, @@ -565,6 +621,11 @@ impl DockerComputeDriver { gpu, sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, + upstream_proxy: docker_config.upstream_proxy.clone(), + provider_spiffe_workload_api_socket: docker_config + .provider_spiffe_workload_api_socket + .clone(), + app_armor_profile: docker_config.app_armor_profile.clone(), }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -870,7 +931,7 @@ impl DockerComputeDriver { ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, )); @@ -1261,7 +1322,7 @@ impl DockerComputeDriver { PendingSandboxRecord { sandbox: pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, provisioning_condition(), false, ), @@ -1316,7 +1377,7 @@ impl DockerComputeDriver { cleanup_sandbox_token_file(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, - &self.config.sandbox_namespace, + &self.config.sandbox_label, error_condition(failure.reason, &failure.message), false, ); @@ -1522,7 +1583,7 @@ impl DockerComputeDriver { } async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); + let filters = managed_container_label_filters(&self.config.sandbox_label, []); self.docker .list_containers(Some( ListContainersOptionsBuilder::default() @@ -1547,7 +1608,7 @@ impl DockerComputeDriver { } let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); + managed_container_label_filters(&self.config.sandbox_label, label_filter_values); let containers = self .docker .list_containers(Some( @@ -1565,7 +1626,7 @@ impl DockerComputeDriver { }; let namespace_matches = labels .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); + .is_some_and(|value| value == &self.config.sandbox_label); let id_matches = sandbox_id.is_empty() || labels .get(LABEL_SANDBOX_ID) @@ -1583,9 +1644,8 @@ impl DockerComputeDriver { sandbox_id: &str, image: &str, ) -> Result { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - let inspect = match policy.as_str() { - "" | "ifnotpresent" => { + let inspect = match self.config.image_pull_policy { + ImagePullPolicy::IfNotPresent => { if let Ok(inspect) = self.docker.inspect_image(image).await { self.publish_docker_progress( sandbox_id, @@ -1602,14 +1662,14 @@ impl DockerComputeDriver { .map_err(|err| internal_status("inspect Docker image after pull", err))? } } - "always" => { + ImagePullPolicy::Always => { self.pull_image(sandbox_id, image).await?; self.docker .inspect_image(image) .await .map_err(|err| internal_status("inspect Docker image after pull", err))? } - "never" => match self.docker.inspect_image(image).await { + ImagePullPolicy::Never => match self.docker.inspect_image(image).await { Ok(inspect) => { self.publish_docker_progress( sandbox_id, @@ -1621,15 +1681,15 @@ impl DockerComputeDriver { } Err(err) if is_not_found_error(&err) => { return Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" + "docker image '{image}' is not present locally and image_pull_policy = \"never\"" ))); } Err(err) => return Err(internal_status("inspect Docker image", err)), }, - other => { - return Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))); + ImagePullPolicy::Newer => { + return Err(Status::failed_precondition( + "image_pull_policy = \"newer\" is supported only by the Podman compute driver", + )); } }; @@ -2608,6 +2668,49 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } +/// Verify the configured credential without exposing its contents. Docker +/// bind-mounts the root-owned file directly, unlike Podman which uses a native +/// secret object; this preflight makes a bad file fail before any sandbox is +/// created. +fn validate_docker_proxy_auth_file(config: &UpstreamProxyConfig) -> CoreResult<()> { + let Some(path) = config.proxy_auth_file.as_ref() else { + return Ok(()); + }; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file( + path.to_str() + .ok_or_else(|| Error::config("proxy_auth_file must be valid UTF-8"))?, + ) + .map_err(Error::config)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map_err(|error| Error::config(format!("proxy_auth_file is invalid: {error}")))?; + Ok(()) +} + +/// Build immutable operator-owned proxy arguments. Credentials never appear on +/// argv: only the fixed in-container root-only file path is supplied. +fn docker_upstream_proxy_cli_args(config: &UpstreamProxyConfig) -> Vec { + let mut args = Vec::new(); + if let Some(url) = config.https_proxy.as_ref() { + args.extend(["--upstream-proxy".to_string(), url.clone()]); + } + if let Some(no_proxy) = config.no_proxy.as_ref() { + args.extend(["--upstream-no-proxy".to_string(), no_proxy.clone()]); + } + if config.proxy_auth_file.is_some() { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if config.proxy_auth_allow_insecure == Some(true) { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if config.proxy_connect_by_hostname == Some(true) { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args +} + fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2637,6 +2740,23 @@ fn build_binds( SANDBOX_TOKEN_MOUNT_PATH )); } + if let Some(path) = config.upstream_proxy.proxy_auth_file.as_ref() { + binds.push(format!( + "{}:{}:ro,z", + path.display(), + UPSTREAM_PROXY_AUTH_MOUNT_PATH + )); + } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { + let parent = socket.parent().ok_or_else(|| { + Status::failed_precondition("provider SPIFFE socket has no parent directory") + })?; + binds.push(format!( + "{}:{}:ro", + parent.display(), + PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + )); + } Ok(binds) } @@ -2653,7 +2773,7 @@ fn sandbox_token_host_path_by_id( ) -> Result { openshell_core::driver_utils::sandbox_token_path( "docker-sandbox-tokens", - Some(&config.sandbox_namespace), + Some(&config.sandbox_label), sandbox_id, ) .map_err(|err| { @@ -2819,6 +2939,15 @@ fn build_environment_for_oci_user( TLS_KEY_MOUNT_PATH.to_string(), ); } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() + && let Ok(path) = + openshell_core::driver_utils::projected_provider_spiffe_socket_path(socket) + { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + path, + ); + } environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); @@ -3010,13 +3139,13 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_WORKSPACE.to_string(), sandbox.workspace.clone(), ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use + // The list/get/find paths filter by `config.sandbox_label`, so use // the same value here. `DriverSandbox.namespace` is unset on the request // path (the gateway elides it), and using it would produce containers // that the driver itself cannot find afterwards. labels.insert( LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), + config.sandbox_label.clone(), ); Ok(ContainerCreateBody { @@ -3029,7 +3158,11 @@ fn build_container_create_body_for_image( entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + cmd: { + let mut args = vec!["--workdir".to_string(), workspace_root]; + args.extend(docker_upstream_proxy_cli_args(&config.upstream_proxy)); + Some(args) + }, labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3051,17 +3184,13 @@ fn build_container_create_body_for_image( "SYS_PTRACE".to_string(), "SYSLOG".to_string(), ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), + // The default is explicitly Unconfined because the supervisor + // needs mount operations commonly denied by docker-default. + security_opt: config + .app_armor_profile + .as_ref() + .and_then(AppArmorProfile::oci_security_opt) + .map(|option| vec![option]), network_mode: Some(config.network_name.clone()), extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), ..Default::default() @@ -3346,26 +3475,55 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { +fn validate_sandbox_pids_limit(value: Option) -> CoreResult<()> { + if value.is_some_and(|limit| limit.get() < 0) { return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", + "docker sandbox_pids_limit must be positive when set", )); } Ok(()) } -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", +fn validate_image_pull_policy(policy: ImagePullPolicy) -> CoreResult<()> { + if policy == ImagePullPolicy::Newer { + return Err(Error::config( + "docker image_pull_policy = \"newer\" is supported only by the Podman compute driver", )); } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) + Ok(()) +} + +fn validate_docker_app_armor_profile( + profile: Option<&AppArmorProfile>, + info: &SystemInfo, +) -> CoreResult<()> { + let requires_apparmor = matches!( + profile, + Some(AppArmorProfile::RuntimeDefault | AppArmorProfile::Localhost(_)) + ); + if !requires_apparmor { + return Ok(()); + } + let available = info.security_options.as_ref().is_some_and(|options| { + options + .iter() + .any(|option| option.to_ascii_lowercase().contains("apparmor")) + }); + if !available { + return Err(Error::config( + "app_armor_profile requires AppArmor, but Docker reports it is unavailable; enable AppArmor on the daemon host or set app_armor_profile = \"Unconfined\" explicitly", + )); + } + Ok(()) +} + +fn docker_pids_limit(value: Option) -> Result, Status> { + if value.is_some_and(|limit| limit.get() < 0) { + return Err(Status::failed_precondition( + "docker sandbox_pids_limit must be positive when set", + )); } + Ok(value.map(std::num::NonZeroI64::get)) } #[allow(clippy::cast_possible_truncation)] @@ -3618,12 +3776,12 @@ fn label_filters(values: impl IntoIterator) -> HashMap, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), + format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_label}"), ]; values.extend(extra_values); label_filters(values) @@ -3987,8 +4145,8 @@ fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult bool { docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() + || docker_config.guest_tls_cert.is_some() + || docker_config.guest_tls_key.is_some() } pub(crate) fn docker_guest_tls_paths( diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index aff5ba17f8..6137ca1c4a 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -95,8 +95,8 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), + image_pull_policy: ImagePullPolicy::IfNotPresent, + sandbox_label: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { @@ -124,11 +124,161 @@ fn runtime_config() -> DockerDriverRuntimeConfig { cdi_supported: false, wsl_all_gpu_fallback_enabled: false, }, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: None, enable_bind_mounts: false, + upstream_proxy: UpstreamProxyConfig::default(), + provider_spiffe_workload_api_socket: None, + app_armor_profile: Some(AppArmorProfile::Unconfined), } } +#[test] +fn docker_config_uses_canonical_sandbox_label_name() { + let config: DockerComputeConfig = + serde_json::from_value(serde_json::json!({ "sandbox_label": "tenant-a" })).unwrap(); + assert_eq!(config.sandbox_label, "tenant-a"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["sandbox_label"], "tenant-a"); + assert!(serialized.get("sandbox_namespace").is_none()); +} + +#[test] +fn docker_config_rejects_legacy_sandbox_namespace() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_namespace": "tenant-a" + })) + .expect_err("legacy sandbox_namespace must be rejected"); + assert!(error.to_string().contains("sandbox_namespace")); +} + +#[test] +fn docker_config_keeps_explicit_unconfined_apparmor_default() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!(config.app_armor_profile, Some(AppArmorProfile::Unconfined)); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], "Unconfined"); +} + +#[test] +fn docker_config_defaults_to_driver_owned_pids_limit() { + let config: DockerComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Docker config should deserialize"); + assert_eq!( + config.sandbox_pids_limit.map(std::num::NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); +} + +#[test] +fn docker_config_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: DockerComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = validate_sandbox_pids_limit(negative.sandbox_pids_limit).unwrap_err(); + assert!(error.to_string().contains("must be positive")); +} + +#[test] +fn docker_rejects_newer_image_pull_policy() { + let error = validate_image_pull_policy(ImagePullPolicy::Newer).unwrap_err(); + assert!(error.to_string().contains("supported only by the Podman")); +} + +#[test] +fn docker_apparmor_profiles_render_and_require_daemon_capability() { + for (profile, expected) in [ + (AppArmorProfile::RuntimeDefault, None), + ( + AppArmorProfile::Unconfined, + Some(vec!["apparmor=unconfined".to_string()]), + ), + ( + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + Some(vec!["apparmor=openshell-supervisor".to_string()]), + ), + ] { + let mut config = runtime_config(); + config.app_armor_profile = Some(profile.clone()); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + assert_eq!(body.host_config.unwrap().security_opt, expected); + } + + let unavailable = SystemInfo::default(); + assert!( + validate_docker_app_armor_profile(Some(&AppArmorProfile::Unconfined), &unavailable).is_ok() + ); + for confined in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + let error = validate_docker_app_armor_profile(Some(&confined), &unavailable) + .expect_err("confined profile requires daemon AppArmor support"); + assert!( + error + .to_string() + .contains("Docker reports it is unavailable") + ); + } + + let available = SystemInfo { + security_options: Some(vec!["name=apparmor".to_string()]), + ..Default::default() + }; + assert!( + validate_docker_app_armor_profile( + Some(&AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )), + &available + ) + .is_ok() + ); +} + +#[test] +fn docker_config_uses_shared_proxy_contract_and_explicit_apparmor_default() { + let config: DockerComputeConfig = toml::from_str( + r#" +https_proxy = "http://proxy.example:8080" +no_proxy = ".svc" +proxy_auth_file = "/run/secrets/proxy-auth" +proxy_auth_allow_insecure = true +app_armor_profile = "Localhost/openshell-supervisor" +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +"#, + ) + .unwrap(); + assert_eq!( + config.upstream_proxy.https_proxy.as_deref(), + Some("http://proxy.example:8080") + ); + assert_eq!( + config.app_armor_profile, + Some(AppArmorProfile::Localhost( + "openshell-supervisor".to_string() + )) + ); + assert!(config.upstream_proxy.validate().is_ok()); + assert!( + openshell_core::driver_utils::validate_provider_spiffe_unix_socket( + config + .provider_spiffe_workload_api_socket + .as_deref() + .unwrap() + ) + .is_ok() + ); +} + fn json_struct(value: serde_json::Value) -> prost_types::Struct { let serde_json::Value::Object(object) = value else { panic!("expected JSON object"); @@ -553,7 +703,7 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { .build(); let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let mut config = runtime_config(); - config.image_pull_policy = "unsupported".to_string(); + config.image_pull_policy = ImagePullPolicy::Newer; let driver = test_driver_with_config(config); async { @@ -1207,13 +1357,13 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { +fn docker_pids_limit_uses_runtime_default_when_omitted() { assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) + docker_pids_limit(std::num::NonZeroI64::new(2048)).unwrap(), + Some(2048) ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); + assert_eq!(docker_pids_limit(None).unwrap(), None); + assert!(docker_pids_limit(std::num::NonZeroI64::new(-1)).is_err()); } #[test] @@ -1223,10 +1373,21 @@ fn docker_compute_config_disables_bind_mounts_by_default() { } #[test] -fn container_create_body_sets_driver_owned_pids_limit() { +fn container_create_body_omits_pids_limit_by_default() { let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); + assert_eq!(host_config.pids_limit, None); +} + +#[test] +fn container_create_body_emits_configured_positive_pids_limit() { + let mut config = runtime_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(4096); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + assert_eq!( + body.host_config.expect("host config").pids_limit, + Some(4096) + ); } #[test] @@ -2162,6 +2323,51 @@ fn build_environment_uses_token_file_without_raw_token_env() { ))); } +#[test] +fn docker_container_projects_proxy_and_spiffe_without_credential_metadata() { + let mut config = runtime_config(); + config.upstream_proxy = UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }; + config.provider_spiffe_workload_api_socket = Some(PathBuf::from("/run/spire/agent.sock")); + let body = build_container_create_body(&test_sandbox(), &config).unwrap(); + let command = body.cmd.unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy", "https://proxy.example:8443"]) + ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-proxy-auth-file", UPSTREAM_PROXY_AUTH_MOUNT_PATH]) + ); + assert!( + command + .windows(2) + .any(|args| args == ["--upstream-no-proxy", ".svc"]) + ); + assert!(command.contains(&"--upstream-proxy-connect-by-hostname".to_string())); + let binds = body.host_config.unwrap().binds.unwrap(); + assert!( + binds + .iter() + .any(|bind| bind.contains(UPSTREAM_PROXY_AUTH_MOUNT_PATH)) + ); + assert!(binds.contains(&format!( + "/run/spire:{PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR}:ro" + ))); + assert!(binds.iter().all(|bind| !bind.contains("rbind"))); + let env = body.env.unwrap(); + assert!(env.iter().any(|entry| entry + == "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/spiffe-workload-api/agent.sock")); + assert!(!env.iter().any(|entry| entry.contains("proxy-auth"))); +} + #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2693,10 +2899,10 @@ fn build_container_create_body_uses_runtime_namespace_label() { // runtime config, not from `DriverSandbox.namespace`. The gateway // does not populate `DriverSandbox.namespace`, so a container created // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_namespace`), leaking + // queries (which filter on `config.sandbox_label`), leaking // sandboxes that the driver itself cannot observe. let mut config = runtime_config(); - config.sandbox_namespace = "tenant-a".to_string(); + config.sandbox_label = "tenant-a".to_string(); let mut sandbox = test_sandbox(); sandbox.namespace = "ignored-by-driver".to_string(); @@ -3017,6 +3223,32 @@ fn docker_guest_tls_paths_allows_plain_http_without_tls_flags() { assert!(result.is_none()); } +#[test] +fn docker_automatic_tls_detection_is_fail_closed_for_partial_bundles() { + for mask in 0_u8..8 { + let config = DockerComputeConfig { + guest_tls_ca: (mask & 1 != 0).then(|| PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: (mask & 2 != 0).then(|| PathBuf::from("/tmp/cert.pem")), + guest_tls_key: (mask & 4 != 0).then(|| PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + assert_eq!( + docker_guest_tls_configured(&config), + mask != 0, + "TLS presence mask {mask:03b}" + ); + + if mask != 0 && mask != 7 { + let mut inferred = config; + inferred.grpc_endpoint = "https://host.openshell.internal:8080".to_string(); + assert!( + docker_guest_tls_paths(&inferred).is_err(), + "partial TLS presence mask {mask:03b} must fail" + ); + } + } +} + #[test] fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { let image = openshell_core::config::default_supervisor_image(); diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..b64bf0c6e4 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -96,8 +96,11 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. Driver-owned -values must override image-provided environment variables. +material, and the supervisor SSH socket path into the workload. The callback +endpoint is required because the sandbox namespace does not identify the +Gateway Service; Helm renders it from the release topology, while standalone +and raw TOML configurations must set it explicitly. Driver-owned values must +override image-provided environment variables. Sandbox pods run as `service_account_name` and keep `automountServiceAccountToken: false`. The only Kubernetes token exposed to the @@ -152,14 +155,13 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. -The driver can request a Kubernetes AppArmor profile through -`app_armor_profile`. - +The driver uses the shared AppArmor model through `app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`. An empty or unset value omits -`securityContext.appArmorProfile`. Helm deployments default sandbox agent -containers to `Unconfined` because runtime/default AppArmor profiles can block -the supervisor's network namespace mount setup on AppArmor-enabled nodes. +`Localhost/`; an empty or unset value omits +`securityContext.appArmorProfile`. Docker and Podman translate the same values +to OCI security options. Helm deployments default sandbox agent containers to +`Unconfined` because runtime/default AppArmor profiles can block the +supervisor's network namespace mount setup on AppArmor-enabled nodes. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 805c0314b0..d8a7b063c7 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,8 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub use openshell_core::AppArmorProfile; pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; -use openshell_core::config; +use openshell_core::{ImagePullPolicy, config}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; #[cfg(test)] @@ -177,83 +178,6 @@ impl KubernetesSidecarConfig { } } -/// Kubernetes `AppArmor` profile requested for the sandbox agent container. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AppArmorProfile { - RuntimeDefault, - Unconfined, - Localhost(String), -} - -impl AppArmorProfile { - #[must_use] - pub fn to_k8s_type(&self) -> &'static str { - match self { - Self::RuntimeDefault => "RuntimeDefault", - Self::Unconfined => "Unconfined", - Self::Localhost(_) => "Localhost", - } - } - - #[must_use] - pub fn localhost_profile(&self) -> Option<&str> { - match self { - Self::Localhost(profile) => Some(profile), - Self::RuntimeDefault | Self::Unconfined => None, - } - } -} - -impl std::fmt::Display for AppArmorProfile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RuntimeDefault => f.write_str("RuntimeDefault"), - Self::Unconfined => f.write_str("Unconfined"), - Self::Localhost(profile) => write!(f, "Localhost/{profile}"), - } - } -} - -impl FromStr for AppArmorProfile { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "RuntimeDefault" => Ok(Self::RuntimeDefault), - "Unconfined" => Ok(Self::Unconfined), - other => match other.strip_prefix("Localhost/") { - Some("") => Err( - "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" - .to_string(), - ), - Some(profile) => Ok(Self::Localhost(profile.to_string())), - None => Err(format!( - "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" - )), - }, - } - } -} - -impl Serialize for AppArmorProfile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for AppArmorProfile { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::from_str(&value).map_err(serde::de::Error::custom) - } -} - fn deserialize_optional_app_armor_profile<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -263,7 +187,8 @@ where let value = Option::::deserialize(deserializer)?; match value.as_deref() { None | Some("") => Ok(None), - Some(value) => AppArmorProfile::from_str(value) + Some(value) => value + .parse::() .map(Some) .map_err(serde::de::Error::custom), } @@ -306,7 +231,9 @@ pub struct KubernetesComputeConfig { /// the driver's `TokenReview` bootstrap authenticator. pub service_account_name: String, pub default_image: String, - pub image_pull_policy: String, + /// Pull policy for sandbox images. Omit to use Kubernetes's image default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_pull_policy: Option, /// Kubernetes `imagePullSecrets` names attached to sandbox pods. pub image_pull_secrets: Vec, /// Managed-mode SSH ingress isolation. When enabled, the driver creates a @@ -317,9 +244,10 @@ pub struct KubernetesComputeConfig { /// Mounted directly as an image volume, or copied via an init container, /// depending on `supervisor_sideload_method`. pub supervisor_image: String, - /// Kubernetes `imagePullPolicy` for the supervisor image. - /// Empty string delegates to the Kubernetes default. - pub supervisor_image_pull_policy: String, + /// Pull policy for the supervisor image. Omit to use Kubernetes's image + /// default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supervisor_image_pull_policy: Option, /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. @@ -439,15 +367,13 @@ impl Default for KubernetesComputeConfig { operator_namespace_file: None, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(), default_image: openshell_core::image::default_sandbox_image(), - // Default empty so the gateway omits `imagePullPolicy` from pod - // specs and Kubernetes applies its own default (Always for `latest`, - // IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing") - // is Podman vocabulary and is not a valid Kubernetes value. - image_pull_policy: String::new(), + // Omit the field so Kubernetes applies its own default (Always for + // `latest`, IfNotPresent otherwise). + image_pull_policy: None, image_pull_secrets: Vec::new(), managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), - supervisor_image_pull_policy: String::new(), + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), @@ -475,6 +401,16 @@ impl Default for KubernetesComputeConfig { } impl KubernetesComputeConfig { + /// Validate startup configuration without connecting to Kubernetes. + pub fn validate_configuration(&self) -> Result<(), String> { + self.validate_workspace_mode()?; + self.validate_provider_spiffe_workload_api_socket_path()?; + self.validate_sandbox_identity_config()?; + self.validate_proxy_uid()?; + self.validate_image_pull_policies()?; + self.validate_upstream_proxy_config() + } + /// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS, /// MAX_SA_TOKEN_TTL_SECS]` range used by the projected-volume spec. /// Invalid (≤0) values fall back to the default 3600. @@ -506,17 +442,52 @@ impl KubernetesComputeConfig { self.sidecar.validate_proxy_uid() } + /// Reject pull policies Kubernetes cannot express before creating pods. + pub fn validate_image_pull_policies(&self) -> Result<(), String> { + for (field, policy) in [ + ("image_pull_policy", self.image_pull_policy), + ( + "supervisor_image_pull_policy", + self.supervisor_image_pull_policy, + ), + ] { + if policy == Some(ImagePullPolicy::Newer) { + return Err(format!( + "{field} = \"newer\" is supported only by the Podman compute driver" + )); + } + } + Ok(()) + } + + /// Translate a validated shared policy to Kubernetes's API vocabulary. + #[must_use] + pub fn image_pull_policy_value(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "Always", + ImagePullPolicy::IfNotPresent => "IfNotPresent", + ImagePullPolicy::Never => "Never", + ImagePullPolicy::Newer => unreachable!("newer must be rejected during validation"), + } + } + /// Validate the operator-owned corporate upstream proxy configuration. pub fn validate_upstream_proxy_config(&self) -> Result<(), String> { use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - if let Some(url) = &self.https_proxy { - parse_upstream_proxy_url(url).map_err(|err| match err { - UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(), - UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), - err => format!("https_proxy {err}"), - })?; - } + let proxy_addr = self + .https_proxy + .as_deref() + .map(|url| { + parse_upstream_proxy_url(url).map_err(|err| match err { + UpstreamProxyUrlError::Empty => { + "https_proxy must not be empty when set".to_string() + } + UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(), + err => format!("https_proxy {err}"), + }) + }) + .transpose()?; if let Some(list) = self.no_proxy.as_deref() { if list.trim().is_empty() { @@ -575,7 +546,9 @@ impl KubernetesComputeConfig { .to_string(), ); } - if self.proxy_auth_allow_insecure != Some(true) { + if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure) + && self.proxy_auth_allow_insecure != Some(true) + { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } if self.topology == SupervisorTopology::Combined { @@ -928,6 +901,47 @@ mod tests { assert!(cfg.sidecar.process_binary_aware_network_policy); } + #[test] + fn image_pull_policy_uses_shared_canonical_values() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "image_pull_policy": "if_not_present", + "supervisor_image_pull_policy": "never" + })) + .unwrap(); + assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent)); + assert_eq!( + cfg.supervisor_image_pull_policy, + Some(ImagePullPolicy::Never) + ); + + for (policy, expected) in [ + (ImagePullPolicy::Always, "Always"), + (ImagePullPolicy::IfNotPresent, "IfNotPresent"), + (ImagePullPolicy::Never, "Never"), + ] { + assert_eq!( + KubernetesComputeConfig::image_pull_policy_value(policy), + expected + ); + } + } + + #[test] + fn image_pull_policy_rejects_newer_for_sandbox_and_supervisor_images() { + for (sandbox, supervisor) in [ + (Some(ImagePullPolicy::Newer), None), + (None, Some(ImagePullPolicy::Newer)), + ] { + let cfg = KubernetesComputeConfig { + image_pull_policy: sandbox, + supervisor_image_pull_policy: supervisor, + ..KubernetesComputeConfig::default() + }; + let error = cfg.validate_image_pull_policies().unwrap_err(); + assert!(error.contains("supported only by the Podman")); + } + } + #[test] fn serde_override_topology_sidecar() { let json = serde_json::json!({ @@ -1387,6 +1401,18 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } + #[test] + fn upstream_proxy_config_accepts_tls_protected_secret_credentials() { + let cfg = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + https_proxy: Some("https://proxy.corp.example:8443".to_string()), + proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), + proxy_auth_secret_key: Some("credentials".to_string()), + ..KubernetesComputeConfig::default() + }; + assert!(cfg.validate_upstream_proxy_config().is_ok()); + } + #[test] fn toml_deserializes_sidecar_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 3a19917384..888bf1864a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -498,19 +498,7 @@ impl KubernetesComputeDriver { shutdown_rx: tokio::sync::watch::Receiver, ) -> Result { config - .validate_workspace_mode() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_provider_spiffe_workload_api_socket_path() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_sandbox_identity_config() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_proxy_uid() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_upstream_proxy_config() + .validate_configuration() .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, @@ -1520,10 +1508,16 @@ impl KubernetesComputeDriver { let params = SandboxPodParams { default_image: &self.config.default_image, - image_pull_policy: &self.config.image_pull_policy, + image_pull_policy: self + .config + .image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), image_pull_secrets: &self.config.image_pull_secrets, supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, + supervisor_image_pull_policy: self + .config + .supervisor_image_pull_policy + .map(KubernetesComputeConfig::image_pull_policy_value), supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, proxy_uid: self.config.sidecar.proxy_uid, @@ -2768,13 +2762,13 @@ fn supervisor_volume_mount() -> serde_json::Value { /// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. fn supervisor_image_volume( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let mut image_spec = serde_json::json!({ "reference": supervisor_image, }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + image_spec["pullPolicy"] = serde_json::json!(policy); } serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2792,7 +2786,7 @@ fn supervisor_image_volume( /// emissary executor. fn supervisor_init_container( supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, ) -> serde_json::Value { let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ @@ -2810,8 +2804,8 @@ fn supervisor_init_container( "readOnly": false }] }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + if let Some(policy) = supervisor_image_pull_policy { + spec["imagePullPolicy"] = serde_json::json!(policy); } spec } @@ -2819,7 +2813,7 @@ fn supervisor_init_container( fn apply_supervisor_binary_source( spec: &mut serde_json::Map, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, ) { let volumes = spec @@ -2953,7 +2947,7 @@ fn apply_supervisor_sideload_with_params( fn apply_supervisor_sideload( pod_template: &mut serde_json::Value, supervisor_image: &str, - supervisor_image_pull_policy: &str, + supervisor_image_pull_policy: Option<&str>, method: SupervisorSideloadMethod, sandbox_uid: u32, sandbox_gid: u32, @@ -3163,8 +3157,8 @@ fn supervisor_sidecar_container( .into_iter() .map(serde_json::Value::String), ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if params.provider_spiffe_enabled { container["volumeMounts"] @@ -3226,8 +3220,8 @@ fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_jso sidecar_tls_volume_mount(), ] }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + if let Some(policy) = params.supervisor_image_pull_policy { + container["imagePullPolicy"] = serde_json::json!(policy); } if !params.client_tls_secret_name.is_empty() { container["volumeMounts"] @@ -3427,7 +3421,7 @@ fn apply_supervisor_sidecar_topology( fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, - image_pull_policy: &str, + image_pull_policy: Option<&str>, sandbox_gid: u32, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { @@ -3517,8 +3511,8 @@ fn apply_workspace_persistence( "mountPath": WORKSPACE_INIT_MOUNT_PATH }] }); - if !image_pull_policy.is_empty() { - init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + if let Some(policy) = image_pull_policy { + init_spec["imagePullPolicy"] = serde_json::json!(policy); } init_containers.push(init_spec); } @@ -3565,10 +3559,10 @@ fn default_workspace_volume_claim_templates( #[allow(clippy::struct_excessive_bools)] struct SandboxPodParams<'a> { default_image: &'a str, - image_pull_policy: &'a str, + image_pull_policy: Option<&'a str>, image_pull_secrets: &'a [String], supervisor_image: &'a str, - supervisor_image_pull_policy: &'a str, + supervisor_image_pull_policy: Option<&'a str>, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, proxy_uid: u32, @@ -3606,10 +3600,10 @@ impl Default for SandboxPodParams<'_> { fn default() -> Self { Self { default_image: "", - image_pull_policy: "", + image_pull_policy: None, image_pull_secrets: &[], supervisor_image: "", - supervisor_image_pull_policy: "", + supervisor_image_pull_policy: None, supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), proxy_uid: DEFAULT_PROXY_UID, @@ -3934,11 +3928,8 @@ fn sandbox_template_to_k8s_with_validated_config( }; if !image.is_empty() { container.insert("image".to_string(), serde_json::json!(image)); - if !params.image_pull_policy.is_empty() { - container.insert( - "imagePullPolicy".to_string(), - serde_json::json!(params.image_pull_policy), - ); + if let Some(policy) = params.image_pull_policy { + container.insert("imagePullPolicy".to_string(), serde_json::json!(policy)); } } @@ -4240,7 +4231,7 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { let mut value = serde_json::json!({ - "type": profile.to_k8s_type() + "type": profile.kubernetes_type() }); if let Some(localhost_profile) = profile.localhost_profile() { value["localhostProfile"] = serde_json::json!(localhost_profile); @@ -6217,7 +6208,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "custom-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, // sandbox_uid 1500, // sandbox_gid @@ -6254,7 +6245,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1500, 1600, @@ -6301,7 +6292,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6328,7 +6319,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::InitContainer, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6415,7 +6406,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "IfNotPresent", + Some("IfNotPresent"), SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6458,7 +6449,7 @@ mod tests { } #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { + fn supervisor_image_volume_omits_pull_policy_when_unspecified() { let mut pod_template = serde_json::json!({ "spec": { "containers": [{ @@ -6471,7 +6462,7 @@ mod tests { apply_supervisor_sideload( &mut pod_template, "supervisor-image:latest", - "", + None, SupervisorSideloadMethod::ImageVolume, 1000, // sandbox_uid 1000, // sandbox_gid @@ -6481,7 +6472,7 @@ mod tests { assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); assert!( volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + "pullPolicy should be omitted when unspecified" ); } @@ -6491,7 +6482,7 @@ mod tests { topology: SupervisorTopology::Sidecar, supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", + supervisor_image_pull_policy: Some("IfNotPresent"), grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", client_tls_secret_name: "openshell-client-tls", proxy_uid: 2200, @@ -7369,7 +7360,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "openshell/sandbox:latest", - "IfNotPresent", + Some("IfNotPresent"), 1000, // sandbox_gid ); @@ -7428,7 +7419,7 @@ mod tests { apply_workspace_persistence( &mut pod_template, "my-custom-image:v2", - "IfNotPresent", + Some("IfNotPresent"), 1000, ); @@ -7452,7 +7443,7 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence(&mut pod_template, "img:latest", Some("Always"), 1000); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 9b11f5da2a..d949d6c72c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -8,8 +8,8 @@ use std::net::SocketAddr; use std::path::PathBuf; use tracing::info; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{ImagePullPolicy, VERSION}; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, @@ -72,7 +72,7 @@ struct Args { sandbox_image: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, + sandbox_image_pull_policy: Option, #[arg( long, @@ -94,8 +94,10 @@ struct Args { )] managed_ssh_gateway_pod_selector: Vec, + /// Gateway callback endpoint reachable from sandbox pods. Kubernetes + /// service topology cannot be inferred from the sandbox namespace. #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, + grpc_endpoint: String, #[arg( long, @@ -114,7 +116,7 @@ struct Args { supervisor_image: Option, #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, + supervisor_image_pull_policy: Option, #[arg( long, @@ -248,7 +250,7 @@ async fn main() -> Result<()> { operator_namespace_file: args.operator_namespace_file, service_account_name: args.sandbox_service_account, default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, image_pull_secrets: args.sandbox_image_pull_secrets, managed_ssh_ingress: ManagedSshIngressConfig { enabled: args.managed_ssh_ingress_enabled, @@ -258,7 +260,7 @@ async fn main() -> Result<()> { supervisor_image: args .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), + supervisor_image_pull_policy: args.supervisor_image_pull_policy, supervisor_sideload_method: args.supervisor_sideload_method, topology: args.topology, sidecar: KubernetesSidecarConfig { @@ -272,7 +274,7 @@ async fn main() -> Result<()> { proxy_auth_secret_key: args.proxy_auth_secret_key, proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true), proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + grpc_endpoint: args.grpc_endpoint, ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), @@ -335,6 +337,13 @@ async fn main() -> Result<()> { mod tests { use super::*; + #[test] + fn requires_explicit_gateway_callback_endpoint() { + let error = Args::try_parse_from(["openshell-driver-kubernetes"]) + .expect_err("Kubernetes service topology must be explicit"); + assert!(error.to_string().contains("--grpc-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ @@ -343,6 +352,8 @@ mod tests { "http://collector.example:4317", "--gateway-name", "kubernetes-dev", + "--grpc-endpoint", + "http://openshell.example:8080", ]) .expect("OTLP endpoint should parse"); diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 9f8baa3446..837913ed1f 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -240,7 +240,7 @@ Step "Prepare DemoDir $DemoDir" New-Item -ItemType Directory -Force $DemoDir | Out-Null Ok "DemoDir ready" -$env:OPENSHELL_DRIVERS = "mxc" +$env:OPENSHELL_COMPUTE_DRIVER = "mxc" $env:OPENSHELL_MXC_SHARE_DIR = $DemoDir # ── Start gateway ───────────────────────────────────────────────────────────── diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index e53ddc9f3f..5781594158 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -275,9 +275,9 @@ Podman follows the same end-to-end contract as the Kubernetes and VM drivers for the in-container SSH relay: gateway config to `PodmanComputeConfig` to sandbox environment to supervisor session registration on that path. -1. `openshell-core` `Config::sandbox_ssh_socket_path` is copied into - `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the - in-process driver. +1. `[openshell.drivers.podman].ssh_socket_path` is deserialized into + `PodmanComputeConfig::ssh_socket_path` when the gateway builds the in-process + driver. The field defaults to `/run/openshell/ssh.sock` when omitted. 2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that value, alongside required vars such as `OPENSHELL_ENDPOINT` and `OPENSHELL_SANDBOX_ID`. These driver-controlled entries overwrite template @@ -356,8 +356,12 @@ signals succeeds: - `test -S` on the configured supervisor Unix socket path. - The prior TCP check for a listener on the in-container SSH port. -The Unix socket check allows relay-only readiness when the supervisor exposes -the socket without the old marker or published-port signal. +The Unix socket check allows relay-only backend readiness when the supervisor +exposes the socket without the old marker or published-port signal. Omitting +`health_check_interval_secs` disables these Podman/conmon probes, but it does +not bypass public readiness gating: the gateway keeps a backend-ready sandbox +in `Provisioning` with `SupervisorNotConnected` until its supervisor control +session is connected. ### Deletion Flow @@ -380,14 +384,14 @@ Podman resources after out-of-band container removal or label drift. |---|---|---|---| | `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | Probes known local Podman API sockets and uses the first responsive socket, then falls back to asking the `podman` CLI for the host-side socket. Fails to start if neither finds one. | Podman API Unix socket path. | | `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | From gateway config | Default OCI image for sandboxes. | -| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, or `newer`. | +| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `if_not_present` | Pull policy: `always`, `if_not_present`, `never`, or `newer`. | | `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks. | | `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | | `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. | -| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | +| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Omission uses OpenShell's `2048` default; explicit `0` is invalid. | | `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. | | `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. | | `OPENSHELL_PODMAN_TLS_CERT` | `--podman-tls-cert` | unset | Host path to the client certificate mounted for sandbox mTLS. | @@ -405,6 +409,17 @@ Through the gateway, the same settings are the `https_proxy`, `no_proxy`, and `proxy_ca_bundle` keys under `[openshell.drivers.podman]`; see `docs/reference/gateway-config.mdx`. +`provider_spiffe_workload_api_socket` accepts either an absolute host UNIX +Workload API socket, projected through a dedicated read-only mount, or an +explicit container-reachable `tcp:IP:port` endpoint. The driver sets the +supervisor's `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` accordingly. +`app_armor_profile` shares the canonical +`RuntimeDefault`, `Unconfined`, or `Localhost/` model with Docker and +Kubernetes. When omitted, the driver sends no override and preserves Podman's +runtime-selected profile. Set `Unconfined` explicitly only when the deployment +requires the supervisor's mount setup to bypass that profile. Explicit confined +choices fail early when Podman reports AppArmor unavailable. + This is an operator-owned egress boundary: the driver passes the settings on the supervisor's command line, so sandbox and template environment — and any `ENV` baked into the sandbox image — cannot override them, and the diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index 508b604ce7..5e0dadfb7f 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -272,9 +272,13 @@ pub struct HostInfo { /// Podman returns `host.security.rootless: true` when the daemon is /// running without root privileges (rootless mode). #[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "camelCase")] pub struct SecurityInfo { #[serde(default)] pub rootless: bool, + /// Whether the Podman host has `AppArmor` support enabled. + #[serde(default)] + pub apparmor_enabled: bool, } // ── Client ─────────────────────────────────────────────────────────────── @@ -959,13 +963,17 @@ mod tests { "cgroupVersion": "v2", "networkBackend": "netavark", "rootlessNetworkCmd": "pasta", - "security": {"rootless": true} + "security": { + "rootless": true, + "apparmorEnabled": true + } } }"#, ) .unwrap(); assert!(info.host.security.rootless); + assert!(info.host.security.apparmor_enabled); assert_eq!(info.host.rootless_network_cmd, "pasta"); } diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 42571f00f1..9f139cde94 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::net::IpAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; -use std::str::FromStr; + +use openshell_core::{AppArmorProfile, ImagePullPolicy}; /// Default Podman bridge network name. pub const DEFAULT_NETWORK_NAME: &str = "openshell"; @@ -11,59 +13,14 @@ pub const MACOS_PODMAN_MACHINE_HOST_GATEWAY_IP: &str = "192.168.127.254"; /// Default Podman stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_PODMAN_STOP_TIMEOUT_SECS: u32 = 45; -// Re-export the shared default so existing imports inside this crate keep working. -pub use openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT; - -/// Image pull policy for sandbox and supervisor images. -/// -/// Controls when the Podman driver fetches a newer copy of an OCI image -/// from the registry. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ImagePullPolicy { - /// Always pull, even if a local copy exists. - Always, - /// Pull only when no local copy exists (default). - #[default] - Missing, - /// Never pull; fail if not available locally. - Never, - /// Pull only if the remote image is newer. - Newer, -} - -impl ImagePullPolicy { - /// Return the policy string expected by the Podman libpod API. - #[must_use] - pub fn as_str(&self) -> &'static str { - match self { - Self::Always => "always", - Self::Missing => "missing", - Self::Never => "never", - Self::Newer => "newer", - } - } -} - -impl std::fmt::Display for ImagePullPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ImagePullPolicy { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "always" => Ok(Self::Always), - "missing" => Ok(Self::Missing), - "never" => Ok(Self::Never), - "newer" => Ok(Self::Newer), - other => Err(format!( - "invalid pull policy '{other}'; expected one of: always, missing, never, newer" - )), - } +/// Translate the shared pull-policy vocabulary to the Podman libpod API. +#[must_use] +pub const fn podman_image_pull_policy(policy: ImagePullPolicy) -> &'static str { + match policy { + ImagePullPolicy::Always => "always", + ImagePullPolicy::IfNotPresent => "missing", + ImagePullPolicy::Never => "never", + ImagePullPolicy::Newer => "newer", } } @@ -90,7 +47,7 @@ pub struct PodmanComputeConfig { /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, /// Unix socket path the in-container supervisor bridges relay traffic to. - pub sandbox_ssh_socket_path: String, + pub ssh_socket_path: String, /// Name of the Podman bridge network. /// Created automatically if it does not exist. pub network_name: String, @@ -120,8 +77,13 @@ pub struct PodmanComputeConfig { pub guest_tls_key: Option, /// Container cgroup PID limit for Podman-managed sandboxes. /// - /// Set to `0` to leave Podman's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, + /// Omit the field to use `OpenShell`'s 2048-process sandbox limit. Explicit + /// zero is invalid. + #[serde( + default = "openshell_core::config::default_sandbox_pids_limit", + skip_serializing_if = "Option::is_none" + )] + pub sandbox_pids_limit: Option, /// Allow sandbox requests to attach host bind mounts through /// `template.driver_config`. #[serde(default)] @@ -129,14 +91,19 @@ pub struct PodmanComputeConfig { /// Host path to a SPIFFE Workload API Unix socket exposed to sandbox /// supervisors for provider token exchange client assertions. pub provider_spiffe_workload_api_socket: Option, + /// `AppArmor` confinement requested for sandbox containers. Omission sends + /// no override and preserves Podman's runtime-selected profile. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_armor_profile: Option, /// Health check interval in seconds for sandbox containers. /// /// Podman runs the health check command at this interval to determine /// container readiness. Lower values detect readiness faster but /// increase process churn (each check spawns a conmon subprocess). - /// Set to `0` to disable health checks entirely. - /// Defaults to [`DEFAULT_HEALTH_CHECK_INTERVAL_SECS`] (10 seconds). - pub health_check_interval_secs: u64, + /// Omit the field to disable health checks entirely. Explicit zero is + /// invalid. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub health_check_interval_secs: Option, /// Corporate forward proxy URL passed to the in-container supervisor /// (e.g. `http://proxy.corp.com:8080` or `https://proxy.corp.com:3130`). /// @@ -217,8 +184,6 @@ pub struct PodmanComputeConfig { pub gidmap: Vec, } -pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; - /// Parse a single `"container_id:host_id:size"` mapping entry. /// /// Returns `(container_id, host_id, size)` on success. @@ -256,6 +221,30 @@ pub fn parse_id_map_entry( } impl PodmanComputeConfig { + /// Validate and normalize startup configuration without connecting to Podman. + pub fn validate_configuration(&mut self) -> Result<(), crate::client::PodmanApiError> { + self.validate_tls_config()?; + self.validate_runtime_limits()?; + self.validate_host_gateway_ip()?; + self.validate_proxy_config()?; + self.validate_app_armor_profile()?; + if let Some(socket) = self.provider_spiffe_workload_api_socket.as_deref() { + let raw = socket.to_str().ok_or_else(|| { + crate::client::PodmanApiError::InvalidInput( + "provider_spiffe_workload_api_socket must be valid UTF-8".to_string(), + ) + })?; + // Preserve pass-through support for an explicitly configured + // container-reachable Workload API TCP endpoint. + if !raw.starts_with("tcp:") { + openshell_core::driver_utils::validate_provider_spiffe_unix_socket(socket) + .map_err(crate::client::PodmanApiError::InvalidInput)?; + } + } + self.canonicalize_userns()?; + self.validate_userns_mappings() + } + /// Returns `true` when all three TLS paths are configured. #[must_use] pub fn tls_enabled(&self) -> bool { @@ -298,9 +287,9 @@ impl PodmanComputeConfig { /// Validate runtime resource-limit configuration. pub fn validate_runtime_limits(&self) -> Result<(), crate::client::PodmanApiError> { - if self.sandbox_pids_limit < 0 { + if self.sandbox_pids_limit.is_some_and(|limit| limit.get() < 0) { return Err(crate::client::PodmanApiError::InvalidInput( - "sandbox_pids_limit must be zero or greater".to_string(), + "sandbox_pids_limit must be positive when set".to_string(), )); } Ok(()) @@ -318,91 +307,19 @@ impl PodmanComputeConfig { /// the URL is rejected because it would otherwise be stored in /// `gateway.toml` and exposed in container metadata. pub fn validate_proxy_config(&self) -> Result<(), crate::client::PodmanApiError> { - use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url}; - let proxy_secure = if let Some(url) = &self.https_proxy { - let addr = parse_upstream_proxy_url(url).map_err(|err| { - crate::client::PodmanApiError::InvalidInput(match err { - UpstreamProxyUrlError::Empty => { - "https_proxy must not be empty when set".to_string() - } - UpstreamProxyUrlError::InlineCredentials => { - "https_proxy must not embed credentials in the URL; supply them via \ - proxy_auth_file so they are not stored in config or container metadata" - .to_string() - } - err => format!("https_proxy {err}"), - }) - })?; - addr.secure - } else { - false - }; - - // The supervisor treats a present-but-empty driver-supplied argument - // as a fatal misconfiguration, so never accept (and later pass) one. - if let Some(list) = self.no_proxy.as_deref() { - if list.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy must not be empty when set; omit it instead".to_string(), - )); - } - // A bypass list only makes sense relative to a proxy boundary. An - // operator who set one believed proxying was in effect, so accepting - // it while all egress dials directly would hide a fail-open state. - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "no_proxy is set but no https_proxy is configured".to_string(), - )); - } + // Keep the Podman-only CA-bundle behaviour below, but delegate the + // shared URL, bypass-list, credential-file, and acknowledgement + // contract to openshell-core so Docker and VM cannot drift. + openshell_core::UpstreamProxyConfig { + https_proxy: self.https_proxy.clone(), + no_proxy: self.no_proxy.clone(), + proxy_auth_file: self.proxy_auth_file.as_ref().map(PathBuf::from), + proxy_auth_allow_insecure: self.proxy_auth_allow_insecure, + proxy_connect_by_hostname: self.proxy_connect_by_hostname, } + .validate() + .map_err(crate::client::PodmanApiError::InvalidInput)?; - if let Some(path) = self.proxy_auth_file.as_deref() { - if path.trim().is_empty() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file must not be empty when set".to_string(), - )); - } - if self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file is set but no https_proxy is configured".to_string(), - )); - } - // Basic auth over the plain-TCP proxy connection is readable by - // anyone on the network path; sending it requires an explicit - // operator acknowledgement rather than being an implicit side - // effect of configuring credentials. For an https:// proxy the - // credential is inside the verified TLS session, so the - // acknowledgement is unnecessary (but tolerated). - if self.proxy_auth_allow_insecure != Some(true) && !proxy_secure { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_file sends the credential as cleartext Basic auth over the \ - plain-TCP connection to the http:// proxy; set proxy_auth_allow_insecure \ - = true to accept that exposure, or remove proxy_auth_file" - .to_string(), - )); - } - } else if self.proxy_auth_allow_insecure.is_some() { - // The acknowledgement without credentials means the operator - // believed an auth file was configured; surface the mismatch. - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_auth_allow_insecure is set but no proxy_auth_file is configured".to_string(), - )); - } - - // The CONNECT-target mode only means something relative to a proxy - // boundary the operator believed was in effect. - if self.proxy_connect_by_hostname.is_some() && self.https_proxy.is_none() { - return Err(crate::client::PodmanApiError::InvalidInput( - "proxy_connect_by_hostname is set but no https_proxy is configured".to_string(), - )); - } - - // A CA bundle only makes sense relative to a proxy boundary (an - // https:// proxy handshake, or a TLS-intercepting proxy's re-sign CA). - // Mirror the proxy_auth_file pairing so a stray setting cannot hide a - // fail-open state. The file's readability and certificate content are - // checked at sandbox-create time (see the driver) and fail closed in - // the supervisor. if let Some(path) = self.proxy_ca_bundle.as_deref() { if path.trim().is_empty() { return Err(crate::client::PodmanApiError::InvalidInput( @@ -501,6 +418,18 @@ impl PodmanComputeConfig { } /// Validate optional host gateway override. + /// Validate `AppArmor` syntax before contacting the runtime. Drivers check + /// runtime availability after querying their backend. + pub fn validate_app_armor_profile(&self) -> Result<(), crate::client::PodmanApiError> { + if matches!(self.app_armor_profile, Some(AppArmorProfile::Localhost(ref name)) if name.is_empty()) + { + return Err(crate::client::PodmanApiError::InvalidInput( + "app_armor_profile Localhost profile must not be empty".to_string(), + )); + } + Ok(()) + } + pub fn validate_host_gateway_ip(&self) -> Result<(), crate::client::PodmanApiError> { let trimmed = self.host_gateway_ip.trim(); if trimmed.is_empty() { @@ -536,7 +465,7 @@ impl Default for PodmanComputeConfig { image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - sandbox_ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), + ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS, @@ -544,10 +473,11 @@ impl Default for PodmanComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, + sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), enable_bind_mounts: false, provider_spiffe_workload_api_socket: None, - health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + app_armor_profile: None, + health_check_interval_secs: None, https_proxy: None, no_proxy: None, proxy_auth_file: None, @@ -566,10 +496,10 @@ impl std::fmt::Debug for PodmanComputeConfig { f.debug_struct("PodmanComputeConfig") .field("socket_path", &self.socket_path) .field("default_image", &self.default_image) - .field("image_pull_policy", &self.image_pull_policy.as_str()) + .field("image_pull_policy", &self.image_pull_policy) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) - .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) + .field("ssh_socket_path", &self.ssh_socket_path) .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) .field("stop_timeout_secs", &self.stop_timeout_secs) @@ -583,6 +513,7 @@ impl std::fmt::Debug for PodmanComputeConfig { "provider_spiffe_workload_api_socket", &self.provider_spiffe_workload_api_socket, ) + .field("app_armor_profile", &self.app_armor_profile) .field( "health_check_interval_secs", &self.health_check_interval_secs, @@ -606,14 +537,72 @@ mod tests { use super::*; #[test] - fn default_config_sets_health_check_interval() { - let cfg = PodmanComputeConfig::default(); + fn shared_image_pull_policies_map_to_podman_vocabulary() { + for (policy, expected) in [ + (ImagePullPolicy::Always, "always"), + (ImagePullPolicy::IfNotPresent, "missing"), + (ImagePullPolicy::Never, "never"), + (ImagePullPolicy::Newer, "newer"), + ] { + assert_eq!(podman_image_pull_policy(policy), expected); + } + } + + #[test] + fn config_uses_canonical_ssh_socket_path_name() { + let config: PodmanComputeConfig = + serde_json::from_value(serde_json::json!({ "ssh_socket_path": "/run/test.sock" })) + .unwrap(); + assert_eq!(config.ssh_socket_path, "/run/test.sock"); + + let serialized = serde_json::to_value(config).unwrap(); + assert_eq!(serialized["ssh_socket_path"], "/run/test.sock"); + assert!(serialized.get("sandbox_ssh_socket_path").is_none()); + } + + #[test] + fn config_rejects_legacy_sandbox_ssh_socket_path() { + let error = serde_json::from_value::(serde_json::json!({ + "sandbox_ssh_socket_path": "/run/test.sock" + })) + .expect_err("legacy sandbox_ssh_socket_path must be rejected"); + assert!(error.to_string().contains("sandbox_ssh_socket_path")); + } + + #[test] + fn default_config_disables_health_checks() { assert_eq!( - cfg.health_check_interval_secs, - DEFAULT_HEALTH_CHECK_INTERVAL_SECS + PodmanComputeConfig::default().health_check_interval_secs, + None ); } + #[test] + fn omitted_apparmor_profile_preserves_runtime_default() { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("omitted AppArmor profile should deserialize"); + assert_eq!(config.app_armor_profile, None); + + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert!(serialized.get("app_armor_profile").is_none()); + } + + #[test] + fn explicit_apparmor_profiles_round_trip() { + for value in [ + "RuntimeDefault", + "Unconfined", + "Localhost/openshell-supervisor", + ] { + let config: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "app_armor_profile": value, + })) + .expect("explicit AppArmor profile should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["app_armor_profile"], value); + } + } + #[test] fn default_config_sets_podman_stop_timeout() { let cfg = PodmanComputeConfig::default(); @@ -623,9 +612,21 @@ mod tests { #[test] fn default_config_sets_driver_owned_pids_limit() { let cfg = PodmanComputeConfig::default(); - assert_eq!(cfg.sandbox_pids_limit, DEFAULT_SANDBOX_PIDS_LIMIT); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); assert!(!cfg.enable_bind_mounts); - assert!(cfg.validate_runtime_limits().is_ok()); + } + + #[test] + fn omitted_pids_limit_uses_driver_owned_default() { + let cfg: PodmanComputeConfig = serde_json::from_value(serde_json::json!({})) + .expect("default Podman config should deserialize"); + assert_eq!( + cfg.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); } #[test] @@ -655,13 +656,28 @@ mod tests { } #[test] - fn runtime_limit_validation_rejects_negative_pids_limit() { - let cfg = PodmanComputeConfig { - sandbox_pids_limit: -1, - ..PodmanComputeConfig::default() - }; - let err = cfg.validate_runtime_limits().unwrap_err(); - assert!(err.to_string().contains("sandbox_pids_limit")); + fn runtime_limit_rejects_invalid_pids_limits() { + let zero = serde_json::from_value::(serde_json::json!({ + "sandbox_pids_limit": 0 + })) + .expect_err("zero PID limit must be rejected"); + assert!(zero.to_string().contains("invalid value: integer `0`")); + + let negative: PodmanComputeConfig = serde_json::from_value(serde_json::json!({ + "sandbox_pids_limit": -1 + })) + .expect("nonzero integer deserializes before semantic validation"); + let error = negative.validate_runtime_limits().unwrap_err(); + assert!(error.to_string().contains("must be positive")); + } + + #[test] + fn health_check_interval_rejects_zero() { + let error = serde_json::from_value::(serde_json::json!({ + "health_check_interval_secs": 0 + })) + .expect_err("zero health-check interval must be rejected"); + assert!(error.to_string().contains("invalid value: integer `0`")); } // ── Proxy config validation ─────────────────────────────────────── diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a81ee13e1d..541cb2f756 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -216,8 +216,14 @@ struct ContainerSpec { cap_add: Vec, no_new_privileges: bool, seccomp_profile_path: String, + /// Podman's container create API accepts `AppArmor` through the dedicated + /// `apparmor_profile` `SpecGenerator` field. This is not Docker's + /// `security_opt` representation. + #[serde(skip_serializing_if = "Option::is_none")] + apparmor_profile: Option, image_pull_policy: String, - healthconfig: HealthConfig, + #[serde(skip_serializing_if = "Option::is_none")] + healthconfig: Option, resource_limits: ResourceLimits, /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. /// Podman's libpod `SpecGenerator` uses `secret_env` (a flat map) for @@ -334,8 +340,15 @@ struct SecretMount { struct ResourceLimits { cpu: CpuLimits, memory: MemoryLimits, - #[serde(rename = "PidsLimit", skip_serializing_if = "Option::is_none")] - pids_limit: Option, + // Podman's libpod API consumes the OCI LinuxResources shape. A Docker-style + // scalar PidsLimit is silently ignored and leaves the runtime default. + #[serde(skip_serializing_if = "Option::is_none")] + pids: Option, +} + +#[derive(Serialize)] +struct PidsLimits { + limit: i64, } #[derive(Serialize)] @@ -526,7 +539,7 @@ fn build_env( ); env.insert( openshell_core::sandbox_env::SSH_SOCKET_PATH.into(), - config.sandbox_ssh_socket_path.clone(), + config.ssh_socket_path.clone(), ); env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); let main_process = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(spec) @@ -652,14 +665,12 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, - pids_limit: podman_pids_limit(config.sandbox_pids_limit), + pids: config + .sandbox_pids_limit + .map(|limit| PidsLimits { limit: limit.get() }), } } -fn podman_pids_limit(value: i64) -> Option { - if value > 0 { Some(value) } else { None } -} - pub fn podman_driver_volume_mount_sources( sandbox: &DriverSandbox, enable_bind_mounts: bool, @@ -940,6 +951,14 @@ fn validate_tmpfs_options(options: &[String]) -> Result, String> { .collect() } +fn podman_apparmor_profile(profile: Option<&openshell_core::AppArmorProfile>) -> Option { + match profile { + None | Some(openshell_core::AppArmorProfile::RuntimeDefault) => None, + Some(openshell_core::AppArmorProfile::Unconfined) => Some("unconfined".to_string()), + Some(openshell_core::AppArmorProfile::Localhost(profile)) => Some(profile.clone()), + } +} + /// Build the Podman container creation JSON spec. #[cfg(test)] #[must_use] @@ -1175,21 +1194,22 @@ pub fn build_container_spec_for_image( // locks itself down. no_new_privileges: true, seccomp_profile_path: "unconfined".into(), + apparmor_profile: podman_apparmor_profile(config.app_armor_profile.as_ref()), image_pull_policy: "never".to_string(), - healthconfig: HealthConfig { + healthconfig: config.health_check_interval_secs.map(|interval_secs| HealthConfig { test: vec![ "CMD-SHELL".into(), format!( "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", - config.sandbox_ssh_socket_path, + config.ssh_socket_path, openshell_core::config::DEFAULT_SSH_PORT ), ], - interval: config.health_check_interval_secs * 1_000_000_000, + interval: interval_secs.get() * 1_000_000_000, timeout: 2_000_000_000, retries: 10, start_period: 5_000_000_000, - }, + }), resource_limits, secret_env: BTreeMap::new(), secrets: { @@ -1544,7 +1564,7 @@ mod tests { } #[test] - fn container_spec_applies_cpu_and_memory_limits() { + fn container_spec_applies_resource_limits() { use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, }; @@ -1561,7 +1581,8 @@ mod tests { }), ..Default::default() }); - let config = test_config(); + let mut config = test_config(); + config.sandbox_pids_limit = std::num::NonZeroI64::new(2048); let spec = build_container_spec(&sandbox, &config); assert_eq!( @@ -1573,8 +1594,8 @@ mod tests { Some(2 * 1024 * 1024 * 1024) ); assert_eq!( - spec["resource_limits"]["PidsLimit"].as_i64(), - Some(crate::config::DEFAULT_SANDBOX_PIDS_LIMIT) + spec["resource_limits"]["pids"]["limit"].as_i64(), + Some(2048) ); } @@ -1582,10 +1603,44 @@ mod tests { fn container_spec_can_inherit_runtime_pids_limit() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.sandbox_pids_limit = 0; + config.sandbox_pids_limit = None; let spec = build_container_spec(&sandbox, &config); - assert!(spec["resource_limits"].get("PidsLimit").is_none()); + assert!(spec["resource_limits"].get("pids").is_none()); + } + + #[test] + fn container_spec_uses_podman_apparmor_profile_field() { + let sandbox = test_sandbox("test-id", "test-name"); + + for (profile, expected) in [ + (openshell_core::AppArmorProfile::Unconfined, "unconfined"), + ( + openshell_core::AppArmorProfile::Localhost("openshell-supervisor".to_string()), + "openshell-supervisor", + ), + ] { + let mut config = test_config(); + config.app_armor_profile = Some(profile); + let spec = build_container_spec(&sandbox, &config); + + assert_eq!(spec["apparmor_profile"].as_str(), Some(expected)); + assert!(spec.get("security_opt").is_none()); + } + } + + #[test] + fn container_spec_omits_podman_apparmor_profile_for_runtime_default() { + let sandbox = test_sandbox("test-id", "test-name"); + + for profile in [None, Some(openshell_core::AppArmorProfile::RuntimeDefault)] { + let mut config = test_config(); + config.app_armor_profile = profile; + let spec = build_container_spec(&sandbox, &config); + + assert!(spec.get("apparmor_profile").is_none()); + assert!(spec.get("security_opt").is_none()); + } } #[test] @@ -1961,7 +2016,8 @@ mod tests { #[test] fn container_spec_healthcheck_accepts_supervisor_socket() { let sandbox = test_sandbox("test-id", "test-name"); - let config = test_config(); + let mut config = test_config(); + config.health_check_interval_secs = std::num::NonZeroU64::new(10); let spec = build_container_spec(&sandbox, &config); let healthcheck = spec["healthconfig"]["test"] @@ -1977,11 +2033,18 @@ mod tests { ); } + #[test] + fn container_spec_omits_healthcheck_when_disabled() { + let sandbox = test_sandbox("test-id", "test-name"); + let spec = build_container_spec(&sandbox, &test_config()); + assert!(spec.get("healthconfig").is_none()); + } + #[test] fn container_spec_healthcheck_interval_from_config() { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); - config.health_check_interval_secs = 30; + config.health_check_interval_secs = std::num::NonZeroU64::new(30); let spec = build_container_spec(&sandbox, &config); let interval = spec["healthconfig"]["Interval"] @@ -2407,7 +2470,7 @@ mod tests { default_image: "test-image:latest".to_string(), grpc_endpoint: "http://localhost:50051".to_string(), host_gateway_ip: String::new(), - sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), ..PodmanComputeConfig::default() } } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 50eb014691..2852c43328 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -4,7 +4,7 @@ //! Podman compute driver. use crate::client::{ContainerListEntry, PodmanApiError, PodmanClient, VolumeInspect}; -use crate::config::PodmanComputeConfig; +use crate::config::{PodmanComputeConfig, podman_image_pull_policy}; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ self, LifecycleEventFences, WatchStream, driver_sandbox_from_inspect, @@ -13,8 +13,9 @@ use crate::watcher::{ use openshell_core::ComputeDriverError; use openshell_core::config::CDI_GPU_DEVICE_ALL; use openshell_core::driver_utils::{ - SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + GatewayCallbackTopology, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, + gateway_callback_endpoint, supervisor_image_should_refresh, temp_extract_container_name, + validate_linux_elf_binary, write_cache_binary_atomic, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -367,15 +368,10 @@ impl PodmanComputeDriver { } } - // Validate TLS configuration before connecting. Partial configs - // (e.g. CA set but cert/key missing) are rejected early so operators - // get a clear error instead of a silent fallback to plaintext HTTP. - config.validate_tls_config()?; - config.validate_runtime_limits()?; - config.validate_host_gateway_ip()?; - config.validate_proxy_config()?; - config.canonicalize_userns()?; - config.validate_userns_mappings()?; + // Validate and normalize configuration before connecting. Partial TLS + // and invalid resource, proxy, SPIFFE, AppArmor, or userns settings + // fail before the runtime is contacted. + config.validate_configuration()?; let client = PodmanClient::new(socket_path); @@ -412,11 +408,16 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } + validate_apparmor_support( + config.app_armor_profile.as_ref(), + info.host.security.apparmor_enabled, + )?; info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, rootless = info.host.security.rootless, rootless_network_cmd = %info.host.rootless_network_cmd, + apparmor_enabled = info.host.security.apparmor_enabled, "Connected to Podman" ); (info.host.security.rootless, info.host.rootless_network_cmd) @@ -438,14 +439,10 @@ impl PodmanComputeDriver { // Auto-detect the gRPC callback endpoint before deciding whether this // topology needs the Podman bridge gateway address. if config.grpc_endpoint.is_empty() { - let scheme = if config.tls_enabled() { - "https" - } else { - "http" - }; - config.grpc_endpoint = format!( - "{scheme}://host.containers.internal:{}", - config.gateway_port + config.grpc_endpoint = gateway_callback_endpoint( + GatewayCallbackTopology::Podman, + config.gateway_port, + config.tls_enabled(), ); info!( grpc_endpoint = %config.grpc_endpoint, @@ -806,7 +803,7 @@ impl PodmanComputeDriver { .to_string(), )); } - let pull_policy = self.config.image_pull_policy.as_str(); + let pull_policy = podman_image_pull_policy(self.config.image_pull_policy); info!(image = %image, policy = %pull_policy, "Ensuring sandbox image"); self.client .pull_image(image, pull_policy) @@ -1402,6 +1399,26 @@ impl PodmanComputeDriver { } } +fn validate_apparmor_support( + profile: Option<&openshell_core::AppArmorProfile>, + apparmor_enabled: bool, +) -> Result<(), PodmanApiError> { + let requires_apparmor = matches!( + profile, + Some( + openshell_core::AppArmorProfile::RuntimeDefault + | openshell_core::AppArmorProfile::Localhost(_) + ) + ); + if requires_apparmor && !apparmor_enabled { + return Err(PodmanApiError::InvalidInput( + "app_armor_profile requires AppArmor, but Podman reports AppArmor is unavailable; install/enable AppArmor or use Unconfined explicitly" + .to_string(), + )); + } + Ok(()) +} + fn supervisor_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" @@ -2218,6 +2235,26 @@ mod tests { ); } + #[test] + fn confined_apparmor_profiles_follow_podman_capability() { + use openshell_core::AppArmorProfile; + + for profile in [ + AppArmorProfile::RuntimeDefault, + AppArmorProfile::Localhost("openshell-supervisor".to_string()), + ] { + validate_apparmor_support(Some(&profile), true) + .expect("confined profile should be accepted when Podman reports AppArmor"); + let error = validate_apparmor_support(Some(&profile), false) + .expect_err("confined profile must fail when AppArmor is unavailable"); + assert!(error.to_string().contains("AppArmor is unavailable")); + } + validate_apparmor_support(Some(&AppArmorProfile::Unconfined), false) + .expect("Unconfined does not require AppArmor support"); + validate_apparmor_support(None, false) + .expect("an omitted profile preserves Podman's runtime behavior"); + } + #[test] #[cfg(target_os = "linux")] fn rootless_pasta_requests_default_route_interface() { diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 4c42ba9699..e4554602f4 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -5,15 +5,13 @@ use clap::Parser; use miette::{IntoDiagnostic, Result}; use std::future::Future; use std::net::SocketAddr; +use std::num::{NonZeroI64, NonZeroU64}; use std::path::PathBuf; use tracing::info; -use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; +use openshell_core::{AppArmorProfile, ImagePullPolicy, VERSION}; +use openshell_driver_podman::config::{DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS}; use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; #[derive(Parser)] @@ -50,7 +48,7 @@ struct Args { #[arg( long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing + default_value_t = ImagePullPolicy::IfNotPresent )] sandbox_image_pull_policy: ImagePullPolicy, @@ -89,14 +87,19 @@ struct Args { #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] stop_timeout: u32, - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit - /// Podman's runtime/default PID limit. + /// Container cgroup PID limit for sandbox containers. Omit to use + /// `OpenShell`'s 2048-process default. + #[arg(long, env = "OPENSHELL_SANDBOX_PIDS_LIMIT", default_value = "2048")] + sandbox_pids_limit: Option, + + /// Health check interval in seconds. Omit it in gateway TOML to disable + /// health checks; the standalone driver keeps its prior 10-second default. #[arg( long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + env = "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + default_value = "10" )] - sandbox_pids_limit: i64, + health_check_interval_secs: Option, /// OCI image containing the openshell-sandbox supervisor binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] @@ -114,6 +117,14 @@ struct Args { #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] podman_tls_key: Option, + /// Host UNIX socket projected into supervisors for provider SPIFFE token exchange. + #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] + provider_spiffe_workload_api_socket: Option, + + /// `AppArmor` model: `RuntimeDefault`, `Unconfined`, or `Localhost/`. + #[arg(long, env = "OPENSHELL_APP_ARMOR_PROFILE")] + app_armor_profile: Option, + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, /// in explicit `http://host:port` form (scheme and port required). /// Credentials must not be embedded in the URL; use @@ -193,7 +204,7 @@ async fn main() -> Result<()> { host_gateway_ip: args .host_gateway_ip .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, supervisor_image: args @@ -202,7 +213,10 @@ async fn main() -> Result<()> { guest_tls_ca: args.podman_tls_ca, guest_tls_cert: args.podman_tls_cert, guest_tls_key: args.podman_tls_key, + provider_spiffe_workload_api_socket: args.provider_spiffe_workload_api_socket, + app_armor_profile: args.app_armor_profile, sandbox_pids_limit: args.sandbox_pids_limit, + health_check_interval_secs: args.health_check_interval_secs, https_proxy: args.sandbox_https_proxy, no_proxy: args.sandbox_no_proxy, proxy_auth_file: args.sandbox_proxy_auth_file, @@ -213,7 +227,6 @@ async fn main() -> Result<()> { uidmap: args.uidmap, gidmap: args.gidmap, enable_bind_mounts: args.enable_bind_mounts, - ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; @@ -309,4 +322,25 @@ mod tests { ); assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); } + + #[test] + fn standalone_defaults_preserve_health_checks_and_reject_zero_limits() { + let defaults = Args::try_parse_from(["openshell-driver-podman"]) + .expect("standalone driver defaults should parse"); + assert_eq!( + defaults.health_check_interval_secs.map(NonZeroU64::get), + Some(10) + ); + assert_eq!( + defaults.sandbox_pids_limit.map(NonZeroI64::get), + Some(openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT) + ); + + for flag in ["--sandbox-pids-limit", "--health-check-interval-secs"] { + let result = Args::try_parse_from(["openshell-driver-podman", flag, "0"]); + assert!(result.is_err(), "zero must be rejected for {flag}"); + let error = result.err().expect("error was asserted above"); + assert!(error.to_string().contains("invalid value"), "flag: {flag}"); + } + } } diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index c3903a0067..b8b6827394 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -439,7 +439,12 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { Some(HealthState { status }) if status == "starting" => { ("False", "HealthCheckStarting", String::new()) } - _ => ("False", CONDITION_STARTING, String::new()), + None => ( + "True", + CONDITION_RUNNING, + "Container is running".to_string(), + ), + Some(_) => ("False", CONDITION_STARTING, String::new()), }, "created" => ("False", "ContainerCreated", String::new()), "exited" | "stopped" => { @@ -578,6 +583,44 @@ mod tests { assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); } + #[test] + fn condition_running_without_healthcheck_is_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: None, + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let cond = condition_from_state(&state); + assert_eq!(cond.r#type, "Ready"); + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, CONDITION_RUNNING); + assert_eq!(cond.message, "Container is running"); + assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); + } + + #[test] + fn condition_running_with_pending_healthcheck_is_not_ready() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: Some(HealthState { + status: "starting".to_string(), + }), + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let condition = condition_from_state(&state); + assert_eq!(condition.r#type, "Ready"); + assert_eq!(condition.status, "False"); + assert_eq!(condition.reason, "HealthCheckStarting"); + } + #[test] fn condition_oom_killed() { let state = ContainerState { diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 5c61ae1823..a83a1df103 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -113,22 +113,23 @@ codesign \ mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm cat > .cache/gateway-vm/gateway.toml <` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | +| `grpc_endpoint` | topology-derived | Optional override for the URL the sandbox guest dials to reach the gateway. The gateway derives `http(s)://host.openshell.internal:` when absent. Use `host.containers.internal`, `host.docker.internal`, or another routable host only for a non-standard topology. Loopback URLs are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | | `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | @@ -151,17 +152,22 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | -| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend (GPU sandboxes) has no such NAT and its nftables rules expose only the gateway port to the guest, so a gateway-host proxy URL is rejected at launch there; use an address routable from the guest's masqueraded egress. | +| `sandbox_uid` / `sandbox_gid` | image `sandbox` account, otherwise `1000` / UID | Explicit values override the image account; when both are omitted, a supplied image `sandbox` account is preserved and an image without one gets `1000:1000`. Each overlay records its effective UID/GID. During migration, an unmarked overlay recovers identity from its upper layer or prepared rootfs, an explicit override, or the current image. Legacy `10001:10001` is retained only when persisted state reports it. | +| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend has no such NAT, so a gateway-host proxy URL is rejected before GPU sandbox launch; use an address routable from the guest's masqueraded egress. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | -| `proxy_auth_file` | unset | Gateway-host path to a `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox. | +| `proxy_auth_file` | unset | Gateway-host path to a validated `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox; credentials never enter logs or process arguments. | | `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | -| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. | -| `proxy_ca_bundle` | unset | Gateway-host path to a PEM CA bundle trusted for an `https://` proxy and for certificates a TLS-intercepting proxy re-signs. | +| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP CONNECT targets. | +| `proxy_ca_bundle` | unset | Gateway-host PEM CA bundle trusted for the corporate proxy and TLS-intercepted server certificates. The driver validates it and stages it at a fixed non-secret guest path in the protected overlay. Requires `https_proxy`. | +| `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. | -The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. +The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor through a protected per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. + +For gateway-managed VM drivers, configure `guest_tls_ca`, `guest_tls_cert`, and +`guest_tls_key` together under `[openshell.gateway]`; the gateway validates and +injects that bundle into only the selected local driver. The standalone +`openshell-driver-vm` CLI retains its `--guest-tls-*` inputs for independent +operation. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -257,7 +263,7 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain -- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection +- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation, identity inspection, and QEMU environment injection. Explicit `sandbox_uid`/`sandbox_gid` values do not remove this runtime prerequisite. - Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. @@ -282,8 +288,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical On Debian-family Linux amd64 and arm64 systems, `install.sh` installs the Debian package from the selected `OPENSHELL_VERSION` release tag. That package includes `openshell-gateway` and `openshell-driver-vm`, but leaves -`OPENSHELL_DRIVERS` unset so the gateway uses its normal runtime -auto-detection. Set `OPENSHELL_DRIVERS=vm` to force the VM driver. +`OPENSHELL_COMPUTE_DRIVER` unset so the gateway uses its normal runtime +auto-detection. Set `OPENSHELL_COMPUTE_DRIVER=vm` to force the VM driver. On RPM-family Linux x86_64 and aarch64 systems, `install.sh` installs the `openshell` and `openshell-gateway` RPM packages from the selected release tag. @@ -294,7 +300,7 @@ formula from the selected release in the `nvidia/openshell` Homebrew tap. Homebrew installs `openshell`, `openshell-gateway`, and `openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless +also leaves `OPENSHELL_COMPUTE_DRIVER` unset so driver choice remains automatic unless the user explicitly overrides it. ## TODOs diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 32d6ed1dff..df8fc3dad6 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -29,7 +29,6 @@ BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) GVPROXY_GATEWAY_IP="192.168.127.1" GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" GATEWAY_IP="$GVPROXY_GATEWAY_IP" -SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized" GPU_ENABLED="${GPU_ENABLED:-false}" VM_NET_IP="${VM_NET_IP:-}" @@ -90,7 +89,9 @@ sandbox_owner_from_passwd() { done < "$passwd_path" fi - printf '10001:10001\n' + # New images use the conventional sandbox UID. Existing images retain the + # sandbox account read above, including the legacy 10001:10001 identity. + printf '1000:1000\n' } source_overlay_env_if_present() { @@ -103,6 +104,21 @@ source_overlay_env_if_present() { ensure_target_runtime() { local image_root="$1" + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local replace_account=0 + + # An omitted identity means the image owns its sandbox account contract. + # Fall back to 1000 only when the image has no sandbox account at all. + if [ -n "$sandbox_uid" ] || [ -n "$sandbox_gid" ]; then + sandbox_uid="${sandbox_uid:-1000}" + sandbox_gid="${sandbox_gid:-$sandbox_uid}" + replace_account=1 + elif ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sandbox_uid=1000 + sandbox_gid=1000 + replace_account=1 + fi mkdir -p \ "$image_root/srv" \ @@ -119,31 +135,31 @@ ensure_target_runtime() { fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" - if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then - printf 'sandbox:x:10001:\n' >> "$image_root/etc/group" - fi - if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then - printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" - fi - if ! grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then - printf 'sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n' >> "$image_root/etc/passwd" - fi - if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then - printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + if [ "$replace_account" -eq 1 ]; then + if grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_gid}:|" "$image_root/etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$image_root/etc/group" + fi + if ! grep -q '^sandbox:' "$image_root/etc/gshadow" 2>/dev/null; then + printf 'sandbox:!::\n' >> "$image_root/etc/gshadow" + fi + if grep -q '^sandbox:' "$image_root/etc/passwd" 2>/dev/null; then + sed -i "s|^sandbox:.*|sandbox:x:${sandbox_uid}:${sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/sh|" "$image_root/etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$image_root/etc/passwd" + fi + if ! grep -q '^sandbox:' "$image_root/etc/shadow" 2>/dev/null; then + printf 'sandbox:!:20123:0:99999:7:::\n' >> "$image_root/etc/shadow" + fi fi local owner - local owner_normalized=0 owner="$(sandbox_owner_for_root "$image_root")" - if chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 - elif chown -R 10001:10001 "$image_root/sandbox" 2>/dev/null; then - owner_normalized=1 + if ! chown -R "$owner" "$image_root/sandbox" 2>/dev/null; then + ts "FATAL: failed to apply sandbox image ownership (${owner})" + exit 1 fi chmod 0755 "$image_root/sandbox" - if [ "$owner_normalized" -eq 1 ]; then - mkdir -p "$image_root/opt/openshell" - printf '1\n' > "$image_root${SANDBOX_OWNER_NORMALIZED_MARKER}" - fi } prepare_guest_image_rootfs() { @@ -275,16 +291,14 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" \ - "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@" fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$supervisor" "$@" fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ - --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@" fi done @@ -609,6 +623,50 @@ setup_gpu() { fi } +reconcile_sandbox_account() { + local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}" + local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}" + local etc + + [ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0 + [[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || { + ts "FATAL: invalid requested sandbox identity" + exit 1 + } + etc="$(root_path /etc)" + mkdir -p "$etc" + touch "$etc/passwd" "$etc/group" "$etc/shadow" "$etc/gshadow" + if grep -q '^sandbox:' "$etc/group"; then + if ! awk -F: -v OFS=: -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = gid } { print }' \ + "$etc/group" >"$etc/group.openshell"; then + rm -f "$etc/group.openshell" + ts "FATAL: failed to reconcile sandbox group" + exit 1 + fi + mv "$etc/group.openshell" "$etc/group" + else + printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group" + fi + if grep -q '^sandbox:' "$etc/passwd"; then + # Preserve image-owned account metadata (home, shell, and description) + # while restoring the UID/GID contract recorded for this overlay. + if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \ + '$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \ + "$etc/passwd" >"$etc/passwd.openshell"; then + rm -f "$etc/passwd.openshell" + ts "FATAL: failed to reconcile sandbox account" + exit 1 + fi + mv "$etc/passwd.openshell" "$etc/passwd" + else + printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd" + fi + grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow" + grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow" + ts "reconciled sandbox account (${sandbox_uid}:${sandbox_gid})" +} + setup_sandbox_workdir() { local sandbox_dir local owner @@ -617,10 +675,13 @@ setup_sandbox_workdir() { owner="$(sandbox_owner)" mkdir -p "$sandbox_dir" current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)" - if [ "$current_owner" != "$owner" ] \ - || [ ! -f "$(root_path "$SANDBOX_OWNER_NORMALIZED_MARKER")" ]; then + if [ "$owner" = "10001:10001" ]; then + ts "preserving legacy sandbox ownership (10001:10001)" + fi + if [ "$current_owner" != "$owner" ]; then if ! chown -R "$owner" "$sandbox_dir" 2>/dev/null; then - chown -R 10001:10001 "$sandbox_dir" + ts "FATAL: failed to apply sandbox ownership (${owner})" + exit 1 fi fi chmod 0755 "$sandbox_dir" @@ -747,6 +808,7 @@ run_post_overlay_setup() { echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true fi + reconcile_sandbox_account setup_sandbox_workdir configure_hostname @@ -897,12 +959,13 @@ if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then fi read_supervisor_extra_args +set -- --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then - exec_supervisor_in_newroot + exec_supervisor_in_newroot "$@" fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" +exec /opt/openshell/bin/openshell-sandbox "$@" } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 896daa48a5..9f68b80608 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11,6 +11,7 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, + sandbox_guest_user_ids_from_image, sandbox_guest_user_ids_from_overlay_image, set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; @@ -30,6 +31,7 @@ use oci_client::manifest::{ }; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; +use openshell_core::UpstreamProxyConfig; use openshell_core::gpu::{ driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, }; @@ -190,6 +192,9 @@ const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4"; const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates"; const OVERLAY_TEMPLATE_CACHE_LAYOUT_VERSION: &str = "sandbox-overlay-ext4-v1"; const SANDBOX_OVERLAY_IMAGE: &str = "overlay.ext4"; +const SANDBOX_OWNER_STATE_FILE: &str = "sandbox-owner-state"; +const SANDBOX_OWNER_STATE_V1: &str = "sandbox-owner-v1"; +const SANDBOX_OWNER_STATE_V2: &str = "sandbox-owner-v2"; const SANDBOX_REQUEST_FILE: &str = "sandbox.pb"; const SANDBOX_STOPPED_FILE: &str = "stopped"; /// Durable tombstone preventing driver restart from relaunching a sandbox @@ -205,6 +210,7 @@ const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; const IMAGE_PREP_INIT_MODE: &str = "image-prep"; static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0); +static OWNER_STATE_WRITE_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone)] struct VmDriverTlsPaths { @@ -241,8 +247,9 @@ enum GuestImagePayloadSource { } #[derive(Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct VmDriverConfig { - pub openshell_endpoint: String, + pub grpc_endpoint: String, pub state_dir: PathBuf, pub launcher_bin: Option, pub default_image: String, @@ -255,80 +262,30 @@ pub struct VmDriverConfig { pub guest_tls_ca: Option, pub guest_tls_cert: Option, pub guest_tls_key: Option, + /// Corporate forward proxy settings delivered to the guest init script. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + /// Gateway-host PEM CA bundle staged into the guest overlay for the + /// corporate proxy and TLS-intercepted server certificates. + pub proxy_ca_bundle: Option, + /// Guest-reachable SPIFFE Workload API TCP endpoint. A VM cannot safely + /// project a host UNIX socket; this must be a deliberately exposed TCP + /// listener and requires `provider_spiffe_allow_guest_tcp`. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - /// Resolved sandbox UID for rootfs `/etc/passwd` entry. - /// When empty, defaults to 10001 (the legacy hardcoded value). + /// Optional UID override for the sandbox account in newly prepared rootfs images. + /// When both identity fields are empty, an image-provided sandbox account is preserved. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, - /// Resolved sandbox GID for rootfs `/etc/passwd` and `/etc/group` entries. - /// When empty, defaults to the resolved UID. + /// Optional GID override for rootfs `/etc/passwd` and `/etc/group` entries. + /// When one override is supplied, its missing counterpart defaults to the UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, - /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) - /// passed to the in-guest supervisor. - /// - /// The supervisor chains policy-approved TLS tunnels through this proxy - /// with HTTP CONNECT instead of dialing destinations directly. This is an - /// operator-owned egress boundary: it travels on the supervisor's argv, - /// which sandbox spec/template environment and image `ENV` cannot - /// influence. A proxy on the gateway host's loopback is reachable from the - /// guest only through the gvproxy host alias - /// (`host.openshell.internal`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub https_proxy: Option, - - /// Comma-separated `NO_PROXY` list passed alongside the proxy URL. - /// - /// Matching destinations are dialed directly instead of through the - /// corporate proxy. This bypasses only the corporate proxy, never - /// `OpenShell` policy evaluation. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub no_proxy: Option, - - /// Path (on the gateway host) to a file containing the corporate proxy - /// credential in `user:pass` form. - /// - /// The driver validates it at sandbox-create time and stages it into the - /// per-sandbox overlay at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], root-only. - /// Credentials are never embedded in the proxy URL and never reach the - /// guest environment. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_auth_file: Option, - - /// Explicit acknowledgement that proxy credentials are sent in cleartext. - /// - /// `Proxy-Authorization: Basic` over the plain-TCP connection to an - /// `http://` proxy is recoverable by anyone on the network path, so - /// [`Self::proxy_auth_file`] requires this acknowledgement. An `https://` - /// proxy carries the credential inside the verified TLS session and does - /// not need it. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_auth_allow_insecure: Option, - - /// Send the destination hostname in CONNECT requests instead of a - /// validated IP. - /// - /// The default binds the tunnel to an address that passed the sandbox's - /// SSRF and `allowed_ips` validation. Set this only when the proxy's ACLs - /// filter on hostnames and reject IP CONNECT targets: the proxy then - /// resolves the name itself and its own ACLs become the effective egress - /// control for proxied TLS. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_connect_by_hostname: Option, - - /// Path (on the gateway host) to a PEM CA bundle trusted for the - /// corporate proxy. - /// - /// The driver stages it into the per-sandbox overlay at - /// [`GUEST_PROXY_CA_PATH`] and passes that path via - /// `--upstream-proxy-ca-bundle`. It is trusted both for the handshake - /// with an `https://` proxy and for server certificates re-signed by a - /// TLS-intercepting proxy. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub proxy_ca_bundle: Option, /// Directory where rootfs tar files must be staged before they can be /// referenced in a `CreateSandbox` request. Defaults to `/rootfs-tar-staging`. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -345,7 +302,7 @@ pub struct VmDriverConfig { impl std::fmt::Debug for VmDriverConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("VmDriverConfig") - .field("openshell_endpoint", &self.openshell_endpoint) + .field("grpc_endpoint", &self.grpc_endpoint) .field("state_dir", &self.state_dir) .field("launcher_bin", &self.launcher_bin) .field("default_image", &self.default_image) @@ -363,25 +320,52 @@ impl std::fmt::Debug for VmDriverConfig { .field("gpu_vcpus", &self.gpu_vcpus) .field("sandbox_uid", &self.sandbox_uid) .field("sandbox_gid", &self.sandbox_gid) - .field("https_proxy", &self.https_proxy.is_some()) - .field("no_proxy", &self.no_proxy) - .field("proxy_auth_file", &self.proxy_auth_file.is_some()) - .field("proxy_auth_allow_insecure", &self.proxy_auth_allow_insecure) - .field("proxy_connect_by_hostname", &self.proxy_connect_by_hostname) - .field("proxy_ca_bundle", &self.proxy_ca_bundle) + .field( + "upstream_proxy_configured", + &self.upstream_proxy.https_proxy.is_some(), + ) + .field( + "no_proxy_configured", + &self.upstream_proxy.no_proxy.is_some(), + ) + .field( + "proxy_auth_file_configured", + &self.upstream_proxy.proxy_auth_file.is_some(), + ) + .field( + "proxy_auth_allow_insecure", + &self.upstream_proxy.proxy_auth_allow_insecure, + ) + .field( + "proxy_connect_by_hostname", + &self.upstream_proxy.proxy_connect_by_hostname, + ) + .field( + "proxy_ca_bundle_configured", + &self.proxy_ca_bundle.is_some(), + ) + .field( + "provider_spiffe_workload_api_tcp_endpoint_configured", + &self.provider_spiffe_workload_api_tcp_endpoint.is_some(), + ) + .field( + "provider_spiffe_allow_guest_tcp", + &self.provider_spiffe_allow_guest_tcp, + ) .field("rootfs_tar_staging_dir", &self.rootfs_tar_staging_dir) .field("rootfs_tar_max_bytes", &self.rootfs_tar_max_bytes) .finish() } } -/// Default sandbox UID used by the VM driver when no config value is set. -pub const DEFAULT_SANDBOX_UID: u32 = 10001; +/// Fallback sandbox UID for images without a `sandbox` account and partial +/// operator identity overrides. +pub const DEFAULT_SANDBOX_UID: u32 = 1000; impl Default for VmDriverConfig { fn default() -> Self { Self { - openshell_endpoint: String::new(), + grpc_endpoint: String::new(), state_dir: PathBuf::from("target/openshell-vm-driver"), launcher_bin: None, default_image: String::new(), @@ -394,17 +378,15 @@ impl Default for VmDriverConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, + upstream_proxy: UpstreamProxyConfig::default(), + proxy_ca_bundle: None, + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, sandbox_uid: None, sandbox_gid: None, - https_proxy: None, - no_proxy: None, - proxy_auth_file: None, - proxy_auth_allow_insecure: None, - proxy_connect_by_hostname: None, - proxy_ca_bundle: None, rootfs_tar_staging_dir: None, rootfs_tar_max_bytes: None, } @@ -412,16 +394,51 @@ impl Default for VmDriverConfig { } impl VmDriverConfig { - /// Resolve the sandbox UID, falling back to `DEFAULT_SANDBOX_UID`. + /// Resolve a fallback sandbox UID for an image that has no sandbox account. pub fn resolve_sandbox_uid(&self) -> u32 { self.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID) } - /// Resolve the sandbox GID, falling back to the resolved UID. + /// Resolve a fallback sandbox GID from the selected UID. pub fn resolve_sandbox_gid(&self, resolved_uid: u32) -> u32 { self.sandbox_gid.unwrap_or(resolved_uid) } + pub fn validate_runtime_security_config(&self) -> Result<(), String> { + self.upstream_proxy.validate()?; + if let Some(path) = self.proxy_ca_bundle.as_ref() { + if path.as_os_str().is_empty() { + return Err("proxy_ca_bundle must not be empty when set".to_string()); + } + if self.upstream_proxy.https_proxy.is_none() { + return Err("proxy_ca_bundle is set but no https_proxy is configured".to_string()); + } + } + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + )?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err("provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured".to_string()); + } + Ok(()) + } + + pub fn validate_rootfs_tar_config(&self) -> Result<(), String> { + if self + .rootfs_tar_staging_dir + .as_ref() + .is_some_and(|path| path.as_os_str().is_empty()) + { + return Err("rootfs_tar_staging_dir must not be empty when set".to_string()); + } + if self.rootfs_tar_max_bytes == Some(0) { + return Err("rootfs_tar_max_bytes must be greater than zero when set".to_string()); + } + Ok(()) + } + pub fn validate_sandbox_identity(&self) -> Result<(), String> { let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; if let Some(uid) = self.sandbox_uid @@ -445,29 +462,6 @@ impl VmDriverConfig { Ok(()) } - /// Validate the operator's corporate upstream-proxy settings, fail-closed. - /// - /// Delegates to the validator shared with the Podman and Kubernetes - /// drivers and with the in-guest supervisor, so a value accepted here is - /// never rejected inside the guest — and no misconfiguration can silently - /// degrade to a direct dial. - /// - /// # Errors - /// - /// Returns a message naming the offending key. - pub fn validate_proxy_config(&self) -> Result<(), String> { - openshell_core::driver_utils::validate_upstream_proxy_settings( - &openshell_core::driver_utils::UpstreamProxySettings { - url: self.https_proxy.as_deref(), - no_proxy: self.no_proxy.as_deref(), - auth_file: self.proxy_auth_file.as_deref(), - auth_allow_insecure: self.proxy_auth_allow_insecure, - connect_by_hostname: self.proxy_connect_by_hostname, - ca_bundle: self.proxy_ca_bundle.as_deref(), - }, - ) - } - fn rootfs_tar_staging_dir(&self) -> PathBuf { self.rootfs_tar_staging_dir .clone() @@ -480,7 +474,7 @@ impl VmDriverConfig { } fn requires_tls_materials(&self) -> bool { - self.openshell_endpoint.starts_with("https://") + self.grpc_endpoint.starts_with("https://") } fn tls_paths(&self) -> Result, String> { @@ -573,6 +567,25 @@ enum OverlayPreparation { PreserveExisting, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SandboxOwnerIdentity { + uid: u32, + gid: u32, +} + +impl SandboxOwnerIdentity { + fn guest_environment(self) -> [String; 2] { + [ + format!("OPENSHELL_VM_SANDBOX_UID={}", self.uid), + format!("OPENSHELL_VM_SANDBOX_GID={}", self.gid), + ] + } + + fn marker_contents(self) -> String { + format!("{SANDBOX_OWNER_STATE_V2}:{}:{}\n", self.uid, self.gid) + } +} + fn provisioning_span( parent: &opentelemetry::Context, sandbox_id: &str, @@ -619,11 +632,12 @@ impl VmDriver { .validate() .map_err(|err| err.message().to_string())?; config.validate_sandbox_identity()?; - config.validate_proxy_config()?; - if config.openshell_endpoint.trim().is_empty() { + config.validate_runtime_security_config()?; + config.validate_rootfs_tar_config()?; + if config.grpc_endpoint.trim().is_empty() { return Err("openshell endpoint is required".to_string()); } - validate_openshell_endpoint(&config.openshell_endpoint)?; + validate_openshell_endpoint(&config.grpc_endpoint)?; let _ = config.tls_paths()?; #[cfg(target_os = "linux")] @@ -1052,6 +1066,7 @@ impl VmDriver { let disk_paths = sandbox_runtime_disk_paths(&state_dir); let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; + let owner_source_disk = image_disk.as_ref().unwrap_or(&root_disk).clone(); let overlay_disk = disk_paths.overlay_disk; self.publish_platform_event( @@ -1063,9 +1078,11 @@ impl VmDriver { "Preparing writable VM overlay disk".to_string(), ), ); - if let Err(err) = self + let sandbox_owner_state = self .prepare_runtime_overlay( + &state_dir, &overlay_disk, + &owner_source_disk, tls_paths.as_ref(), sandbox .spec @@ -1075,11 +1092,7 @@ impl VmDriver { overlay_preparation, ) .await - { - return Err(Status::internal(format!( - "prepare guest overlay disk failed: {err}" - ))); - } + .map_err(|err| Status::internal(format!("prepare guest overlay disk failed: {err}")))?; self.ensure_provisioning_active(&sandbox.id).await?; if let Err(err) = @@ -1226,7 +1239,7 @@ impl VmDriver { let endpoint_override = if plan.backend == VmBackend::Qemu { plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) + guest_visible_openshell_endpoint_for_tap(&self.config.grpc_endpoint, host_ip) }) } else { None @@ -1290,6 +1303,9 @@ impl VmDriver { for env in &plan.env { command.arg("--vm-env").arg(env); } + for env in sandbox_owner_state.guest_environment() { + command.arg("--vm-env").arg(env); + } info!( sandbox_id = %sandbox.id, @@ -2033,7 +2049,7 @@ impl VmDriver { "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + plan.gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); } // The corporate-proxy host-loopback recipe is a libkrun/gvproxy @@ -2043,7 +2059,7 @@ impl VmDriver { // compare against is this sandbox's own TAP host address. Fail the // create with the reason rather than boot a sandbox whose // policy-approved CONNECTs all time out against an unreachable proxy. - if let Some(url) = self.config.https_proxy.as_deref() + if let Some(url) = self.config.upstream_proxy.https_proxy.as_deref() && proxy_url_targets_gateway_host(url, plan.host_ip.as_deref()) { let tap_host = plan.host_ip.as_deref().unwrap_or("the TAP host address"); @@ -2186,7 +2202,7 @@ impl VmDriver { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + let gateway_port = gateway_port_from_endpoint(&self.config.grpc_endpoint); let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) @@ -2401,17 +2417,20 @@ impl VmDriver { )] async fn prepare_runtime_overlay( &self, + state_dir: &Path, overlay_disk: &Path, + owner_source_disk: &Path, tls_paths: Option<&VmDriverTlsPaths>, sandbox_token: Option<&str>, preparation: OverlayPreparation, - ) -> Result<(), String> { + ) -> Result { let span_status = openshell_otel::ErrorStatusGuard::current(); let tls_materials = match tls_paths { Some(paths) => Some(read_guest_tls_materials(paths).await?), None => None, }; let sandbox_token = sandbox_token.map(str::to_string); + let proxy_auth = self.read_proxy_auth_credential().await?; let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2423,6 +2442,22 @@ impl VmDriver { self.config.overlay_disk_mib ) })?; + let (owner_state, write_owner_state) = sandbox_owner_state_for_launch( + state_dir, + &overlay_disk, + owner_source_disk, + &self.config, + preparation, + ) + .await?; + let owner_state_written_before_prepare = + write_owner_state && preparation == OverlayPreparation::Fresh; + if owner_state_written_before_prepare { + // Persist the selected identity before creating the overlay. A + // crash during preparation can then retry without misclassifying + // the partial overlay as legacy state. + write_sandbox_owner_state(state_dir, owner_state).await?; + } let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes); if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? { @@ -2441,13 +2476,38 @@ impl VmDriver { &overlay_disk, tls_materials.as_ref(), sandbox_token.as_deref(), + proxy_auth.as_deref(), preparation, overlay_size_bytes, ) }) .await .map_err(|err| format!("overlay image preparation panicked: {err}"))?; - span_status.finish(result) + result?; + if write_owner_state && !owner_state_written_before_prepare { + write_sandbox_owner_state(state_dir, owner_state).await?; + } + span_status.finish(Ok(owner_state)) + } + + async fn read_proxy_auth_credential(&self) -> Result, String> { + let Some(path) = self.config.upstream_proxy.proxy_auth_file.as_ref() else { + return Ok(None); + }; + let path = path.clone(); + Ok(Some( + tokio::task::spawn_blocking(move || { + let path = path + .to_str() + .ok_or_else(|| "proxy_auth_file must be valid UTF-8".to_string())?; + let raw = openshell_core::driver_utils::read_upstream_proxy_credential_file(path)?; + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) + .map(str::to_owned) + .map_err(|error| format!("proxy_auth_file is invalid: {error}")) + }) + .await + .map_err(|error| format!("proxy_auth_file read task failed: {error}"))??, + )) } fn resolved_sandbox_image(&self, sandbox: &Sandbox) -> Option { @@ -2824,7 +2884,7 @@ impl VmDriver { image_identity: &str, bootstrap_root_disk: &Path, ) -> Result { - let cache_identity = prepared_image_cache_identity(image_identity); + let cache_identity = prepared_image_cache_identity(image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -2924,7 +2984,7 @@ impl VmDriver { ))); } }; - let cache_identity = rootfs_tar_cache_identity(&source_digest); + let cache_identity = rootfs_tar_cache_identity(&source_digest, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); let tar_display = tar_path.display().to_string(); @@ -3078,7 +3138,7 @@ impl VmDriver { "failed to resolve vm sandbox image '{image_ref}': {err}" )) })?; - let cache_identity = prepared_image_cache_identity(&source_image_identity); + let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config); let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity); if tokio::fs::metadata(&image_path).await.is_ok() { @@ -3281,6 +3341,7 @@ impl VmDriver { Ok(()) } + #[allow(clippy::similar_names)] async fn run_image_prep_vm( &self, bootstrap_root_disk: &Path, @@ -3309,6 +3370,14 @@ impl VmDriver { command .arg("--vm-env") .arg(format!("OPENSHELL_VM_INIT_MODE={IMAGE_PREP_INIT_MODE}")); + if let Some((uid, gid)) = configured_sandbox_identity(&self.config) { + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_UID={uid}")); + command + .arg("--vm-env") + .arg(format!("OPENSHELL_VM_SANDBOX_GID={gid}")); + } let mut child = command .spawn() @@ -3426,19 +3495,20 @@ impl VmDriver { let image_identity_owned = image_identity.to_string(); let exported_rootfs_for_build = exported_rootfs.clone(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!( - "Preparing VM rootfs for local image \"{image_ref}\" (sandbox uid={sandbox_uid})" - ), + format!("Preparing VM rootfs for local image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "local_docker".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -3567,17 +3637,20 @@ impl VmDriver { let image_ref_owned = image_ref.to_string(); let image_identity_owned = image_identity.to_string(); let prepared_rootfs_for_build = prepared_rootfs.clone(); - let sandbox_uid = self.config.resolve_sandbox_uid(); - let sandbox_gid = self.config.resolve_sandbox_gid(sandbox_uid); + let (sandbox_uid, sandbox_gid) = configured_sandbox_identity(&self.config) + .map_or((None, None), |(uid, gid)| (Some(uid), Some(gid))); self.publish_vm_progress( sandbox_id, "PreparingRootfs", - format!("Preparing VM rootfs for image \"{image_ref}\" (sandbox uid={sandbox_uid})"), + format!("Preparing VM rootfs for image \"{image_ref}\""), HashMap::from([ ("image_ref".to_string(), image_ref.to_string()), ("image_source".to_string(), "registry".to_string()), ("image_identity".to_string(), image_identity.to_string()), - ("sandbox_uid".to_string(), sandbox_uid.to_string()), + ( + "sandbox_uid".to_string(), + sandbox_uid.map_or_else(|| "image".to_string(), |uid| uid.to_string()), + ), ]), ); let prepare_result = tokio::task::spawn_blocking(move || { @@ -4932,8 +5005,8 @@ fn write_stream_to_file(mut reader: impl Read, dst: &Path, max_bytes: u64) -> Re /// seconds-truncated mtime cannot distinguish two writes within the same /// second. A fixed-length digest also keeps the cache directory name inside /// filesystem component limits regardless of how long the source path was. -fn rootfs_tar_cache_identity(digest: &str) -> String { - prepared_image_cache_identity(&format!("rootfs-tar:sha256:{digest}")) +fn rootfs_tar_cache_identity(digest: &str, config: &VmDriverConfig) -> String { + prepared_image_cache_identity(&format!("rootfs-tar:sha256:{digest}"), config) } fn extract_layer_blob_to_dir( @@ -5156,7 +5229,7 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { /// Rewrites loopback host references in a gateway URL to a hostname the guest /// can reach via gvproxy. /// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which +/// The driver receives the gateway endpoint from `--grpc-endpoint`, which /// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is /// useless inside the guest because the guest's loopback interface is its own, /// not the host's. Inside the guest we need a name that gvproxy will translate @@ -5267,7 +5340,7 @@ fn build_guest_environment( endpoint_override: Option<&str>, ) -> Vec { let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), + || guest_visible_openshell_endpoint(&config.grpc_endpoint), String::from, ); // 1. User-supplied environment (lowest priority). @@ -5336,6 +5409,12 @@ fn build_guest_environment( GUEST_TLS_KEY_PATH.to_string(), ); } + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + environment.insert( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET.to_string(), + endpoint.clone(), + ); + } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), @@ -5419,6 +5498,204 @@ fn sandbox_runtime_disk_paths(state_dir: &Path) -> SandboxRuntimeDiskPaths { } } +/// Select the exact identity the guest must use for this overlay and whether a +/// successful preparation must create or upgrade its state marker. +/// +/// Persisted overlays are resolved only from concrete state. In particular, +/// absence of a marker is not evidence that an overlay used the historical +/// 10001 identity: it can also mean fresh provisioning was interrupted. +async fn sandbox_owner_state_for_launch( + state_dir: &Path, + overlay_disk: &Path, + owner_source_disk: &Path, + config: &VmDriverConfig, + preparation: OverlayPreparation, +) -> Result<(SandboxOwnerIdentity, bool), String> { + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + match tokio::fs::read_to_string(&marker_path).await { + Ok(contents) if contents.trim() == SANDBOX_OWNER_STATE_V1 => { + // The v1 marker recorded no identity. Resolve it through the same + // evidence-based migration path as an unmarked overlay. + } + Ok(contents) => { + let identity = parse_sandbox_owner_state(&contents).map_err(|error| { + format!( + "invalid sandbox owner state {}: {error}", + marker_path.display() + ) + })?; + return Ok((identity, false)); + } + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!( + "read sandbox owner state {}: {error}", + marker_path.display() + )); + } + Err(_) => {} + } + + let overlay_exists = match tokio::fs::metadata(overlay_disk).await { + Ok(metadata) => metadata.is_file(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "stat overlay disk {}: {error}", + overlay_disk.display() + )); + } + }; + + if preparation == OverlayPreparation::PreserveExisting && overlay_exists { + match sandbox_owner_identity_from_overlay(overlay_disk).await { + Ok(Some(identity)) => return Ok((identity, true)), + Ok(None) => {} + Err(error) => warn!( + overlay_path = %overlay_disk.display(), + error = %error, + "could not read sandbox identity from VM overlay upper layer" + ), + } + if let Some(identity) = persisted_sandbox_owner_identity(state_dir, config).await? { + return Ok((identity, true)); + } + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + return sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)); + } + + if let Some((uid, gid)) = configured_sandbox_identity(config) { + return Ok((SandboxOwnerIdentity { uid, gid }, true)); + } + sandbox_owner_identity_from_image(owner_source_disk) + .await + .map(|identity| (identity, true)) +} + +async fn persisted_sandbox_owner_identity( + state_dir: &Path, + config: &VmDriverConfig, +) -> Result, String> { + let prior_identity = match tokio::fs::read_to_string(state_dir.join(IMAGE_IDENTITY_FILE)).await + { + Ok(identity) if !identity.trim().is_empty() => identity, + Ok(_) => String::new(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(format!("read persisted VM image identity: {error}")), + }; + + if !prior_identity.is_empty() { + let prior_disk = image_cache_rootfs_image(&config.state_dir, prior_identity.trim()); + if tokio::fs::metadata(&prior_disk).await.is_ok() { + match sandbox_owner_identity_from_image(&prior_disk).await { + Ok(identity) => return Ok(Some(identity)), + Err(error) => warn!( + image_path = %prior_disk.display(), + error = %error, + "could not read sandbox identity from persisted VM rootfs; using compatibility fallback" + ), + } + } + } + + Ok(None) +} + +async fn sandbox_owner_identity_from_image( + image_path: &Path, +) -> Result { + let image_path = image_path.to_path_buf(); + let identity = + tokio::task::spawn_blocking(move || sandbox_guest_user_ids_from_image(&image_path)) + .await + .map_err(|error| format!("read sandbox identity task failed: {error}"))??; + let (uid, gid) = identity.unwrap_or((DEFAULT_SANDBOX_UID, DEFAULT_SANDBOX_UID)); + validate_sandbox_owner_identity(uid, gid)?; + Ok(SandboxOwnerIdentity { uid, gid }) +} + +async fn sandbox_owner_identity_from_overlay( + overlay_path: &Path, +) -> Result, String> { + let overlay_path = overlay_path.to_path_buf(); + let identity = tokio::task::spawn_blocking(move || { + sandbox_guest_user_ids_from_overlay_image(&overlay_path) + }) + .await + .map_err(|error| format!("read sandbox overlay identity task failed: {error}"))??; + identity + .map(|(uid, gid)| { + validate_sandbox_owner_identity(uid, gid)?; + Ok(SandboxOwnerIdentity { uid, gid }) + }) + .transpose() +} + +fn parse_sandbox_owner_state(contents: &str) -> Result { + let mut fields = contents.trim().split(':'); + if fields.next() != Some(SANDBOX_OWNER_STATE_V2) { + return Err("unsupported version".to_string()); + } + let uid = fields + .next() + .ok_or_else(|| "missing uid".to_string())? + .parse::() + .map_err(|error| format!("invalid uid: {error}"))?; + let gid = fields + .next() + .ok_or_else(|| "missing gid".to_string())? + .parse::() + .map_err(|error| format!("invalid gid: {error}"))?; + if fields.next().is_some() { + return Err("unexpected fields".to_string()); + } + validate_sandbox_owner_identity(uid, gid)?; + Ok(SandboxOwnerIdentity { uid, gid }) +} + +async fn write_sandbox_owner_state( + state_dir: &Path, + identity: SandboxOwnerIdentity, +) -> Result<(), String> { + validate_sandbox_owner_identity(identity.uid, identity.gid)?; + let marker_path = state_dir.join(SANDBOX_OWNER_STATE_FILE); + let sequence = OWNER_STATE_WRITE_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary_path = state_dir.join(format!( + ".{SANDBOX_OWNER_STATE_FILE}.{}.{sequence}.tmp", + std::process::id() + )); + write_private_file(&temporary_path, identity.marker_contents().into_bytes()) + .await + .map_err(|err| format!("write temporary sandbox owner state: {err}"))?; + if let Err(error) = tokio::fs::rename(&temporary_path, &marker_path).await { + let _ = tokio::fs::remove_file(&temporary_path).await; + return Err(format!("install sandbox owner state: {error}")); + } + Ok(()) +} + +fn validate_sandbox_owner_identity(uid: u32, gid: u32) -> Result<(), String> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + if !range.contains(&uid) { + return Err(format!( + "uid {uid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + if !range.contains(&gid) { + return Err(format!( + "gid {gid} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + )); + } + Ok(()) +} + #[allow(clippy::result_large_err)] fn validate_sandbox_state_dir(root: &Path, state_dir: &Path) -> Result<(), Status> { let sandboxes_root = sandboxes_root_dir(root); @@ -5576,9 +5853,20 @@ fn bootstrap_image_cache_identity(image_identity: &str) -> String { ) } -fn prepared_image_cache_identity(image_identity: &str) -> String { +fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> { + (config.sandbox_uid.is_some() || config.sandbox_gid.is_some()).then(|| { + let uid = config.sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + (uid, config.sandbox_gid.unwrap_or(uid)) + }) +} + +fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String { + let identity = configured_sandbox_identity(config).map_or_else( + || "image-account".to_string(), + |(uid, gid)| format!("configured-{uid}-{gid}"), + ); format!( - "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", + "{PREPARED_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{identity}:{image_identity}", openshell_core::VERSION ) } @@ -5802,6 +6090,7 @@ fn create_sandbox_overlay_image_from_template( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, ) -> Result<(), String> { clone_or_copy_sparse_file(template_path, overlay_disk)?; if let Some(tls) = tls_materials { @@ -5810,6 +6099,9 @@ fn create_sandbox_overlay_image_from_template( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } Ok(()) } @@ -5818,6 +6110,7 @@ fn prepare_sandbox_overlay_image( overlay_disk: &Path, tls_materials: Option<&GuestTlsMaterials>, sandbox_token: Option<&str>, + proxy_auth: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { @@ -5830,6 +6123,9 @@ fn prepare_sandbox_overlay_image( if let Some(token) = sandbox_token { inject_guest_sandbox_token(overlay_disk, token)?; } + if let Some(credential) = proxy_auth { + inject_guest_proxy_auth(overlay_disk, credential)?; + } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5861,6 +6157,7 @@ fn prepare_sandbox_overlay_image( overlay_disk, tls_materials, sandbox_token, + proxy_auth, ) } @@ -5889,6 +6186,12 @@ fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), St set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) } +fn inject_guest_proxy_auth(overlay_disk: &Path, credential: &str) -> Result<(), String> { + let path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH); + write_rootfs_image_file(overlay_disk, &path, format!("{credential}\n").as_bytes())?; + set_rootfs_image_file_mode(overlay_disk, &path, 0o600) +} + #[allow(clippy::result_large_err)] #[tracing::instrument( name = "vm.prepare_guest", @@ -5944,15 +6247,15 @@ fn inject_guest_init_dropins( /// the supervisor reads the credential from that file. fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { let mut args = Vec::new(); - if let Some(url) = &config.https_proxy { + if let Some(url) = &config.upstream_proxy.https_proxy { args.push("--upstream-proxy".to_string()); args.push(url.clone()); } - if let Some(list) = &config.no_proxy { + if let Some(list) = &config.upstream_proxy.no_proxy { args.push("--upstream-no-proxy".to_string()); args.push(list.clone()); } - if config.proxy_auth_file.is_some() { + if config.upstream_proxy.proxy_auth_file.is_some() { args.push("--upstream-proxy-auth-file".to_string()); // The guest path, never the gateway-host path the operator configured. args.push(GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string()); @@ -5960,16 +6263,17 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { // Config validation guarantees the acknowledgement is `true` whenever an // auth file is configured against an http:// proxy; the supervisor // independently refuses credentials without it. - if config.proxy_auth_allow_insecure == Some(true) { + if config.upstream_proxy.proxy_auth_allow_insecure == Some(true) { args.push("--upstream-proxy-auth-allow-insecure".to_string()); } // Absent means the default validated-IP CONNECT binding; only the // explicit hostname opt-in is passed through. - if config.proxy_connect_by_hostname == Some(true) { + if config.upstream_proxy.proxy_connect_by_hostname == Some(true) { args.push("--upstream-proxy-connect-by-hostname".to_string()); } if config.proxy_ca_bundle.is_some() { args.push("--upstream-proxy-ca-bundle".to_string()); + // The guest path, never the gateway-host path the operator configured. args.push(GUEST_PROXY_CA_PATH.to_string()); } args @@ -6010,40 +6314,47 @@ fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> { /// Uses the validators shared with the supervisor, so a credential accepted /// here is never rejected inside the guest. The error never carries the file /// contents. -async fn read_sandbox_proxy_credential(path: &str) -> Result { - let path_owned = path.to_string(); +async fn read_sandbox_proxy_credential(path: &Path) -> Result { + let path_owned = path.to_path_buf(); + let display_path = path.display().to_string(); let raw = tokio::task::spawn_blocking(move || { - openshell_core::driver_utils::read_upstream_proxy_credential_file(&path_owned) + let path = path_owned + .to_str() + .ok_or_else(|| "proxy_auth_file path is not valid UTF-8".to_string())?; + openshell_core::driver_utils::read_upstream_proxy_credential_file(path) }) .await .map_err(|err| Status::internal(format!("proxy_auth_file read task failed: {err}")))? .map_err(Status::invalid_argument)?; - let credential = openshell_core::driver_utils::parse_upstream_proxy_credential(&raw) - .map_err(|err| Status::invalid_argument(format!("proxy_auth_file '{path}': {err}")))?; + let credential = + openshell_core::driver_utils::parse_upstream_proxy_credential(&raw).map_err(|err| { + Status::invalid_argument(format!("proxy_auth_file '{display_path}': {err}")) + })?; Ok(credential.to_string()) } /// Read and validate the corporate proxy CA bundle from the gateway host. /// -/// Uses the reader shared with the supervisor, so the bundle is bounded and -/// non-regular files are rejected (an operator path such as `/dev/zero` can -/// otherwise exhaust driver memory), and a bundle accepted here contributes at -/// least one trust anchor rustls accepts rather than merely looking like PEM. -/// Checked here rather than only in the guest so the operator gets an error -/// attributable to `proxy_ca_bundle` instead of an opaque supervisor startup -/// failure inside every sandbox. The error never carries the file contents. -async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { - let path_owned = path.to_string(); - let pem = tokio::task::spawn_blocking(move || { - openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file( - &path_owned, - "proxy_ca_bundle", - ) +/// The validation rejects symlinks and non-regular files, bounds the read, +/// requires PEM certificate markers, and never includes file contents in an +/// error. +async fn read_sandbox_proxy_ca_bundle(path: &Path) -> Result, Status> { + let path_owned = path.to_path_buf(); + let display_path = path.display().to_string(); + tokio::task::spawn_blocking(move || { + let path = path_owned + .to_str() + .ok_or_else(|| "proxy_ca_bundle path is not valid UTF-8".to_string())?; + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(path, "proxy_ca_bundle") + .map(String::into_bytes) }) .await .map_err(|err| Status::internal(format!("proxy_ca_bundle read task failed: {err}")))? - .map_err(Status::invalid_argument)?; - Ok(pem.into_bytes()) + .map_err(|err| { + Status::invalid_argument(format!( + "proxy_ca_bundle '{display_path}' could not be read: {err}" + )) + }) } /// Stage the corporate upstream-proxy configuration into the guest overlay. @@ -6056,7 +6367,7 @@ async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { /// * the supervisor argument list at [`GUEST_SUPERVISOR_ARGS_PATH`], mode /// `0644`. /// -/// All three are written on every launch, empty when the corresponding +/// Both are written on every launch, empty when the corresponding /// setting is absent. Writing rather than skipping is what makes the channel /// unforgeable: the upperdir copy always shadows the read-only image layer, so /// a sandbox image cannot supply its own arguments or credential by baking a @@ -6077,7 +6388,7 @@ async fn inject_guest_upstream_proxy( // the operator removed a setting clears material a previous launch staged // into a preserved overlay, and shadows anything an image baked at these // paths, so a staged file is only ever the one this launch produced. - let credential = match config.proxy_auth_file.as_deref() { + let credential = match config.upstream_proxy.proxy_auth_file.as_deref() { Some(path) => format!("{}\n", read_sandbox_proxy_credential(path).await?).into_bytes(), None => Vec::new(), }; @@ -6636,6 +6947,37 @@ mod tests { assert_eq!(attempts.load(Ordering::Relaxed), 3); } + #[test] + fn vm_config_uses_canonical_grpc_endpoint_name() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let serialized = serde_json::to_value(&config).unwrap(); + assert_eq!(serialized["grpc_endpoint"], "http://127.0.0.1:8080"); + assert!(serialized.get("openshell_endpoint").is_none()); + + let parsed: VmDriverConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.grpc_endpoint, "http://127.0.0.1:8080"); + } + + #[test] + fn vm_config_rejects_legacy_openshell_endpoint() { + let config = VmDriverConfig { + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let mut serialized = serde_json::to_value(config).unwrap(); + serialized.as_object_mut().unwrap().insert( + "openshell_endpoint".to_string(), + serde_json::json!("http://127.0.0.1:8080"), + ); + + let error = serde_json::from_value::(serialized) + .expect_err("legacy openshell_endpoint must be rejected as unknown"); + assert!(error.to_string().contains("openshell_endpoint")); + } + struct TestTracing { exporter: opentelemetry_sdk::trace::InMemorySpanExporter, _provider: opentelemetry_sdk::trace::SdkTracerProvider, @@ -7093,7 +7435,14 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay( + Path::new("/unused"), + Path::new("/unused"), + Path::new("/unused"), + None, + None, + OverlayPreparation::Fresh, + ) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -7679,6 +8028,391 @@ mod tests { } } + #[tokio::test] + async fn unmarked_overlay_uses_current_image_instead_of_blind_legacy_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let source = dir.join("current-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let current_rootfs = dir.join("current-rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, ¤t_rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + ¤t_rootfs, + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_without_identity_evidence_fails_safely() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"unreadable partial overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let error = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .expect_err("ambiguous overlay must not receive a guessed identity"); + + assert!(error.contains("missing-current-rootfs")); + assert!(!error.contains("10001")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_overlay_uses_explicit_identity_without_image_inspection() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000 + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn fresh_image_without_sandbox_account_uses_default_identity() { + let dir = unique_temp_dir(); + let source = dir.join("rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write(source.join("etc/passwd"), "root:x:0:0:root:/root:/bin/sh\n").unwrap(); + let rootfs = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&source, &rootfs, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, _) = sandbox_owner_state_for_launch( + &dir, + &dir.join(SANDBOX_OVERLAY_IMAGE), + &rootfs, + &config, + OverlayPreparation::Fresh, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: DEFAULT_SANDBOX_UID, + gid: DEFAULT_SANDBOX_UID, + } + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_legacy_overlay_uses_explicit_config_when_old_rootfs_is_missing() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_owner_marker_rejects_malformed_or_unknown_state() { + for marker in [ + "sandbox-owner-v3:1000:1000", + "sandbox-owner-v2", + "sandbox-owner-v2:nope:1000", + "sandbox-owner-v2:0:1000", + "sandbox-owner-v2:1000:0", + "sandbox-owner-v2:1000:1000:extra", + ] { + assert!( + parse_sandbox_owner_state(marker).is_err(), + "marker should be rejected: {marker}" + ); + } + } + + #[tokio::test] + async fn image_and_overlay_owner_evidence_rejects_root_identity() { + let dir = unique_temp_dir(); + let image_source = dir.join("image-source"); + std::fs::create_dir_all(image_source.join("etc")).unwrap(); + std::fs::write( + image_source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:0:0:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let image = dir.join("rootfs.ext4"); + create_ext4_image_from_dir_with_size(&image_source, &image, 32 * 1024 * 1024).unwrap(); + let image_error = sandbox_owner_identity_from_image(&image) + .await + .expect_err("root image identity must be rejected"); + assert!(image_error.contains("uid 0 is outside the allowed range")); + + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:0:0:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let overlay_error = sandbox_owner_identity_from_overlay(&overlay) + .await + .expect_err("root overlay identity must be rejected"); + assert!(overlay_error.contains("uid 0 is outside the allowed range")); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn legacy_owner_marker_uses_evidence_migration_and_new_markers_are_private() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join(SANDBOX_OWNER_STATE_FILE), + format!("{SANDBOX_OWNER_STATE_V1}\n"), + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let config = VmDriverConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 2000, + gid: 3000 + } + ); + assert!(write_marker); + + write_sandbox_owner_state(&dir, identity).await.unwrap(); + let marker = dir.join(SANDBOX_OWNER_STATE_FILE); + assert_eq!( + std::fs::read_to_string(&marker).unwrap(), + "sandbox-owner-v2:2000:3000\n" + ); + assert_eq!( + std::fs::metadata(marker).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn persisted_owner_marker_preserves_exact_identity() { + let dir = unique_temp_dir(); + std::fs::create_dir_all(&dir).unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"current overlay").unwrap(); + let expected = SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + }; + write_sandbox_owner_state(&dir, expected).await.unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!(identity, expected); + assert!(!write_marker); + assert_eq!( + std::fs::read_to_string(dir.join(SANDBOX_OWNER_STATE_FILE)).unwrap(), + "sandbox-owner-v2:4242:4343\n" + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_upper_passwd() { + let dir = unique_temp_dir(); + let overlay_source = dir.join("overlay-source"); + std::fs::create_dir_all(overlay_source.join("upper/etc")).unwrap(); + std::fs::write( + overlay_source.join("upper/etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:10001:10001:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let overlay = dir.join(SANDBOX_OVERLAY_IMAGE); + create_ext4_image_from_dir_with_size(&overlay_source, &overlay, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: dir.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 10001, + gid: 10001, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(dir); + } + + #[tokio::test] + async fn unmarked_overlay_recovers_identity_from_persisted_rootfs() { + let root = unique_temp_dir(); + let state_dir = root.join("sandboxes/sandbox-1"); + std::fs::create_dir_all(&state_dir).unwrap(); + let overlay = state_dir.join(SANDBOX_OVERLAY_IMAGE); + std::fs::write(&overlay, b"legacy overlay").unwrap(); + let prior_identity = "legacy-cache:sha256:abc"; + std::fs::write( + state_dir.join(IMAGE_IDENTITY_FILE), + format!("{prior_identity}\n"), + ) + .unwrap(); + let source = root.join("legacy-rootfs-source"); + std::fs::create_dir_all(source.join("etc")).unwrap(); + std::fs::write( + source.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/sh\nsandbox:x:4242:4343:Sandbox:/sandbox:/bin/sh\n", + ) + .unwrap(); + let prior_disk = image_cache_rootfs_image(&root, prior_identity); + create_ext4_image_from_dir_with_size(&source, &prior_disk, 32 * 1024 * 1024).unwrap(); + let config = VmDriverConfig { + state_dir: root.clone(), + ..Default::default() + }; + + let (identity, write_marker) = sandbox_owner_state_for_launch( + &state_dir, + &overlay, + Path::new("/missing-current-rootfs"), + &config, + OverlayPreparation::PreserveExisting, + ) + .await + .unwrap(); + + assert_eq!( + identity, + SandboxOwnerIdentity { + uid: 4242, + gid: 4343, + } + ); + assert!(write_marker); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn sandbox_state_dir_rejects_path_unsafe_ids() { let err = sandbox_state_dir(Path::new("/tmp/openshell-vm"), "../escape") @@ -7827,6 +8561,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -7850,6 +8585,7 @@ mod tests { &overlay, None, None, + None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -8080,7 +8816,7 @@ mod tests { #[test] fn build_guest_environment_sets_supervisor_defaults() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8102,6 +8838,16 @@ mod tests { ))); } + #[test] + fn new_vm_sandbox_identity_defaults_to_1000() { + let config = VmDriverConfig::default(); + assert_eq!(config.resolve_sandbox_uid(), 1000); + assert_eq!( + config.resolve_sandbox_gid(config.resolve_sandbox_uid()), + 1000 + ); + } + #[test] fn validate_sandbox_identity_accepts_non_root_system_ids() { let config = VmDriverConfig { @@ -8115,7 +8861,7 @@ mod tests { #[test] fn persisted_legacy_sandbox_without_command_uses_scratch_main() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; // Requests persisted before the canonical-main contract have a @@ -8149,7 +8895,7 @@ mod tests { #[test] fn build_guest_environment_preserves_main_command_spaces() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8080".to_string(), + grpc_endpoint: "https://127.0.0.1:8080".to_string(), ..Default::default() }; let command = vec![ @@ -8186,7 +8932,7 @@ mod tests { #[test] fn build_guest_environment_uses_token_file_without_raw_token_env() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8218,7 +8964,7 @@ mod tests { #[test] fn build_guest_environment_strips_gateway_tls_server_name() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8255,7 +9001,7 @@ mod tests { )], || { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8294,7 +9040,7 @@ mod tests { #[test] fn build_guest_environment_clears_unsupported_network_capabilities() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8324,7 +9070,7 @@ mod tests { #[test] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; let sandbox = Sandbox { @@ -8523,10 +9269,81 @@ mod tests { ); } + #[test] + fn vm_driver_rejects_invalid_rootfs_tar_limits_before_startup() { + let config = VmDriverConfig { + rootfs_tar_max_bytes: Some(0), + ..Default::default() + }; + let error = config.validate_rootfs_tar_config().unwrap_err(); + assert!(error.contains("rootfs_tar_max_bytes")); + + let config = VmDriverConfig { + rootfs_tar_staging_dir: Some(PathBuf::new()), + ..Default::default() + }; + let error = config.validate_rootfs_tar_config().unwrap_err(); + assert!(error.contains("rootfs_tar_staging_dir")); + } + + #[test] + fn vm_proxy_and_spiffe_config_require_explicit_safe_acknowledgements() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.example:8080".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: None, + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: false, + ..Default::default() + }; + let error = config.validate_runtime_security_config().unwrap_err(); + assert!(error.contains("provider_spiffe_allow_guest_tcp")); + + let config = VmDriverConfig { + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + assert!(config.validate_runtime_security_config().is_ok()); + } + + #[test] + fn build_guest_environment_projects_spiffe_endpoint_without_operator_proxy() { + let config = VmDriverConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.example:8443".to_string()), + no_proxy: Some(".svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/run/secrets/proxy-auth")), + proxy_auth_allow_insecure: None, + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let sandbox = Sandbox { + id: "vm-spiffe".to_string(), + name: "vm-spiffe".to_string(), + ..Default::default() + }; + let env = build_guest_environment(&sandbox, &config, None); + assert!(env.contains( + &"OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=tcp:192.0.2.10:8081".to_string() + )); + assert!( + !env.iter().any(|value| value.contains("UPSTREAM_PROXY")), + "operator proxy settings must use protected guest argument staging: {env:?}" + ); + } + #[test] fn build_guest_environment_includes_tls_paths_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), guest_tls_cert: Some(PathBuf::from("/host/tls.crt")), guest_tls_key: Some(PathBuf::from("/host/tls.key")), @@ -8548,7 +9365,7 @@ mod tests { #[test] fn vm_driver_config_requires_tls_materials_for_https_endpoint() { let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8443".to_string(), + grpc_endpoint: "https://127.0.0.1:8443".to_string(), ..Default::default() }; let err = config @@ -8803,14 +9620,48 @@ mod tests { } #[test] - fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { + fn prepared_image_cache_identity_includes_layout_version_and_owner_contract() { + let image = "sha256:local-image"; + let image_account = prepared_image_cache_identity(image, &VmDriverConfig::default()); assert_eq!( - prepared_image_cache_identity("sha256:local-image"), + image_account, format!( - "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:sha256:local-image", + "sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:{image}", openshell_core::VERSION ) ); + + let identities = [ + VmDriverConfig { + sandbox_uid: Some(1000), + sandbox_gid: Some(1000), + ..Default::default() + }, + VmDriverConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }, + VmDriverConfig { + sandbox_uid: Some(2000), + ..Default::default() + }, + VmDriverConfig { + sandbox_gid: Some(3000), + ..Default::default() + }, + ] + .map(|config| prepared_image_cache_identity(image, &config)); + + assert!(identities.iter().all(|identity| identity != &image_account)); + for (index, identity) in identities.iter().enumerate() { + assert!( + identities[index + 1..] + .iter() + .all(|other| other != identity), + "owner contracts must use distinct cache keys" + ); + } } #[test] @@ -8843,7 +9694,10 @@ mod tests { &staging_dir, &GuestImagePayload { image_ref: "ghcr.io/example/app:latest".to_string(), - image_identity: prepared_image_cache_identity("sha256:abc"), + image_identity: prepared_image_cache_identity( + "sha256:abc", + &VmDriverConfig::default(), + ), source: GuestImagePayloadSource::RegistryOciLayout { layout_dir }, }, ) @@ -9233,23 +10087,37 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } - /// Identity must follow the bytes, not the path. The gateway hands every - /// request its own staging directory, so a path-derived key would miss the - /// cache on every single create. + /// Identity must follow the bytes and configured owner contract, not the + /// path. The gateway hands every request its own staging directory, so a + /// path-derived key would miss the cache on every single create. #[test] - fn rootfs_tar_cache_identity_tracks_contents_not_path() { - let same_a = rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"rootfs-bytes")); - let same_b = rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"rootfs-bytes")); - let different = rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"other-bytes")); + fn rootfs_tar_cache_identity_tracks_contents_and_owner_contract() { + let config = VmDriverConfig::default(); + let same_a = rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"rootfs-bytes"), &config); + let same_b = rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"rootfs-bytes"), &config); + let different = + rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"other-bytes"), &config); + let configured_owner = rootfs_tar_cache_identity( + &compute_bytes_sha256_hex(b"rootfs-bytes"), + &VmDriverConfig { + sandbox_uid: Some(1234), + sandbox_gid: Some(5678), + ..VmDriverConfig::default() + }, + ); assert_eq!( same_a, same_b, - "identical contents must share one prepared disk" + "identical contents and owner contracts must share one prepared disk" ); assert_ne!( same_a, different, "different contents must not collide on one prepared disk" ); + assert_ne!( + same_a, configured_owner, + "different owner contracts must not share a prepared disk" + ); } /// The old key was `path + seconds-truncated mtime` run through a @@ -9258,7 +10126,7 @@ mod tests { #[test] fn rootfs_tar_cache_identity_is_bounded_and_separator_safe() { let long_path_digest = compute_bytes_sha256_hex(&vec![7_u8; 4096]); - let identity = rootfs_tar_cache_identity(&long_path_digest); + let identity = rootfs_tar_cache_identity(&long_path_digest, &VmDriverConfig::default()); let sanitized = sanitize_image_identity(&identity); assert!( @@ -9267,8 +10135,14 @@ mod tests { sanitized.len() ); assert_ne!( - rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"/tmp/a/b.tar")), - rootfs_tar_cache_identity(&compute_bytes_sha256_hex(b"/tmp/a-b.tar")), + rootfs_tar_cache_identity( + &compute_bytes_sha256_hex(b"/tmp/a/b.tar"), + &VmDriverConfig::default(), + ), + rootfs_tar_cache_identity( + &compute_bytes_sha256_hex(b"/tmp/a-b.tar"), + &VmDriverConfig::default(), + ), "separator-colliding inputs must not share an identity" ); } @@ -9389,7 +10263,7 @@ mod tests { let (events, _) = broadcast::channel(WATCH_BUFFER); VmDriver { config: VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), vcpus: 2, mem_mib: 2048, gpu_vcpus: 8, @@ -9411,7 +10285,7 @@ mod tests { fn test_driver_with_proxy(https_proxy: &str) -> VmDriver { let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - driver.config.https_proxy = Some(https_proxy.to_string()); + driver.config.upstream_proxy.https_proxy = Some(https_proxy.to_string()); driver } @@ -9746,17 +10620,15 @@ mod tests { } /// A driver config carrying only corporate proxy settings. - fn proxy_config( - https_proxy: Option<&str>, - auth_file: Option<&str>, - ca_bundle: Option<&str>, - ) -> VmDriverConfig { + fn proxy_config(https_proxy: Option<&str>, auth_file: Option<&str>) -> VmDriverConfig { VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), - https_proxy: https_proxy.map(ToString::to_string), - proxy_auth_file: auth_file.map(ToString::to_string), - proxy_auth_allow_insecure: auth_file.map(|_| true), - proxy_ca_bundle: ca_bundle.map(ToString::to_string), + grpc_endpoint: "http://127.0.0.1:8080".to_string(), + upstream_proxy: UpstreamProxyConfig { + https_proxy: https_proxy.map(ToString::to_string), + proxy_auth_file: auth_file.map(PathBuf::from), + proxy_auth_allow_insecure: auth_file.map(|_| true), + ..UpstreamProxyConfig::default() + }, ..Default::default() } } @@ -9770,7 +10642,6 @@ mod tests { proxy_config( Some("http://user:secret@proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ) ); assert!( @@ -9782,14 +10653,10 @@ mod tests { "the credential path must be logged as presence only: {rendered}" ); assert!( - rendered.contains("https_proxy: true") && rendered.contains("proxy_auth_file: true"), + rendered.contains("upstream_proxy_configured: true") + && rendered.contains("proxy_auth_file_configured: true"), "presence of each must still be visible for debugging: {rendered}" ); - // A CA path is not sensitive and stays readable. - assert!( - rendered.contains("corp-ca.pem"), - "the CA bundle path is not a secret and should stay legible: {rendered}" - ); } #[test] @@ -9826,11 +10693,11 @@ mod tests { #[test] fn upstream_proxy_args_pass_guest_paths_not_host_paths() { - let config = proxy_config( + let mut config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ); + config.proxy_ca_bundle = Some(PathBuf::from("/etc/openshell/tls/corp-ca.pem")); let args = upstream_proxy_cli_args(&config); // The credential and CA live at fixed guest paths; the gateway-host @@ -9855,8 +10722,8 @@ mod tests { #[test] fn upstream_proxy_args_pass_only_explicit_opt_ins() { - let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None, None); - config.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string()); + let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None); + config.upstream_proxy.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string()); let args = upstream_proxy_cli_args(&config); assert_eq!( args, @@ -9870,13 +10737,13 @@ mod tests { // `Some(false)` must not be passed as the presence flag it is on the // supervisor side. - config.proxy_connect_by_hostname = Some(false); + config.upstream_proxy.proxy_connect_by_hostname = Some(false); assert!( !upstream_proxy_cli_args(&config) .iter() .any(|arg| arg == "--upstream-proxy-connect-by-hostname") ); - config.proxy_connect_by_hostname = Some(true); + config.upstream_proxy.proxy_connect_by_hostname = Some(true); assert!( upstream_proxy_cli_args(&config) .iter() @@ -9916,18 +10783,30 @@ mod tests { #[test] fn proxy_config_validation_rejects_settings_without_a_proxy_url() { let config = VmDriverConfig { - no_proxy: Some("10.0.0.0/8".to_string()), + upstream_proxy: UpstreamProxyConfig { + no_proxy: Some("10.0.0.0/8".to_string()), + ..UpstreamProxyConfig::default() + }, ..Default::default() }; let err = config - .validate_proxy_config() + .validate_runtime_security_config() .expect_err("a bypass list without a proxy would hide a fail-open state"); assert!(err.contains("no_proxy"), "{err}"); - let config = proxy_config(Some("http://proxy.corp.test:3128"), None, None); + let config = proxy_config(Some("http://proxy.corp.test:3128"), None); config - .validate_proxy_config() + .validate_runtime_security_config() .expect("a lone proxy URL is a complete configuration"); + + let config = VmDriverConfig { + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/corp-ca.pem")), + ..Default::default() + }; + let err = config + .validate_runtime_security_config() + .expect_err("a CA bundle without a proxy URL must fail closed"); + assert!(err.contains("proxy_ca_bundle"), "{err}"); } #[test] @@ -9935,78 +10814,14 @@ mod tests { let mut config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - None, ); - config.proxy_auth_allow_insecure = None; + config.upstream_proxy.proxy_auth_allow_insecure = None; let err = config - .validate_proxy_config() + .validate_runtime_security_config() .expect_err("Basic auth to an http:// proxy is cleartext on the wire"); assert!(err.contains("proxy_auth_allow_insecure"), "{err}"); } - #[tokio::test] - async fn proxy_ca_bundle_without_a_certificate_fails_the_sandbox() { - let dir = std::env::temp_dir().join(format!("openshell-vm-ca-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("not-a-ca.pem"); - std::fs::write(&path, b"this is not a certificate\n").unwrap(); - - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("a certificate-free bundle must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("no PEM certificate"), "{err}"); - - std::fs::write(&path, b"").unwrap(); - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("an empty bundle must fail closed"); - assert!(err.message().contains("no PEM certificate"), "{err}"); - - // PEM framing that base64-decodes but is not X.509 DER: accepted by - // `rustls_pemfile` alone, contributes zero trust anchors at runtime, - // and so would make every guest supervisor fail after boot. - std::fs::write( - &path, - b"-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n", - ) - .unwrap(); - let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap()) - .await - .expect_err("a bundle with invalid DER must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("no usable trust anchors"), "{err}"); - - let err = read_sandbox_proxy_ca_bundle(dir.join("missing.pem").to_str().unwrap()) - .await - .expect_err("an unreadable bundle must fail closed"); - assert!(err.message().contains("could not be read"), "{err}"); - - // A special file must be rejected on its type, not read: an - // unbounded read of /dev/zero would exhaust driver memory. - #[cfg(unix)] - { - let err = read_sandbox_proxy_ca_bundle("/dev/zero") - .await - .expect_err("a non-regular bundle path must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("not a regular file"), "{err}"); - } - - // Oversized regular file: rejected on the stat'd length, again - // without reading it whole. - let oversized = dir.join("oversized.pem"); - let bound = openshell_core::driver_utils::MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES; - std::fs::write(&oversized, vec![b'x'; usize::try_from(bound).unwrap() + 1]).unwrap(); - let err = read_sandbox_proxy_ca_bundle(oversized.to_str().unwrap()) - .await - .expect_err("an oversized bundle must fail closed"); - assert_eq!(err.code(), Code::InvalidArgument); - assert!(err.message().contains("exceeds"), "{err}"); - - std::fs::remove_dir_all(&dir).unwrap(); - } - #[test] fn qemu_backend_rejects_a_gateway_host_proxy() { // gvproxy's host-loopback NAT has no QEMU/TAP equivalent, so a proxy @@ -10112,7 +10927,7 @@ mod tests { std::fs::write(&path, "proxyuser:proxypass\n").unwrap(); assert_eq!( - read_sandbox_proxy_credential(path.to_str().unwrap()) + read_sandbox_proxy_credential(&path) .await .expect("a well-formed credential is accepted"), "proxyuser:proxypass" @@ -10120,7 +10935,7 @@ mod tests { // Rejected here rather than inside every sandbox's supervisor. std::fs::write(&path, "no-separator\n").unwrap(); - let err = read_sandbox_proxy_credential(path.to_str().unwrap()) + let err = read_sandbox_proxy_credential(&path) .await .expect_err("a malformed credential must fail closed"); assert_eq!(err.code(), Code::InvalidArgument); @@ -10140,7 +10955,6 @@ mod tests { let config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), - Some("/etc/openshell/tls/corp-ca.pem"), ); let sandbox = Sandbox { id: "sb-proxy".to_string(), @@ -10186,7 +11000,6 @@ mod tests { "proxy_auth_file", "proxy_auth_allow_insecure", "proxy_connect_by_hostname", - "proxy_ca_bundle", ] { let template = SandboxTemplate { driver_config: Some(Struct { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 1570ba8419..8f1cc649d7 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -91,8 +91,8 @@ struct Args { #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, + #[arg(long = "grpc-endpoint", env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] default_image: String, @@ -116,6 +116,52 @@ struct Args { #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] guest_tls_key: Option, + /// Corporate forward proxy for supervisor TLS egress. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY")] + upstream_proxy: Option, + + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_NO_PROXY")] + upstream_no_proxy: Option, + + /// Root-owned gateway-host file containing `user:pass` proxy credentials. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE")] + upstream_proxy_auth_file: Option, + + /// Explicitly acknowledge cleartext Basic authentication to an http proxy. + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE", + default_value_t = false + )] + upstream_proxy_auth_allow_insecure: bool, + + #[arg( + long, + env = "OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", + default_value_t = false + )] + upstream_proxy_connect_by_hostname: bool, + + /// Gateway-host PEM CA bundle trusted for the corporate proxy and for + /// server certificates re-signed by a TLS-intercepting proxy. + #[arg(long, env = "OPENSHELL_VM_UPSTREAM_PROXY_CA_BUNDLE")] + upstream_proxy_ca_bundle: Option, + + /// Guest-reachable SPIFFE Workload API endpoint (`tcp:IP:port`). + #[arg( + long = "provider-spiffe-workload-api-tcp-endpoint", + env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT" + )] + provider_spiffe_workload_api_tcp_endpoint: Option, + + /// Explicit acknowledgement that the configured Workload API listener is exposed to VM guests. + #[arg( + long, + env = "OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP", + default_value_t = false + )] + provider_spiffe_allow_guest_tcp: bool, + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] krun_log_level: u32, @@ -143,30 +189,6 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] sandbox_gid: Option, - // Corporate forward proxy for sandbox egress. Operator-owned: these reach - // the guest supervisor on its argv, which the sandbox image and the - // user-supplied environment cannot influence. - #[arg(long, env = "OPENSHELL_VM_HTTPS_PROXY")] - https_proxy: Option, - - #[arg(long, env = "OPENSHELL_VM_NO_PROXY")] - no_proxy: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_FILE")] - proxy_auth_file: Option, - - // Value-taking rather than a presence flag so an explicit `false` in - // `[openshell.drivers.vm]` survives the gateway -> driver hop and still - // trips the "acknowledgement without a credential" check. - #[arg(long, env = "OPENSHELL_VM_PROXY_AUTH_ALLOW_INSECURE")] - proxy_auth_allow_insecure: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_CONNECT_BY_HOSTNAME")] - proxy_connect_by_hostname: Option, - - #[arg(long, env = "OPENSHELL_VM_PROXY_CA_BUNDLE")] - proxy_ca_bundle: Option, - #[arg(long, env = "OPENSHELL_VM_ROOTFS_TAR_STAGING_DIR")] rootfs_tar_staging_dir: Option, @@ -241,8 +263,8 @@ async fn main() -> Result<()> { } let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint + grpc_endpoint: args + .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, state_dir: args.state_dir.clone(), launcher_bin: None, @@ -256,17 +278,23 @@ async fn main() -> Result<()> { guest_tls_ca: args.guest_tls_ca.clone(), guest_tls_cert: args.guest_tls_cert.clone(), guest_tls_key: args.guest_tls_key.clone(), + upstream_proxy: openshell_core::UpstreamProxyConfig { + https_proxy: args.upstream_proxy.clone(), + no_proxy: args.upstream_no_proxy.clone(), + proxy_auth_file: args.upstream_proxy_auth_file.clone(), + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure.then_some(true), + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname.then_some(true), + }, + proxy_ca_bundle: args.upstream_proxy_ca_bundle.clone(), + provider_spiffe_workload_api_tcp_endpoint: args + .provider_spiffe_workload_api_tcp_endpoint + .clone(), + provider_spiffe_allow_guest_tcp: args.provider_spiffe_allow_guest_tcp, gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, sandbox_uid: args.sandbox_uid, sandbox_gid: args.sandbox_gid, - https_proxy: args.https_proxy.clone(), - no_proxy: args.no_proxy.clone(), - proxy_auth_file: args.proxy_auth_file.clone(), - proxy_auth_allow_insecure: args.proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.proxy_connect_by_hostname, - proxy_ca_bundle: args.proxy_ca_bundle.clone(), rootfs_tar_staging_dir: args.rootfs_tar_staging_dir.clone(), rootfs_tar_max_bytes: args.rootfs_tar_max_bytes, }) @@ -644,57 +672,47 @@ mod tests { fn corporate_proxy_flags_parse_into_driver_settings() { let args = Args::parse_from([ "openshell-driver-vm", - "--openshell-endpoint", - "https://host.openshell.internal:17670", - "--https-proxy", + "--upstream-proxy", "http://proxy.corp.com:8080", - "--no-proxy", + "--upstream-no-proxy", "10.0.0.0/8,.svc.cluster.local", - "--proxy-auth-file", + "--upstream-proxy-auth-file", "/etc/openshell/secrets/proxy-auth", - "--proxy-auth-allow-insecure", - "true", - "--proxy-connect-by-hostname", - "false", - "--proxy-ca-bundle", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--upstream-proxy-ca-bundle", "/etc/openshell/tls/proxy-ca.pem", ]); assert_eq!( - args.https_proxy.as_deref(), + args.upstream_proxy.as_deref(), Some("http://proxy.corp.com:8080") ); assert_eq!( - args.no_proxy.as_deref(), + args.upstream_no_proxy.as_deref(), Some("10.0.0.0/8,.svc.cluster.local") ); assert_eq!( - args.proxy_auth_file.as_deref(), - Some("/etc/openshell/secrets/proxy-auth") + args.upstream_proxy_auth_file.as_deref(), + Some(PathBuf::from("/etc/openshell/secrets/proxy-auth").as_path()) ); - assert_eq!(args.proxy_auth_allow_insecure, Some(true)); - // Value-taking rather than a presence flag, so the gateway can - // forward an explicit `false` from `[openshell.drivers.vm]`. - assert_eq!(args.proxy_connect_by_hostname, Some(false)); + assert!(args.upstream_proxy_auth_allow_insecure); + assert!(args.upstream_proxy_connect_by_hostname); assert_eq!( - args.proxy_ca_bundle.as_deref(), - Some("/etc/openshell/tls/proxy-ca.pem") + args.upstream_proxy_ca_bundle.as_deref(), + Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem").as_path()) ); } #[test] fn corporate_proxy_settings_default_to_unset() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--openshell-endpoint", - "https://host.openshell.internal:17670", - ]); - assert!(args.https_proxy.is_none()); - assert!(args.no_proxy.is_none()); - assert!(args.proxy_auth_file.is_none()); - assert!(args.proxy_auth_allow_insecure.is_none()); - assert!(args.proxy_connect_by_hostname.is_none()); - assert!(args.proxy_ca_bundle.is_none()); + let args = Args::parse_from(["openshell-driver-vm"]); + assert!(args.upstream_proxy.is_none()); + assert!(args.upstream_no_proxy.is_none()); + assert!(args.upstream_proxy_auth_file.is_none()); + assert!(!args.upstream_proxy_auth_allow_insecure); + assert!(!args.upstream_proxy_connect_by_hostname); + assert!(args.upstream_proxy_ca_bundle.is_none()); } #[test] @@ -772,6 +790,28 @@ mod tests { assert!(err.contains("--bind-socket is required")); } + #[test] + fn accepts_canonical_grpc_endpoint_flag() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--grpc-endpoint", + "http://127.0.0.1:8080", + ]) + .unwrap(); + assert_eq!(args.grpc_endpoint.as_deref(), Some("http://127.0.0.1:8080")); + } + + #[test] + fn rejects_legacy_openshell_endpoint_flag() { + let error = Args::try_parse_from([ + "openshell-driver-vm", + "--openshell-endpoint", + "http://127.0.0.1:8080", + ]) + .expect_err("legacy --openshell-endpoint must be rejected"); + assert!(error.to_string().contains("--openshell-endpoint")); + } + #[test] fn accepts_gateway_otlp_configuration() { let args = Args::try_parse_from([ diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 5e8de5f695..5bd7d80f35 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -20,8 +20,7 @@ pub const GZIP_MAGIC: [u8; 2] = [0x1f, 0x8b]; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; -const SANDBOX_OWNER_NORMALIZED_MARKER: &str = - openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const DEFAULT_SANDBOX_UID: u32 = 1000; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -35,8 +34,8 @@ pub const fn sandbox_guest_init_path() -> &'static str { pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, image_identity: &str, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { prepare_sandbox_rootfs(rootfs, sandbox_uid, sandbox_gid)?; validate_sandbox_rootfs(rootfs)?; @@ -375,7 +374,11 @@ fn append_symlink_to_archive( } #[allow(clippy::similar_names)] -fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Result<(), String> { +fn prepare_sandbox_rootfs( + rootfs: &Path, + sandbox_uid: Option, + sandbox_gid: Option, +) -> Result<(), String> { for relative in ["opt/openshell/.initialized", "opt/openshell/.rootfs-type"] { remove_rootfs_path(rootfs, relative)?; } @@ -589,8 +592,7 @@ fn normalize_sandbox_owner_in_rootfs_image(source: &Path, image_path: &Path) -> return Ok(()); } - run_debugfs_batch(image_path, &commands)?; - write_rootfs_image_file(image_path, SANDBOX_OWNER_NORMALIZED_MARKER, b"1\n") + run_debugfs_batch(image_path, &commands) } fn collect_sandbox_owner_commands( @@ -677,6 +679,82 @@ fn debugfs_quote_argument(argument: &str) -> Option { Some(quoted) } +/// Read the sandbox account identity directly from an ext4 rootfs image. +/// +/// Persisted VM overlays may outlive the prepared-image cache layout that +/// created them. Reading the matching old lower disk lets the driver preserve +/// that overlay's real ownership contract during an upgrade. +pub fn sandbox_guest_user_ids_from_image(image_path: &Path) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/etc/passwd") +} + +/// Read a sandbox account copied into an overlay upper layer. +/// +/// An upper-layer passwd file is the most direct evidence of the identity an +/// existing overlay observed, so migration consults it before any lower image. +pub fn sandbox_guest_user_ids_from_overlay_image( + image_path: &Path, +) -> Result, String> { + sandbox_guest_user_ids_from_image_path(image_path, "/upper/etc/passwd") +} + +fn sandbox_guest_user_ids_from_image_path( + image_path: &Path, + guest_path: &str, +) -> Result, String> { + let metadata = fs::metadata(image_path) + .map_err(|error| format!("stat rootfs image {}: {error}", image_path.display()))?; + if !metadata.is_file() { + return Err(format!( + "rootfs image {} is not a regular file", + image_path.display() + )); + } + + let quoted_path = debugfs_quote_absolute_path(guest_path) + .expect("the static passwd path is a valid debugfs path"); + let command = format!("cat {quoted_path}"); + let mut last_error = None; + + for candidate in e2fs_tool_candidates("debugfs") { + let label = candidate.display().to_string(); + match Command::new(&candidate) + .arg("-R") + .arg(&command) + .arg(image_path) + .output() + { + Ok(output) if output.status.success() => { + let passwd = String::from_utf8(output.stdout).map_err(|error| { + format!( + "read {guest_path} from {} as UTF-8: {error}", + image_path.display() + ) + })?; + return parse_sandbox_guest_user_ids(&passwd, &image_path.display().to_string()); + } + Ok(output) => { + last_error = Some(format!( + "{label} failed with status {}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + last_error = Some(format!("{label} not found")); + } + Err(error) => last_error = Some(format!("run {label}: {error}")), + } + } + + Err(format!( + "debugfs command '{command}' failed for {}: {}. Install e2fsprogs (debugfs) and retry", + image_path.display(), + last_error.unwrap_or_else(|| "debugfs not found".to_string()) + )) +} + fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd_path = rootfs.join("etc/passwd"); if !passwd_path.exists() { @@ -685,6 +763,10 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let passwd = fs::read_to_string(&passwd_path) .map_err(|e| format!("read {}: {e}", passwd_path.display()))?; + parse_sandbox_guest_user_ids(&passwd, &passwd_path.display().to_string()) +} + +fn parse_sandbox_guest_user_ids(passwd: &str, source: &str) -> Result, String> { for line in passwd.lines() { let mut parts = line.split(':'); if parts.next() != Some("sandbox") { @@ -693,14 +775,14 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result, String> { let _password = parts.next(); let uid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing uid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing uid"))? .parse::() - .map_err(|e| format!("sandbox uid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox uid in {source} is invalid: {e}"))?; let gid = parts .next() - .ok_or_else(|| format!("sandbox entry in {} is missing gid", passwd_path.display()))? + .ok_or_else(|| format!("sandbox entry in {source} is missing gid"))? .parse::() - .map_err(|e| format!("sandbox gid in {} is invalid: {e}", passwd_path.display()))?; + .map_err(|e| format!("sandbox gid in {source} is invalid: {e}"))?; return Ok(Some((uid, gid))); } @@ -811,9 +893,18 @@ fn temporary_injection_path(image_path: &Path) -> PathBuf { #[allow(clippy::similar_names)] fn ensure_sandbox_guest_user( rootfs: &Path, - sandbox_uid: u32, - sandbox_gid: u32, + sandbox_uid: Option, + sandbox_gid: Option, ) -> Result<(), String> { + // An image's sandbox account is part of its filesystem contract. Leave it + // intact unless an operator explicitly configured either side of the + // identity. A missing account still gets the OpenShell default. + if sandbox_uid.is_none() && sandbox_gid.is_none() && sandbox_guest_user_ids(rootfs)?.is_some() { + return Ok(()); + } + + let sandbox_uid = sandbox_uid.unwrap_or(DEFAULT_SANDBOX_UID); + let sandbox_gid = sandbox_gid.unwrap_or(sandbox_uid); let etc_dir = rootfs.join("etc"); fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; @@ -844,23 +935,32 @@ fn ensure_line_in_file( line: &str, exists: impl Fn(&str) -> bool, ) -> Result<(), String> { - let mut contents = if path.exists() { + let contents = if path.exists() { fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? } else { String::new() }; - if contents.lines().any(exists) { - return Ok(()); + let mut replaced = false; + let mut updated = String::new(); + for existing in contents.lines() { + if exists(existing) { + if !replaced { + updated.push_str(line); + updated.push('\n'); + replaced = true; + } + } else { + updated.push_str(existing); + updated.push('\n'); + } } - - if !contents.is_empty() && !contents.ends_with('\n') { - contents.push('\n'); + if !replaced { + updated.push_str(line); + updated.push('\n'); } - contents.push_str(line); - contents.push('\n'); - fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) + fs::write(path, updated).map_err(|e| format!("write {}: {e}", path.display())) } fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { @@ -981,10 +1081,10 @@ mod tests { write_fake_runtime_binaries(&rootfs); fs::write( rootfs.join("etc/passwd"), - "root:x:0:0:root:/root:/bin/bash\n", + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n", ) .expect("write passwd"); - fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:10001:\n").expect("write group"); fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); fs::create_dir_all(rootfs.join("bin")).expect("create bin"); fs::create_dir_all(rootfs.join("sbin")).expect("create sbin"); @@ -996,7 +1096,7 @@ mod tests { // Use a non-standard UID so the test doesn't collide with the default. let uid = 20001; - prepare_sandbox_rootfs(&rootfs, uid, uid).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(uid), Some(uid)).expect("prepare sandbox rootfs"); validate_sandbox_rootfs(&rootfs).expect("validate sandbox rootfs"); assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); @@ -1024,6 +1124,12 @@ mod tests { .expect("read group") .contains(&format!("sandbox:x:{uid}:")) ); + assert!( + !fs::read_to_string(rootfs.join("etc/passwd")) + .expect("read passwd") + .contains("sandbox:x:10001:"), + "newly prepared rootfs must replace the legacy sandbox account" + ); assert_eq!( fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), "127.0.0.1 localhost\n" @@ -1042,7 +1148,7 @@ mod tests { fs::create_dir_all(rootfs.join("sandbox")).expect("create sandbox workdir"); fs::write(rootfs.join("sandbox/app.py"), "print('hello')\n").expect("write app"); - prepare_sandbox_rootfs(&rootfs, 10001, 10001).expect("prepare sandbox rootfs"); + prepare_sandbox_rootfs(&rootfs, Some(10001), Some(10001)).expect("prepare sandbox rootfs"); assert!(rootfs.join("sandbox").is_dir()); assert_eq!( @@ -1152,6 +1258,61 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn sandbox_user_preserves_image_account_when_identity_is_omitted() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((4242, 4343))); + assert!( + fs::read_to_string(rootfs.join("etc/passwd")) + .unwrap() + .contains("Image:/image-home:/bin/false") + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_defaults_to_1000_when_image_has_no_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + ensure_sandbox_guest_user(&rootfs, None, None).unwrap(); + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((1000, 1000))); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn sandbox_user_explicit_identity_overrides_image_account() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write( + rootfs.join("etc/passwd"), + "sandbox:x:4242:4343:Image:/image-home:/bin/false\n", + ) + .unwrap(); + fs::write(rootfs.join("etc/group"), "sandbox:x:4343:\n").unwrap(); + + ensure_sandbox_guest_user(&rootfs, Some(2000), Some(3000)).unwrap(); + + assert_eq!(sandbox_guest_user_ids(&rootfs).unwrap(), Some((2000, 3000))); + assert!( + fs::read_to_string(rootfs.join("etc/group")) + .unwrap() + .contains("sandbox:x:3000:") + ); + let _ = fs::remove_dir_all(dir); + } + #[test] fn sandbox_guest_user_ids_reads_existing_sandbox_user() { let dir = unique_temp_dir(); diff --git a/crates/openshell-gateway-interceptors/src/lib.rs b/crates/openshell-gateway-interceptors/src/lib.rs index d5bb5df59e..8d05878dd2 100644 --- a/crates/openshell-gateway-interceptors/src/lib.rs +++ b/crates/openshell-gateway-interceptors/src/lib.rs @@ -43,6 +43,11 @@ pub enum InterceptorError { pub type Result = std::result::Result; +/// Validate static interceptor configuration without opening a transport. +pub fn validate_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { + plan::validate_interceptor_configs(configs) +} + pub(crate) type ExtensionChannel = InterceptedService; /// Return `None` when no interceptors are configured. diff --git a/crates/openshell-gateway-interceptors/src/plan.rs b/crates/openshell-gateway-interceptors/src/plan.rs index c89de6f6ee..3328348249 100644 --- a/crates/openshell-gateway-interceptors/src/plan.rs +++ b/crates/openshell-gateway-interceptors/src/plan.rs @@ -740,7 +740,7 @@ fn validate_service_config(config: &GatewayInterceptorConfig) -> Result<()> { Ok(()) } -fn validate_interceptor_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { +pub fn validate_interceptor_configs(configs: &[GatewayInterceptorConfig]) -> Result<()> { let mut names = BTreeSet::new(); for config in configs { validate_service_config(config)?; diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index f9d02027e0..2bb0227171 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -26,6 +26,7 @@ tokio = { workspace = true } openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +openshell-policy = { path = "../openshell-policy", optional = true } hyper-util = { workspace = true, optional = true } nix = { workspace = true, optional = true } serde = { workspace = true, optional = true } @@ -43,6 +44,7 @@ in-tree-compute-drivers = [ "dep:openshell-driver-docker", "dep:openshell-driver-kubernetes", "dep:openshell-driver-podman", + "dep:openshell-policy", "dep:openshell-otel", "dep:hyper-util", "dep:nix", @@ -70,3 +72,4 @@ workspace = true [dev-dependencies] tempfile = "3" +toml = { workspace = true } diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 54ae5e5de3..2f222c6cb8 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -70,6 +70,13 @@ struct UnsupportedWindowsFactory { #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for UnsupportedWindowsFactory { + fn validate_config( + &self, + _context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Err(unsupported_windows_compute_driver(self.name)) + } + async fn build( &self, _context: openshell_server::ComputeDriverBuildContext<'_>, @@ -90,6 +97,14 @@ struct MxcFactory; #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for MxcFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let _: openshell_driver_mxc::MxcComputeConfig = context.driver_config()?; + Ok(()) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, @@ -117,16 +132,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("kubernetes")) .without_mtls_user_auth() .with_in_process_tracing(openshell_driver_kubernetes::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ]) }), ComputeDriverRegistration::new( "podman", @@ -139,14 +144,6 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("podman")) .with_local_singleplayer() .with_in_process_tracing(openshell_driver_podman::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new( "docker", @@ -159,26 +156,11 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { .with_telemetry_category(TelemetryComputeDriver::anonymous_category("docker")) .with_local_singleplayer() .with_in_process_tracing(openshell_driver_docker::otel_tracing::TRACING) - .with_inherited_config_keys(&[ - "sandbox_namespace", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { registration .with_telemetry_category(TelemetryComputeDriver::anonymous_category("vm")) .with_local_singleplayer() - .with_inherited_config_keys(&[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]) }), ] { registry @@ -194,18 +176,20 @@ struct KubernetesFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for KubernetesFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + kubernetes_config(context)? + .validate_configuration() + .map_err(openshell_core::Error::config) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = - context.driver_config()?; - if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { - config.workspace_default_storage_size = size; - } - if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { - config.workspace_storage_class = storage_class; - } + let config = kubernetes_config(context.config_context())?; let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( config, context.shutdown_receiver(), @@ -219,6 +203,21 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn kubernetes_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = + context.driver_config()?; + if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { + config.workspace_default_storage_size = size; + } + if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { + config.workspace_storage_class = storage_class; + } + Ok(config) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct DockerFactory; @@ -226,11 +225,20 @@ struct DockerFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for DockerFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + config.validate_configuration(context.gateway_bind_address()) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls( &mut config.guest_tls_ca, &mut config.guest_tls_cert, @@ -258,21 +266,21 @@ struct PodmanFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for PodmanFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + podman_config(context)? + .validate_configuration() + .map_err(|error| openshell_core::Error::config(error.to_string())) + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; - config.gateway_port = context.gateway_port(); - if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - config.socket_path = Some(path.into()); - } - if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { - config.host_gateway_ip = ip; - } - if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { - config.userns = Some(mode); - } + let mut config = podman_config(context.config_context())?; + require_guest_tls_for_local_driver(&context, "podman")?; apply_guest_tls( &mut config.guest_tls_ca, &mut config.guest_tls_cert, @@ -289,6 +297,24 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn podman_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + config.gateway_port = context.gateway_port(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { + config.socket_path = Some(path.into()); + } + if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { + config.host_gateway_ip = ip; + } + if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { + config.userns = Some(mode); + } + Ok(config) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct VmFactory; @@ -296,14 +322,28 @@ struct VmFactory; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl openshell_server::ComputeDriverFactory for VmFactory { + fn validate_config( + &self, + context: openshell_server::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + let mut config = vm_config(context)?; + if config.grpc_endpoint.trim().is_empty() { + let scheme = if context.gateway_tls_enabled() { + "https" + } else { + "http" + }; + config.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port()); + } + config.validate_configuration() + } + async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { - let mut config: vm::VmComputeConfig = context.driver_config()?; - if config.state_dir.as_os_str().is_empty() { - config.state_dir = vm::VmComputeConfig::default_state_dir(); - } + let mut config = vm_config(context.config_context())?; + require_guest_tls_for_local_driver(&context, "vm")?; if config.grpc_endpoint.trim().is_empty() && (!context.gateway_tls_enabled() || context.guest_tls_paths().is_some()) { @@ -333,6 +373,43 @@ impl openshell_server::ComputeDriverFactory for VmFactory { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn vm_config( + context: openshell_server::ComputeDriverConfigContext<'_>, +) -> openshell_core::Result { + let mut config: vm::VmComputeConfig = context.driver_config()?; + if config.state_dir.as_os_str().is_empty() { + config.state_dir = vm::VmComputeConfig::default_state_dir(); + } + Ok(config) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn require_guest_tls_for_local_driver( + context: &openshell_server::ComputeDriverBuildContext<'_>, + driver_name: &str, +) -> openshell_core::Result<()> { + validate_local_driver_guest_tls( + context.gateway_tls_enabled(), + context.guest_tls_paths().is_some(), + driver_name, + ) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn validate_local_driver_guest_tls( + gateway_tls_enabled: bool, + has_guest_tls: bool, + driver_name: &str, +) -> openshell_core::Result<()> { + if gateway_tls_enabled && !has_guest_tls { + return Err(openshell_core::Error::config(format!( + "gateway TLS requires guest_tls_ca, guest_tls_cert, and guest_tls_key in [openshell.gateway] when using the {driver_name} compute driver" + ))); + } + Ok(()) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn apply_guest_tls( ca: &mut Option, @@ -351,6 +428,59 @@ fn apply_guest_tls( } } +#[cfg(all(test, not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod local_driver_tests { + use super::{ + apply_guest_tls, install_default_compute_drivers, validate_local_driver_guest_tls, + }; + use std::path::{Path, PathBuf}; + + #[test] + fn linux_builtin_compute_driver_registry_has_expected_names() { + assert_eq!( + install_default_compute_drivers() + .installed_driver_names() + .collect::>(), + ["docker", "kubernetes", "podman", "vm"] + ); + } + + #[test] + fn tls_enabled_local_drivers_require_a_guest_bundle() { + for driver_name in ["docker", "podman", "vm"] { + let error = validate_local_driver_guest_tls(true, false, driver_name) + .expect_err("TLS-enabled local driver must require guest TLS"); + let message = error.to_string(); + assert!(message.contains(driver_name)); + assert!(message.contains("guest_tls_ca")); + } + validate_local_driver_guest_tls(true, true, "docker") + .expect("a complete guest bundle satisfies the requirement"); + validate_local_driver_guest_tls(false, false, "docker") + .expect("plaintext gateways do not require guest TLS"); + } + + #[test] + fn package_managed_guest_bundle_is_injected_when_driver_paths_are_absent() { + let mut ca = None; + let mut cert = None; + let mut key = None; + apply_guest_tls( + &mut ca, + &mut cert, + &mut key, + Some(( + Path::new("/managed/ca.pem"), + Path::new("/managed/client.pem"), + Path::new("/managed/client-key.pem"), + )), + ); + assert_eq!(ca, Some(PathBuf::from("/managed/ca.pem"))); + assert_eq!(cert, Some(PathBuf::from("/managed/client.pem"))); + assert_eq!(key, Some(PathBuf::from("/managed/client-key.pem"))); + } +} + #[cfg(all(test, target_os = "windows", feature = "in-tree-compute-drivers"))] mod windows_tests { use super::*; diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index f52bd9ddfa..07735c7a9c 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -31,7 +31,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{Error, Result}; +use openshell_core::{Error, Result, UpstreamProxyConfig}; #[cfg(unix)] use openshell_otel::TraceContextInterceptor; use openshell_server::AcquiredRemoteDriverEndpoint; @@ -91,6 +91,18 @@ pub struct VmComputeConfig { /// Writable overlay disk size for each VM sandbox, in MiB. pub overlay_disk_mib: u64, + /// Optional UID override for the VM guest sandbox account. + pub sandbox_uid: Option, + + /// Optional GID override for the VM guest sandbox account. + pub sandbox_gid: Option, + + /// Directory shared with the gateway for request-scoped rootfs tar staging. + pub rootfs_tar_staging_dir: Option, + + /// Maximum accepted rootfs tar size, in bytes, before and after decompression. + pub rootfs_tar_max_bytes: Option, + /// Host-side CA certificate for the guest's mTLS client bundle. pub guest_tls_ca: Option, @@ -100,33 +112,20 @@ pub struct VmComputeConfig { /// Host-side private key for the guest's mTLS client bundle. pub guest_tls_key: Option, - /// Corporate forward proxy URL (`http://host:port` or `https://host:port`) - /// for policy-approved TLS egress from VM sandboxes. - /// - /// Deployment-level configuration, not a per-sandbox setting: it is passed - /// to the driver, which puts it on the guest supervisor's argv. A proxy on - /// this host's loopback is reachable from a guest only through the gvproxy - /// host alias `host.openshell.internal`. - pub https_proxy: Option, - - /// Comma-separated `NO_PROXY` list. Bypasses only the corporate proxy, - /// never `OpenShell` policy evaluation. - pub no_proxy: Option, - - /// Path on this host to a `user:pass` corporate proxy credential file. - pub proxy_auth_file: Option, - - /// Acknowledgement that Basic auth to an `http://` proxy is cleartext. - /// Required alongside `proxy_auth_file` unless the proxy is `https://`. - pub proxy_auth_allow_insecure: Option, - - /// Send hostnames rather than validated IPs in CONNECT requests. Last - /// resort for proxies whose ACLs reject IP CONNECT targets. - pub proxy_connect_by_hostname: Option, - - /// Path on this host to a PEM CA bundle trusted for the corporate proxy - /// and for server certificates a TLS-intercepting proxy re-signs. - pub proxy_ca_bundle: Option, + /// Corporate forward-proxy settings passed to the VM driver. Flattening + /// preserves the shared local-driver TOML field names. + #[serde(flatten)] + pub upstream_proxy: UpstreamProxyConfig, + + /// Path on the gateway host to a PEM CA bundle trusted for the corporate + /// proxy and for server certificates re-signed by a TLS-intercepting proxy. + pub proxy_ca_bundle: Option, + + /// Explicit guest-reachable SPIFFE Workload API TCP listener. VM guests + /// cannot receive a host UNIX socket, so this requires acknowledgement. + pub provider_spiffe_workload_api_tcp_endpoint: Option, + #[serde(default)] + pub provider_spiffe_allow_guest_tcp: bool, } impl VmComputeConfig { @@ -163,27 +162,57 @@ impl VmComputeConfig { 4096 } - /// Validate the corporate upstream-proxy settings, fail-closed. - /// - /// Runs in the gateway as well as in the driver so an invalid - /// `[openshell.drivers.vm]` table reports the offending key instead of - /// surfacing as an opaque driver-startup timeout. - /// - /// # Errors - /// - /// Returns a [`Error::config`] naming the offending key. - pub fn validate_proxy_config(&self) -> Result<()> { - openshell_core::driver_utils::validate_upstream_proxy_settings( - &openshell_core::driver_utils::UpstreamProxySettings { - url: self.https_proxy.as_deref(), - no_proxy: self.no_proxy.as_deref(), - auth_file: self.proxy_auth_file.as_deref(), - auth_allow_insecure: self.proxy_auth_allow_insecure, - connect_by_hostname: self.proxy_connect_by_hostname, - ca_bundle: self.proxy_ca_bundle.as_deref(), - }, - ) - .map_err(Error::config) + /// Validate startup configuration without resolving binaries, creating + /// state directories, spawning a process, or connecting a socket. + pub fn validate_configuration(&self) -> Result<()> { + if self.grpc_endpoint.trim().is_empty() { + return Err(Error::config( + "grpc_endpoint is required when using the vm compute driver", + )); + } + validate_vm_sandbox_identity(self)?; + if self + .rootfs_tar_staging_dir + .as_ref() + .is_some_and(|path| path.as_os_str().is_empty()) + { + return Err(Error::config( + "rootfs_tar_staging_dir must not be empty when set", + )); + } + if self.rootfs_tar_max_bytes == Some(0) { + return Err(Error::config( + "rootfs_tar_max_bytes must be greater than zero when set", + )); + } + self.validate_proxy_config()?; + if let Some(endpoint) = self.provider_spiffe_workload_api_tcp_endpoint.as_deref() { + openshell_core::driver_utils::validate_guest_spiffe_tcp_endpoint( + endpoint, + self.provider_spiffe_allow_guest_tcp, + ) + .map_err(Error::config)?; + } else if self.provider_spiffe_allow_guest_tcp { + return Err(Error::config( + "provider_spiffe_allow_guest_tcp is set but no provider_spiffe_workload_api_tcp_endpoint is configured", + )); + } + Ok(()) + } + + fn validate_proxy_config(&self) -> Result<()> { + self.upstream_proxy.validate().map_err(Error::config)?; + if let Some(path) = self.proxy_ca_bundle.as_ref() { + if path.as_os_str().is_empty() { + return Err(Error::config("proxy_ca_bundle must not be empty when set")); + } + if self.upstream_proxy.https_proxy.is_none() { + return Err(Error::config( + "proxy_ca_bundle is set but no https_proxy is configured", + )); + } + } + Ok(()) } #[must_use] @@ -211,15 +240,17 @@ impl Default for VmComputeConfig { vcpus: Self::default_vcpus(), mem_mib: Self::default_mem_mib(), overlay_disk_mib: Self::default_overlay_disk_mib(), + sandbox_uid: None, + sandbox_gid: None, + rootfs_tar_staging_dir: None, + rootfs_tar_max_bytes: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - https_proxy: None, - no_proxy: None, - proxy_auth_file: None, - proxy_auth_allow_insecure: None, - proxy_connect_by_hostname: None, + upstream_proxy: UpstreamProxyConfig::default(), proxy_ca_bundle: None, + provider_spiffe_workload_api_tcp_endpoint: None, + provider_spiffe_allow_guest_tcp: false, } } } @@ -511,14 +542,7 @@ pub async fn spawn( vm_config: &VmComputeConfig, otlp_config: Option<&OtlpConfig>, ) -> Result { - if vm_config.grpc_endpoint.trim().is_empty() { - return Err(Error::config( - "grpc_endpoint is required when using the vm compute driver", - )); - } - - vm_config.validate_proxy_config()?; - + vm_config.validate_configuration()?; let driver_bin = resolve_compute_driver_bin(vm_config)?; let socket_path = compute_driver_socket_path(vm_config); let guest_tls_paths = compute_driver_guest_tls_paths(vm_config)?; @@ -535,9 +559,7 @@ pub async fn spawn( .arg(std::process::id().to_string()); command.arg("--log-level").arg(gateway_log_level); append_otlp_args(&mut command, otlp_config, gateway_name); - command - .arg("--openshell-endpoint") - .arg(&vm_config.grpc_endpoint); + command.arg("--grpc-endpoint").arg(&vm_config.grpc_endpoint); command.arg("--state-dir").arg(&vm_config.state_dir); if !vm_config.default_image.trim().is_empty() { command.arg("--default-image").arg(&vm_config.default_image); @@ -555,12 +577,14 @@ pub async fn spawn( command .arg("--overlay-disk-mib") .arg(vm_config.overlay_disk_mib.to_string()); + append_vm_identity_args(&mut command, vm_config); + append_vm_rootfs_tar_args(&mut command, vm_config); if let Some(tls) = guest_tls_paths { command.arg("--guest-tls-ca").arg(tls.ca); command.arg("--guest-tls-cert").arg(tls.cert); command.arg("--guest-tls-key").arg(tls.key); } - append_upstream_proxy_args(&mut command, vm_config); + append_vm_proxy_and_spiffe_args(&mut command, vm_config); let mut child = command.spawn().map_err(|e| { Error::execution(format!( @@ -575,39 +599,76 @@ pub async fn spawn( )) } -/// Forward the operator's corporate proxy settings to the driver subprocess. -/// -/// Only keys the operator actually set are passed, so the driver keeps the -/// same "omitted means no proxy" contract the supervisor enforces. The -/// booleans travel as explicit values rather than presence flags so an -/// explicit `false` still trips the driver's pairing checks. +fn validate_vm_sandbox_identity(config: &VmComputeConfig) -> Result<()> { + let range = openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID; + for (field, value) in [ + ("sandbox_uid", config.sandbox_uid), + ("sandbox_gid", config.sandbox_gid), + ] { + if let Some(value) = value + && !range.contains(&value) + { + return Err(Error::config(format!( + "{field} {value} is outside the allowed range [{}, {}]", + openshell_policy::MIN_SANDBOX_UID, + openshell_policy::MAX_SANDBOX_UID + ))); + } + } + Ok(()) +} + #[cfg(unix)] -fn append_upstream_proxy_args(command: &mut Command, vm_config: &VmComputeConfig) { - if let Some(url) = &vm_config.https_proxy { - command.arg("--https-proxy").arg(url); +fn append_vm_identity_args(command: &mut Command, config: &VmComputeConfig) { + if let Some(uid) = config.sandbox_uid { + command.arg("--sandbox-uid").arg(uid.to_string()); } - if let Some(list) = &vm_config.no_proxy { - command.arg("--no-proxy").arg(list); + if let Some(gid) = config.sandbox_gid { + command.arg("--sandbox-gid").arg(gid.to_string()); } - if let Some(path) = &vm_config.proxy_auth_file { - command.arg("--proxy-auth-file").arg(path); +} + +#[cfg(unix)] +fn append_vm_rootfs_tar_args(command: &mut Command, config: &VmComputeConfig) { + if let Some(path) = config.rootfs_tar_staging_dir.as_ref() { + command.arg("--rootfs-tar-staging-dir").arg(path); } - if let Some(allow) = vm_config.proxy_auth_allow_insecure { + if let Some(max_bytes) = config.rootfs_tar_max_bytes { command - .arg("--proxy-auth-allow-insecure") - .arg(allow.to_string()); + .arg("--rootfs-tar-max-bytes") + .arg(max_bytes.to_string()); } - if let Some(by_hostname) = vm_config.proxy_connect_by_hostname { - command - .arg("--proxy-connect-by-hostname") - .arg(by_hostname.to_string()); +} + +#[cfg(unix)] +fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConfig) { + let proxy = &config.upstream_proxy; + if let Some(url) = proxy.https_proxy.as_ref() { + command.arg("--upstream-proxy").arg(url); + } + if let Some(no_proxy) = proxy.no_proxy.as_ref() { + command.arg("--upstream-no-proxy").arg(no_proxy); + } + if let Some(auth_file) = proxy.proxy_auth_file.as_ref() { + command.arg("--upstream-proxy-auth-file").arg(auth_file); + } + if proxy.proxy_auth_allow_insecure == Some(true) { + command.arg("--upstream-proxy-auth-allow-insecure"); + } + if proxy.proxy_connect_by_hostname == Some(true) { + command.arg("--upstream-proxy-connect-by-hostname"); + } + if let Some(path) = config.proxy_ca_bundle.as_ref() { + command.arg("--upstream-proxy-ca-bundle").arg(path); } - if let Some(path) = &vm_config.proxy_ca_bundle { - command.arg("--proxy-ca-bundle").arg(path); + if let Some(endpoint) = config.provider_spiffe_workload_api_tcp_endpoint.as_ref() { + command + .arg("--provider-spiffe-workload-api-tcp-endpoint") + .arg(endpoint); + command.arg("--provider-spiffe-allow-guest-tcp"); } } -#[cfg(unix)] fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { if let Some(config) = otlp_config { command.arg("--otlp-endpoint").arg(&config.endpoint); @@ -698,11 +759,13 @@ async fn connect_compute_driver(socket_path: &Path) -> Result { #[cfg(all(test, unix))] mod tests { use super::{ - VmComputeConfig, append_otlp_args, append_upstream_proxy_args, - compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, - prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, - resolve_driver_search_dirs, + VmComputeConfig, append_otlp_args, append_vm_identity_args, + append_vm_proxy_and_spiffe_args, append_vm_rootfs_tar_args, compute_driver_guest_tls_paths, + compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, + prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, + validate_vm_sandbox_identity, }; + use openshell_core::UpstreamProxyConfig; use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; @@ -740,15 +803,19 @@ mod tests { #[test] fn vm_driver_command_forwards_corporate_proxy_settings() { let mut command = tokio::process::Command::new("openshell-driver-vm"); - append_upstream_proxy_args( + append_vm_proxy_and_spiffe_args( &mut command, &VmComputeConfig { - https_proxy: Some("http://proxy.corp.com:8080".to_string()), - no_proxy: Some("10.0.0.0/8".to_string()), - proxy_auth_file: Some("/etc/openshell/secrets/proxy-auth".to_string()), - proxy_auth_allow_insecure: Some(true), - proxy_connect_by_hostname: Some(false), - proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + no_proxy: Some("10.0.0.0/8".to_string()), + proxy_auth_file: Some(PathBuf::from("/etc/openshell/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: Some(true), + }, + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem")), + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, ..VmComputeConfig::default() }, ); @@ -761,20 +828,19 @@ mod tests { assert_eq!( args, [ - "--https-proxy", + "--upstream-proxy", "http://proxy.corp.com:8080", - "--no-proxy", + "--upstream-no-proxy", "10.0.0.0/8", - "--proxy-auth-file", + "--upstream-proxy-auth-file", "/etc/openshell/secrets/proxy-auth", - "--proxy-auth-allow-insecure", - "true", - // Passed as an explicit value, not a presence flag, so the - // driver still sees the operator's `false`. - "--proxy-connect-by-hostname", - "false", - "--proxy-ca-bundle", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--upstream-proxy-ca-bundle", "/etc/openshell/tls/proxy-ca.pem", + "--provider-spiffe-workload-api-tcp-endpoint", + "tcp:192.0.2.10:8081", + "--provider-spiffe-allow-guest-tcp", ] ); } @@ -782,38 +848,156 @@ mod tests { #[test] fn vm_driver_command_omits_unset_corporate_proxy_settings() { let mut command = tokio::process::Command::new("openshell-driver-vm"); - append_upstream_proxy_args(&mut command, &VmComputeConfig::default()); + append_vm_proxy_and_spiffe_args(&mut command, &VmComputeConfig::default()); assert_eq!(command.as_std().get_args().count(), 0); } #[test] fn invalid_corporate_proxy_config_is_rejected_before_the_driver_starts() { - // Without this the operator would see an opaque driver-readiness - // timeout instead of an error naming the offending key. - let err = VmComputeConfig { + let err = UpstreamProxyConfig { https_proxy: Some("socks5://proxy.corp.com:1080".to_string()), - ..VmComputeConfig::default() + ..Default::default() } - .validate_proxy_config() + .validate() .expect_err("only http:// and https:// proxies are supported"); - assert!(err.to_string().contains("https_proxy"), "{err}"); + assert!(err.contains("https_proxy"), "{err}"); - let err = VmComputeConfig { - proxy_ca_bundle: Some("/etc/openshell/tls/proxy-ca.pem".to_string()), - ..VmComputeConfig::default() + VmComputeConfig { + proxy_ca_bundle: Some(PathBuf::from("/etc/openshell/tls/proxy-ca.pem")), + ..Default::default() } .validate_proxy_config() - .expect_err("a CA bundle without a proxy URL would hide a fail-open state"); - assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); + .expect_err("a CA bundle without a proxy URL must fail closed"); VmComputeConfig { - https_proxy: Some("http://proxy.corp.com:8080".to_string()), - ..VmComputeConfig::default() + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("http://proxy.corp.com:8080".to_string()), + ..Default::default() + }, + ..Default::default() } .validate_proxy_config() .expect("a lone proxy URL is a complete configuration"); } + #[test] + fn vm_driver_command_includes_configured_sandbox_identity() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + let config = VmComputeConfig { + sandbox_uid: Some(2000), + sandbox_gid: Some(3000), + ..Default::default() + }; + + validate_vm_sandbox_identity(&config).expect("valid identity"); + append_vm_identity_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(args, ["--sandbox-uid", "2000", "--sandbox-gid", "3000"]); + } + + #[test] + fn vm_gateway_toml_forwards_rootfs_tar_settings() { + let config: VmComputeConfig = toml::from_str( + r#" + grpc_endpoint = "http://127.0.0.1:50051" + rootfs_tar_staging_dir = "/var/lib/openshell/rootfs-tar-staging" + rootfs_tar_max_bytes = 1073741824 + "#, + ) + .expect("schema-v2 VM rootfs tar settings must deserialize"); + config + .validate_configuration() + .expect("rootfs tar settings must pass static validation"); + + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_vm_rootfs_tar_args(&mut command, &config); + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + + assert_eq!( + args, + [ + "--rootfs-tar-staging-dir", + "/var/lib/openshell/rootfs-tar-staging", + "--rootfs-tar-max-bytes", + "1073741824", + ] + ); + } + + #[test] + fn vm_gateway_rejects_zero_rootfs_tar_limit() { + let config = VmComputeConfig { + grpc_endpoint: "http://127.0.0.1:50051".to_string(), + rootfs_tar_max_bytes: Some(0), + ..Default::default() + }; + + let error = config + .validate_configuration() + .expect_err("a zero rootfs tar limit must fail before driver startup"); + assert!(error.to_string().contains("rootfs_tar_max_bytes")); + } + + #[test] + fn vm_driver_command_forwards_proxy_and_spiffe_configuration_without_credentials() { + let config = VmComputeConfig { + upstream_proxy: UpstreamProxyConfig { + https_proxy: Some("https://proxy.internal:8443".to_string()), + no_proxy: Some("localhost,.svc".to_string()), + proxy_auth_file: Some(PathBuf::from("/gateway/secrets/proxy-auth")), + proxy_auth_allow_insecure: Some(true), + proxy_connect_by_hostname: Some(true), + }, + provider_spiffe_workload_api_tcp_endpoint: Some("tcp:192.0.2.10:8081".to_string()), + provider_spiffe_allow_guest_tcp: true, + ..Default::default() + }; + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_vm_proxy_and_spiffe_args(&mut command, &config); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "--upstream-proxy", + "https://proxy.internal:8443", + "--upstream-no-proxy", + "localhost,.svc", + "--upstream-proxy-auth-file", + "/gateway/secrets/proxy-auth", + "--upstream-proxy-auth-allow-insecure", + "--upstream-proxy-connect-by-hostname", + "--provider-spiffe-workload-api-tcp-endpoint", + "tcp:192.0.2.10:8081", + "--provider-spiffe-allow-guest-tcp", + ] + ); + assert!(!args.iter().any(|arg| arg.contains("user:password"))); + } + + #[test] + fn vm_gateway_config_rejects_root_identity() { + let config = VmComputeConfig { + sandbox_uid: Some(0), + ..Default::default() + }; + let error = validate_vm_sandbox_identity(&config).expect_err("root UID must fail"); + assert!(error.to_string().contains("sandbox_uid 0")); + } + #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 2619fee5cc..bf4c3f04e1 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -36,6 +36,7 @@ k8s-openapi = { workspace = true } tokio = { workspace = true } socket2 = { workspace = true } nix = { workspace = true } +libc = "0.2" # gRPC tonic = { workspace = true, features = ["channel", "tls-native-roots"] } diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 9dc10b8401..0a16000999 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -84,7 +84,7 @@ pub struct SandboxJwtIssuer { kid: String, issuer: String, audience: String, - ttl: Duration, + ttl: Option, } impl std::fmt::Debug for SandboxJwtIssuer { @@ -110,10 +110,14 @@ impl SandboxJwtIssuer { signing_key_pem: &[u8], kid: String, gateway_id: &str, - ttl: Duration, + ttl: Option, ) -> Result { crate::install_jsonwebtoken_crypto_provider(); + if ttl.is_some_and(|ttl| ttl.is_zero()) { + return Err("sandbox token TTL must be positive when configured".to_string()); + } + let encoding_key = EncodingKey::from_ed_pem(signing_key_pem) .map_err(|e| format!("failed to parse Ed25519 signing key PEM: {e}"))?; let identity = format!("openshell-gateway:{gateway_id}"); @@ -132,11 +136,9 @@ impl SandboxJwtIssuer { crate::install_jsonwebtoken_crypto_provider(); let now = now_secs(); - let exp = if self.ttl.is_zero() { - 0 - } else { - now.saturating_add(i64::try_from(self.ttl.as_secs()).unwrap_or(3_600)) - }; + let exp = self.ttl.map_or(0, |ttl| { + now.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(3_600)) + }); let claims = SandboxJwtClaims { sub: format!("{SPIFFE_SUBJECT_PREFIX}{sandbox_id}"), iss: self.issuer.clone(), @@ -226,7 +228,7 @@ impl SandboxJwtIssuer { }) } - pub fn ttl(&self) -> Duration { + pub fn sandbox_token_ttl(&self) -> Option { self.ttl } } @@ -417,10 +419,10 @@ mod tests { } fn pair() -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { - pair_with_ttl(Duration::from_secs(3600)) + pair_with_ttl(Some(Duration::from_secs(3600))) } - fn pair_with_ttl(ttl: Duration) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { + fn pair_with_ttl(ttl: Option) -> (SandboxJwtIssuer, SandboxJwtAuthenticator) { let mat = generate_jwt_key().expect("jwt key"); let issuer = SandboxJwtIssuer::from_pem( mat.signing_key_pem.as_bytes(), @@ -446,6 +448,7 @@ mod tests { async fn mint_and_validate_round_trip() { let (issuer, auth) = pair(); let minted = issuer.mint("sandbox-a").unwrap(); + assert!(minted.expires_at_ms > 0); let principal = auth .authenticate(&header_map_with_bearer(&minted.token), "/anything") .await @@ -472,7 +475,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); let auth = SandboxJwtAuthenticator::from_pem( @@ -514,8 +517,8 @@ mod tests { } #[tokio::test] - async fn ttl_zero_mints_non_expiring_token() { - let (issuer, auth) = pair_with_ttl(Duration::ZERO); + async fn ttl_none_mints_non_expiring_token() { + let (issuer, auth) = pair_with_ttl(None); let minted = issuer.mint("sandbox-never").unwrap(); assert_eq!(minted.expires_at_ms, 0); @@ -537,6 +540,19 @@ mod tests { assert_eq!(decoded.claims.exp, 0); } + #[test] + fn ttl_some_zero_is_rejected() { + let mat = generate_jwt_key().expect("jwt key"); + let error = SandboxJwtIssuer::from_pem( + mat.signing_key_pem.as_bytes(), + mat.kid, + "test-gateway", + Some(Duration::ZERO), + ) + .expect_err("Some(Duration::ZERO) must not reintroduce a sentinel"); + assert!(error.contains("must be positive")); + } + #[tokio::test] async fn token_signed_by_other_key_is_rejected() { let (_, auth_a) = pair(); @@ -582,7 +598,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "g", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); let auth = @@ -613,7 +629,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::ZERO, + None, ) .expect("issuer"); let decoding_key = DecodingKey::from_ed_pem(mat.public_key_pem.as_bytes()).unwrap(); @@ -662,7 +678,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid.clone(), "gateway-a", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .expect("issuer"); @@ -692,13 +708,9 @@ mod tests { #[test] fn extension_token_rejects_wrong_audience() { let mat = generate_jwt_key().expect("jwt key"); - let issuer = SandboxJwtIssuer::from_pem( - mat.signing_key_pem.as_bytes(), - mat.kid, - "gateway-a", - Duration::ZERO, - ) - .expect("issuer"); + let issuer = + SandboxJwtIssuer::from_pem(mat.signing_key_pem.as_bytes(), mat.kid, "gateway-a", None) + .expect("issuer"); let minted = issuer .mint_extension_token( &extension_audience("service-a"), @@ -734,7 +746,7 @@ mod tests { #[test] fn extension_token_enforces_positive_bounded_ttl_and_caller_shape() { - let (issuer, _) = pair_with_ttl(Duration::ZERO); + let (issuer, _) = pair_with_ttl(None); for ttl in [ Duration::ZERO, MAX_EXTENSION_TOKEN_TTL + Duration::from_secs(1), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 33008774b9..457f2dd209 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -7,6 +7,8 @@ use clap::parser::ValueSource; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; use openshell_core::config::{DEFAULT_GATEWAY_NAME, DEFAULT_SERVER_PORT}; +use std::collections::BTreeMap; +use std::ffi::OsString; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use tracing::{error, info, warn}; @@ -38,9 +40,34 @@ struct Cli { enum Commands { /// Generate mTLS PKI and write Kubernetes Secrets (Helm pre-install hook). GenerateCerts(certgen::CertgenArgs), + /// Inspect gateway configuration without starting the service. + Config(ConfigArgs), } #[derive(clap::Args, Debug)] +struct ConfigArgs { + #[command(subcommand)] + command: ConfigCommand, +} + +#[derive(clap::Subcommand, Debug)] +enum ConfigCommand { + /// Validate the selected configuration without modifying it or starting the gateway. + Preflight(ConfigPreflightArgs), +} + +#[derive(clap::Args, Debug, Default)] +struct ConfigPreflightArgs { + /// Explicit configuration path. Overrides `OPENSHELL_GATEWAY_CONFIG` and XDG discovery. + #[arg(long, conflicts_with = "gateway_args")] + path: Option, + + /// Gateway daemon arguments to replay after `--`. + #[arg(last = true, allow_hyphen_values = true, value_name = "GATEWAY_ARGS")] + gateway_args: Vec, +} + +#[derive(clap::Args, Clone, Debug)] #[allow(clippy::struct_excessive_bools)] struct RunArgs { /// Path to a TOML configuration file (see RFC 0003). @@ -101,29 +128,25 @@ struct RunArgs { #[arg(long, env = "OPENSHELL_DB_URL")] db_url: Option, - /// Compute drivers configured for this gateway. + /// Compute driver configured for this gateway. /// - /// Accepts a comma-delimited list of registered driver names. The - /// configuration format is future-proofed for multiple drivers, but the - /// gateway currently requires exactly one. When unset, the gateway runs + /// Accepts one registered driver name. When unset, the gateway runs /// detection probes supplied by the drivers compiled into the binary. #[arg( - long, - alias = "driver", - env = "OPENSHELL_DRIVERS", - value_delimiter = ',', + long = "compute-driver", + env = "OPENSHELL_COMPUTE_DRIVER", value_parser = parse_compute_driver )] - drivers: Vec, + compute_driver: Option, /// Path to a Unix domain socket served by a remote compute driver /// implementing `compute_driver.proto`. /// - /// When set, the socket is associated with the single driver name supplied - /// by `--drivers` or `OPENSHELL_DRIVERS` and replaces normal construction - /// for that selected name, including a compiled registration with the same - /// name. The gateway connects to this operator-provided endpoint; it does - /// not provision the remote driver. + /// When set, the socket is associated with the driver name supplied by + /// `--compute-driver` or `OPENSHELL_COMPUTE_DRIVER` and replaces normal + /// construction for that selected name, including a compiled registration + /// with the same name. The gateway connects to this operator-provided + /// endpoint; it does not provision the remote driver. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] compute_driver_socket: Option, @@ -239,10 +262,51 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry match cli.command { Some(Commands::GenerateCerts(args)) => certgen::run(args).await, + Some(Commands::Config(args)) => match args.command { + ConfigCommand::Preflight(args) => { + run_config_preflight_with_drivers(args, cli.run, &matches, &compute_drivers) + } + }, None => Box::pin(run_from_args(cli.run, matches, compute_drivers)).await, } } +fn resolve_legacy_driver_selector_env(args: &mut RunArgs) -> Result { + let Some(raw) = std::env::var_os("OPENSHELL_DRIVERS") else { + return Ok(false); + }; + let raw = raw.into_string().map_err(|_| { + miette::miette!( + "OPENSHELL_DRIVERS contains an invalid compute driver name; expected ASCII letters, digits, '-' or '_'" + ) + })?; + let values = raw.split(',').map(str::trim).collect::>(); + if values.len() != 1 || values[0].is_empty() { + return Err(miette::miette!( + "OPENSHELL_DRIVERS must contain exactly one non-empty compute driver name; comma-delimited lists are not supported" + )); + } + let legacy = openshell_core::config::normalize_compute_driver_name(values[0]).map_err(|_| { + miette::miette!( + "OPENSHELL_DRIVERS contains an invalid compute driver name; expected ASCII letters, digits, '-' or '_'" + ) + })?; + + if let Some(canonical) = args.compute_driver.as_deref() { + let canonical = openshell_core::config::normalize_compute_driver_name(canonical) + .map_err(|error| miette::miette!("{error}"))?; + if canonical != legacy { + return Err(miette::miette!( + "OPENSHELL_DRIVERS conflicts with the canonical compute-driver selection; remove OPENSHELL_DRIVERS" + )); + } + } else { + args.compute_driver = Some(legacy); + } + + Ok(true) +} + #[cfg(test)] fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result { prepare_server_config_with_drivers(args, matches, &ComputeDriverRegistry::new()) @@ -265,14 +329,20 @@ fn prepare_server_config_with_drivers( if let Some(file) = file.as_ref() { merge_file_into_args(args, &file.openshell.gateway, matches); } - normalize_compute_driver_socket_args(args, matches)?; + let legacy_compute_driver_env_seen = resolve_legacy_driver_selector_env(args)?; + normalize_compute_driver_socket_args(args)?; let compute_driver = compute_drivers - .select(&args.drivers) + .select(args.compute_driver.as_deref()) .map_err(|error| miette::miette!("{error}"))?; let selected_registration = compute_drivers.get(compute_driver.name()); let local_tls = apply_runtime_defaults(args)?; - let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); + let guest_tls = GuestTlsPaths::resolve( + file.as_ref().map(|file| &file.openshell.gateway), + local_tls.as_ref(), + args.disable_tls, + ) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; let local_jwt = defaults::complete_local_jwt_config()?; let bind = SocketAddr::new(args.bind_address, args.port); @@ -409,9 +479,11 @@ fn prepare_server_config_with_drivers( config = config.with_metrics_bind_address(addr); } + config = config.with_database_url(db_url); + if let Some(driver) = &args.compute_driver { + config = config.with_compute_driver(driver); + } config = config - .with_database_url(db_url) - .with_compute_drivers(args.drivers.clone()) .with_grpc_rate_limit( args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, @@ -444,8 +516,8 @@ fn prepare_server_config_with_drivers( )?; if let Some(socket) = args.compute_driver_socket.clone() { let driver = args - .drivers - .first() + .compute_driver + .as_ref() .expect("normalize_compute_driver_socket_args sets a driver for socket endpoints"); config = config.with_compute_driver_endpoint(driver.clone(), socket); } @@ -494,6 +566,7 @@ fn prepare_server_config_with_drivers( config_file: file, guest_tls, compute_driver, + legacy_compute_driver_env_seen, }) } @@ -526,6 +599,10 @@ async fn run_from_args( gateway_resource, ); + if prepared.legacy_compute_driver_env_seen { + warn!("OPENSHELL_DRIVERS is deprecated; migrate to OPENSHELL_COMPUTE_DRIVER"); + } + let has_client_ca = prepared .config .tls @@ -591,13 +668,290 @@ fn parse_compute_driver(value: &str) -> std::result::Result { openshell_core::config::normalize_compute_driver_name(value) } +#[cfg(test)] +#[derive(Clone, Copy)] +struct PreflightTestFactory; + +#[cfg(test)] +#[async_trait::async_trait] +impl crate::ComputeDriverFactory for PreflightTestFactory { + fn validate_config( + &self, + _context: crate::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Ok(()) + } + + async fn build( + &self, + _context: crate::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("config preflight tests must not build drivers") + } +} + +#[cfg(test)] +fn detect_preflight_test_driver() -> bool { + true +} + +#[cfg(test)] +fn run_config_preflight( + args: ConfigPreflightArgs, + run: RunArgs, + matches: &ArgMatches, +) -> Result<()> { + let mut registry = ComputeDriverRegistry::new(); + registry.install(crate::ComputeDriverRegistration::new( + "test", + 100, + Some(detect_preflight_test_driver), + PreflightTestFactory, + )?)?; + run_config_preflight_with_drivers(args, run, matches, ®istry) +} + +fn run_config_preflight_with_drivers( + args: ConfigPreflightArgs, + run: RunArgs, + matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, +) -> Result<()> { + if args.gateway_args.is_empty() { + return run_effective_config_preflight(args.path, run, matches, compute_drivers); + } + + let replay_matches = match command().try_get_matches_from( + std::iter::once(OsString::from("openshell-gateway")).chain(args.gateway_args), + ) { + Ok(matches) => matches, + Err(error) + if matches!( + error.kind(), + clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion + ) => + { + return Ok(()); + } + Err(error) => return Err(miette::miette!("{error}")), + }; + let replay = + Cli::from_arg_matches(&replay_matches).map_err(|error| miette::miette!("{error}"))?; + if replay.command.is_some() { + // A valid non-daemon action does not consume gateway startup + // configuration. Let the immediately following invocation perform it. + return Ok(()); + } + run_effective_config_preflight(None, replay.run, &replay_matches, compute_drivers) +} + +fn run_effective_config_preflight( + path_override: Option, + mut run: RunArgs, + matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, +) -> Result<()> { + let path = if path_override.is_some() { + path_override + } else { + resolve_config_path(&run)? + }; + let file = path + .as_ref() + .map(|path| config_file::preflight(path).map_err(|error| miette::miette!("{error}"))) + .transpose()?; + if let Some(file) = file.as_ref() { + merge_file_into_args(&mut run, &file.openshell.gateway, matches); + } + + let validation = (|| { + // These argument relationships are shared with daemon startup and + // remain transport-free. In particular, the deprecated selector must + // fail here exactly when the immediately following daemon invocation + // would fail. + resolve_legacy_driver_selector_env(&mut run)?; + normalize_compute_driver_socket_args(&mut run)?; + + let selection = run + .compute_driver + .as_deref() + .map(|driver| compute_drivers.select(Some(driver))) + .transpose() + .map_err(|error| miette::miette!("{error}"))?; + let selected_registration = selection + .as_ref() + .and_then(|selection| compute_drivers.get(selection.name())); + let empty_file = ConfigFile::default(); + let semantic_file = file.as_ref().unwrap_or(&empty_file); + validate_preflight_semantics(&run, matches, semantic_file, selected_registration)?; + + let mut endpoint_overrides = BTreeMap::new(); + if let Some(selection) = selection.as_ref() + && let Some(socket) = run.compute_driver_socket.clone() + { + endpoint_overrides.insert(selection.name().to_string(), socket); + } + let driver_startup = crate::compute::driver_config::DriverStartupContext { + file: file.as_ref(), + guest_tls: None, + gateway_port: run.port, + gateway_tls_enabled: !run.disable_tls, + endpoint_overrides: &endpoint_overrides, + }; + if let Some(selection) = selection.as_ref() { + crate::validate_compute_driver_config( + compute_drivers, + selection.name(), + run.name.trim(), + SocketAddr::new(run.bind_address, run.port), + &run.log_level, + driver_startup, + )?; + } else if file.is_some() { + // Runtime auto-detection may connect local API sockets or launch a + // bounded discovery command. Preflight must not perform those + // side effects, so validate every configured table that could be + // auto-selected. This fails closed without deserializing unrelated + // opt-in or remote driver tables. + for driver_name in compute_drivers.auto_detectable_driver_names() { + if !semantic_file.openshell.drivers.contains_key(driver_name) { + continue; + } + crate::validate_compute_driver_config( + compute_drivers, + driver_name, + run.name.trim(), + SocketAddr::new(run.bind_address, run.port), + &run.log_level, + driver_startup, + )?; + } + } + Ok(()) + })(); + + match (validation, path.as_ref()) { + (Ok(()), _) => Ok(()), + (Err(_), Some(path)) => Err(miette::miette!( + "{}", + config_file::ConfigPreflightError::invalid_current(path) + )), + (Err(error), None) => Err(error), + } +} + +fn validate_preflight_semantics( + args: &RunArgs, + matches: &ArgMatches, + file: &ConfigFile, + selected_registration: Option<&crate::ComputeDriverRegistration>, +) -> Result<()> { + let gateway = &file.openshell.gateway; + validate_grpc_rate_limit_args( + args.grpc_rate_limit_requests, + args.grpc_rate_limit_window_seconds, + )?; + GuestTlsPaths::validate_configuration(Some(gateway), args.disable_tls) + .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; + + let has_client_ca = args.tls_client_ca.is_some(); + let mtls_auth_enabled = + resolve_mtls_auth_enabled(args, matches, Some(file), selected_registration); + if args.disable_tls && has_client_ca { + return Err(miette::miette!( + "--disable-tls and --tls-client-ca are mutually exclusive" + )); + } + if mtls_auth_enabled && args.disable_tls { + return Err(miette::miette!("mTLS user authentication requires TLS")); + } + if mtls_auth_enabled && !has_client_ca { + return Err(miette::miette!( + "mTLS user authentication requires --tls-client-ca" + )); + } + if mtls_auth_enabled + && selected_registration.is_some_and(|registration| !registration.supports_mtls_user_auth()) + { + return Err(miette::miette!( + "mTLS user authentication is not supported with the selected compute driver" + )); + } + if !args.disable_tls && args.tls_cert.is_some() != args.tls_key.is_some() { + return Err(miette::miette!( + "gateway TLS requires both --tls-cert and --tls-key" + )); + } + if !args.disable_tls && has_client_ca && args.tls_cert.is_none() && args.tls_key.is_none() { + return Err(miette::miette!( + "an explicit --tls-client-ca requires --tls-cert and --tls-key" + )); + } + if !args.disable_tls + && let Some(tls) = gateway.tls.as_ref() + { + crate::tls::validate_external_cert_config( + tls.external_cert_path.as_deref(), + tls.external_key_path.as_deref(), + &tls.external_server_names, + ) + .map_err(|error| miette::miette!("{error}"))?; + } + openshell_gateway_interceptors::validate_configs(&gateway.interceptors) + .map_err(|error| miette::miette!("{error}"))?; + let mut middleware_names = std::collections::HashSet::new(); + for middleware in &file.openshell.supervisor.middleware { + let registration = openshell_core::proto::SupervisorMiddlewareService::try_from(middleware) + .map_err(|error| miette::miette!("{error}"))?; + openshell_supervisor_middleware::validate_registration_config(®istration)?; + if !middleware_names.insert(registration.name) { + return Err(miette::miette!( + "duplicate supervisor middleware registration" + )); + } + } + if args.name.trim().is_empty() { + return Err(miette::miette!("gateway name must not be empty")); + } + + let health_bind = resolve_aux_listener( + args.bind_address, + args.health_port, + matches, + "health_port", + || gateway.health_bind_address, + ); + let metrics_bind = resolve_aux_listener( + args.bind_address, + args.metrics_port, + matches, + "metrics_port", + || gateway.metrics_bind_address, + ); + if health_bind.is_some_and(|address| address.port() == args.port) + || metrics_bind.is_some_and(|address| address.port() == args.port) + || health_bind + .zip(metrics_bind) + .is_some_and(|(health, metrics)| health.port() == metrics.port()) + { + return Err(miette::miette!("gateway listener ports must be distinct")); + } + Ok(()) +} + fn resolve_config_path(args: &RunArgs) -> Result> { if let Some(path) = args.config.clone() { return Ok(Some(path)); } let default_path = defaults::default_gateway_config_path()?; - Ok(default_path.is_file().then_some(default_path)) + match std::fs::symlink_metadata(&default_path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + // Fail closed when the path exists or cannot be inspected. Returning it + // lets the shared loader produce the same safe, path-specific error used + // for an explicit configuration instead of treating it as absent. + Ok(_) | Err(_) => Ok(Some(default_path)), + } } fn apply_runtime_defaults(args: &mut RunArgs) -> Result> { @@ -697,10 +1051,10 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.log_level.clone_from(level); } - if let Some(drivers) = &file.compute_drivers - && arg_defaulted(matches, "drivers") + if let Some(driver) = &file.compute_driver + && arg_defaulted(matches, "compute_driver") { - args.drivers.clone_from(drivers); + args.compute_driver = Some(driver.clone()); } if let Some(sans) = &file.server_sans && args.server_sans.is_empty() @@ -792,7 +1146,7 @@ fn validate_grpc_rate_limit_args(requests: Option, window_seconds: Option Result<()> { +fn normalize_compute_driver_socket_args(args: &mut RunArgs) -> Result<()> { let Some(socket) = args.compute_driver_socket.as_ref() else { return Ok(()); }; @@ -801,24 +1155,21 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches "--compute-driver-socket must not be an empty path" )); } - if arg_defaulted(matches, "drivers") { + if args.compute_driver.is_none() { return Err(miette::miette!( - "--compute-driver-socket requires --drivers or OPENSHELL_DRIVERS= to select a compute driver name" + "--compute-driver-socket requires --compute-driver or OPENSHELL_COMPUTE_DRIVER=" )); } - match args.drivers.as_slice() { - [driver] => { - let driver = openshell_core::config::normalize_compute_driver_name(driver) - .map_err(|err| miette::miette!("{err}"))?; - args.drivers[0] = driver; - Ok(()) - } - drivers => Err(miette::miette!( - "--compute-driver-socket requires exactly one compute driver name, got: {}", - drivers.join(",") - )), - } + let driver = args + .compute_driver + .as_deref() + .expect("explicit compute driver is required for socket endpoints"); + args.compute_driver = Some( + openshell_core::config::normalize_compute_driver_name(driver) + .map_err(|err| miette::miette!("{err}"))?, + ); + Ok(()) } fn is_singleplayer_driver(registration: Option<&crate::ComputeDriverRegistration>) -> bool { @@ -854,6 +1205,7 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; static REGISTRY_DETECTION_CALLS: AtomicUsize = AtomicUsize::new(0); + static REJECTING_VALIDATION_CALLS: AtomicUsize = AtomicUsize::new(0); fn detect_registered_local() -> bool { REGISTRY_DETECTION_CALLS.fetch_add(1, Ordering::SeqCst); @@ -865,6 +1217,13 @@ mod tests { #[async_trait::async_trait] impl crate::ComputeDriverFactory for TestFactory { + fn validate_config( + &self, + _context: crate::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Ok(()) + } + async fn build( &self, _context: crate::ComputeDriverBuildContext<'_>, @@ -873,6 +1232,29 @@ mod tests { } } + #[derive(Clone, Copy)] + struct RejectingValidationFactory; + + #[async_trait::async_trait] + impl crate::ComputeDriverFactory for RejectingValidationFactory { + fn validate_config( + &self, + _context: crate::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + REJECTING_VALIDATION_CALLS.fetch_add(1, Ordering::SeqCst); + Err(openshell_core::Error::config( + "selected driver validation hook invoked", + )) + } + + async fn build( + &self, + _context: crate::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("preflight must not build the selected driver") + } + } + fn test_registry(name: &str, singleplayer: bool, mtls: bool) -> crate::ComputeDriverRegistry { let mut registration = crate::ComputeDriverRegistration::new(name, 100, None, TestFactory).unwrap(); @@ -1112,6 +1494,139 @@ mod tests { assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); } + #[test] + fn command_rejects_legacy_compute_driver_flags() { + for flag in ["--driver", "--drivers"] { + let err = command() + .try_get_matches_from([ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + flag, + "docker", + ]) + .expect_err("legacy compute driver selector must be rejected"); + assert_eq!(err.kind(), clap::error::ErrorKind::UnknownArgument); + } + } + + #[test] + fn legacy_compute_driver_environment_accepts_one_normalized_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", " PodMan "); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + + assert!(super::resolve_legacy_driver_selector_env(&mut args).unwrap()); + assert_eq!(args.compute_driver.as_deref(), Some("podman")); + } + + #[test] + fn legacy_compute_driver_environment_flows_through_server_preparation() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_path = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman"); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--disable-tls", + ]); + let registry = test_registry("podman", true, true); + + let prepared = + super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); + + assert_eq!(prepared.compute_driver.name(), "podman"); + assert!(prepared.legacy_compute_driver_env_seen); + } + + #[test] + fn legacy_compute_driver_environment_rejects_empty_plural_and_invalid_values() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + + for value in ["", " ", ",", "podman,", ",podman", "podman,docker"] { + let legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", value); + let (mut args, _) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("empty and plural legacy selectors must be rejected"); + assert!(error.to_string().contains("exactly one non-empty")); + drop(legacy); + } + + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman/path"); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("invalid legacy selector must be rejected"); + assert!(error.to_string().contains("invalid compute driver name")); + assert!(!error.to_string().contains("podman/path")); + } + + #[test] + fn legacy_compute_driver_environment_allows_equal_canonical_and_rejects_conflict() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "PODMAN"); + + let (mut equal, _) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + ]); + assert!(super::resolve_legacy_driver_selector_env(&mut equal).unwrap()); + assert_eq!(equal.compute_driver.as_deref(), Some("podman")); + + let (mut conflict, _) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "docker", + ]); + let error = super::resolve_legacy_driver_selector_env(&mut conflict) + .expect_err("different canonical and legacy selectors must conflict"); + assert!(error.to_string().contains("conflicts")); + assert!(!error.to_string().contains("podman")); + assert!(!error.to_string().contains("docker")); + } + + #[test] + fn legacy_compute_driver_environment_supports_remote_driver_socket() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _socket = EnvVarGuard::set( + "OPENSHELL_COMPUTE_DRIVER_SOCKET", + "/run/openshell/kyma.sock", + ); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + + assert!(super::resolve_legacy_driver_selector_env(&mut args).unwrap()); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); + assert_eq!( + args.compute_driver_socket.as_deref(), + Some(std::path::Path::new("/run/openshell/kyma.sock")) + ); + } + #[test] fn command_rejects_removed_ssh_endpoint_flags() { for flag in [ @@ -1212,6 +1727,531 @@ mod tests { )); } + #[test] + fn config_preflight_subcommand_parses_without_runtime_requirements() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _db = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + + let cli = Cli::try_parse_from([ + "openshell-gateway", + "config", + "preflight", + "--path", + "/tmp/gateway.toml", + ]) + .expect("config preflight should parse without runtime arguments"); + + assert!(matches!( + cli.command, + Some(super::Commands::Config(super::ConfigArgs { + command: super::ConfigCommand::Preflight(_) + })) + )); + } + + #[test] + fn config_preflight_path_and_daemon_replay_are_mutually_exclusive() { + let error = command() + .try_get_matches_from([ + "openshell-gateway", + "config", + "preflight", + "--path", + "/tmp/gateway.toml", + "--", + "--disable-tls", + ]) + .expect_err("manual path and daemon replay must be mutually exclusive"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + #[test] + fn config_preflight_replay_validates_effective_daemon_flags() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _requests = EnvVarGuard::remove("OPENSHELL_GRPC_RATE_LIMIT_REQUESTS"); + let _window = EnvVarGuard::remove("OPENSHELL_GRPC_RATE_LIMIT_WINDOW_SECONDS"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + let error = super::run_config_preflight( + super::ConfigPreflightArgs { + gateway_args: ["--grpc-rate-limit-requests", "10"] + .map(std::ffi::OsString::from) + .to_vec(), + ..Default::default() + }, + run.clone(), + &matches, + ) + .expect_err("unpaired replayed rate limit must fail preflight"); + assert!(error.to_string().contains("requires both")); + + super::run_config_preflight( + super::ConfigPreflightArgs { + gateway_args: [ + "--grpc-rate-limit-requests", + "10", + "--grpc-rate-limit-window-seconds", + "60", + ] + .map(std::ffi::OsString::from) + .to_vec(), + ..Default::default() + }, + run, + &matches, + ) + .expect("paired replayed rate limit must pass preflight"); + } + + #[test] + fn config_preflight_matches_driver_selector_and_registry_semantics() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "podman,docker"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let registry = test_registry("podman", true, true); + + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("plural legacy selector must fail preflight as it fails startup"); + assert!(error.to_string().contains("exactly one non-empty")); + } + + #[test] + fn config_preflight_validates_selected_driver_without_building_it() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "local", + "--disable-tls", + ]); + let mut registry = crate::ComputeDriverRegistry::new(); + registry + .install( + crate::ComputeDriverRegistration::new( + "local", + 100, + None, + RejectingValidationFactory, + ) + .unwrap(), + ) + .unwrap(); + + REJECTING_VALIDATION_CALLS.store(0, Ordering::SeqCst); + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("selected driver validation hook must run"); + assert!(error.to_string().contains("validation hook invoked")); + assert_eq!(REJECTING_VALIDATION_CALLS.load(Ordering::SeqCst), 1); + } + + #[test] + fn config_preflight_validates_configured_auto_detectable_driver_without_probing() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let config = config_home.path().join("gateway.toml"); + std::fs::write( + &config, + "[openshell]\nversion = 2\n[openshell.gateway]\ndisable_tls = true\n[openshell.drivers.local]\n", + ) + .unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let mut registry = crate::ComputeDriverRegistry::new(); + registry + .install( + crate::ComputeDriverRegistration::new( + "local", + 100, + Some(detect_registered_local), + RejectingValidationFactory, + ) + .unwrap(), + ) + .unwrap(); + + REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); + REJECTING_VALIDATION_CALLS.store(0, Ordering::SeqCst); + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, + run, + &matches, + ®istry, + ) + .expect_err("auto-detectable driver table validation hook must run"); + + assert!(error.to_string().contains("category=malformed")); + assert!(!error.to_string().contains("validation hook invoked")); + assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 0); + assert_eq!(REJECTING_VALIDATION_CALLS.load(Ordering::SeqCst), 1); + } + + #[test] + fn config_preflight_auto_validation_skips_absent_and_opt_in_driver_tables() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let config = config_home.path().join("gateway.toml"); + std::fs::write( + &config, + "[openshell]\nversion = 2\n[openshell.gateway]\ndisable_tls = true\n[openshell.drivers.opt-in]\n", + ) + .unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let mut registry = crate::ComputeDriverRegistry::new(); + for registration in [ + crate::ComputeDriverRegistration::new( + "local", + 100, + Some(detect_registered_local), + RejectingValidationFactory, + ) + .unwrap(), + crate::ComputeDriverRegistration::new("opt-in", 200, None, RejectingValidationFactory) + .unwrap(), + ] { + registry.install(registration).unwrap(); + } + + REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); + REJECTING_VALIDATION_CALLS.store(0, Ordering::SeqCst); + super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, + run, + &matches, + ®istry, + ) + .expect("unconfigured auto-detectable and configured opt-in drivers must be skipped"); + + assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 0); + assert_eq!(REJECTING_VALIDATION_CALLS.load(Ordering::SeqCst), 0); + } + + #[test] + fn config_preflight_applies_selected_driver_mtls_capability() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "shared", + "--tls-cert", + "/tls/server.pem", + "--tls-key", + "/tls/server-key.pem", + "--tls-client-ca", + "/tls/ca.pem", + "--enable-mtls-auth", + "true", + ]); + let registry = test_registry("shared", false, false); + + let error = super::run_config_preflight_with_drivers( + super::ConfigPreflightArgs::default(), + run, + &matches, + ®istry, + ) + .expect_err("selected shared driver must reject mTLS user authentication"); + assert!(error.to_string().contains("not supported")); + } + + #[test] + fn config_preflight_validates_explicit_remote_driver_endpoint() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let (run, matches) = parse_with_args(&[ + "openshell-gateway", + "--compute-driver", + "remote", + "--disable-tls", + ]); + + let error = + super::run_config_preflight(super::ConfigPreflightArgs::default(), run, &matches) + .expect_err("remote driver without socket_path must fail preflight"); + assert!(error.to_string().contains("requires socket_path")); + } + + #[test] + fn config_preflight_validates_explicit_path_without_creating_state() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let state_parent = tempfile::tempdir().unwrap(); + let state_home = state_parent.path().join("not-created"); + let config = config_home.path().join("gateway.toml"); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _state = EnvVarGuard::set("XDG_STATE_HOME", state_home.to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, + run, + &matches, + ) + .expect("valid explicit config passes preflight"); + + assert!( + !state_home.exists(), + "preflight must not create runtime state" + ); + } + + #[test] + fn config_preflight_explicit_path_overrides_environment_selection() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let dir = tempfile::tempdir().unwrap(); + let legacy = dir.path().join("legacy.toml"); + let current = dir.path().join("current.toml"); + std::fs::write(&legacy, "[openshell]\nversion = 1\n").unwrap(); + std::fs::write(¤t, "[openshell]\nversion = 2\n").unwrap(); + let _config_env = EnvVarGuard::set("OPENSHELL_GATEWAY_CONFIG", legacy.to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + let error = super::run_config_preflight( + super::ConfigPreflightArgs::default(), + run.clone(), + &matches, + ) + .expect_err("environment-selected legacy config must fail"); + assert!(error.to_string().contains("category=legacy_schema_v1")); + + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(current), + ..Default::default() + }, + run, + &matches, + ) + .expect("explicit preflight path must override environment selection"); + } + + #[test] + fn config_preflight_allows_absent_auto_discovery_but_rejects_explicit_absence() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _config_home = + EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight(super::ConfigPreflightArgs::default(), run.clone(), &matches) + .expect("absent auto-discovered config is optional"); + + let missing = config_home.path().join("missing.toml"); + let error = super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(missing.clone()), + ..Default::default() + }, + run, + &matches, + ) + .expect_err("explicit missing config must fail"); + assert!(error.to_string().contains("category=missing_path")); + assert!(error.to_string().contains(&missing.display().to_string())); + } + + #[test] + fn config_preflight_rejects_effective_semantic_errors() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let cases = [ + ( + "driver-selector", + "[openshell]\nversion = 2\n[openshell.gateway]\ncompute_driver = 'secret-driver-marker'\ndisable_tls = true\n", + ), + ( + "rate-limit", + "[openshell]\nversion = 2\n[openshell.gateway]\nname = 'secret-semantic-marker'\ngrpc_rate_limit_requests = 10\n", + ), + ( + "guest-tls", + "[openshell]\nversion = 2\n[openshell.gateway]\nguest_tls_ca = '/tls/ca.pem'\n", + ), + ( + "external-tls", + "[openshell]\nversion = 2\n[openshell.gateway.tls]\ncert_path = '/tls/server.pem'\nkey_path = '/tls/server-key.pem'\nexternal_cert_path = '/tls/external.pem'\nexternal_server_names = ['external.example']\n", + ), + ( + "interceptor", + "[openshell]\nversion = 2\n[[openshell.gateway.interceptors]]\nname = ''\ngrpc_endpoint = 'https://interceptor.example'\n", + ), + ( + "middleware", + "[openshell]\nversion = 2\n[[openshell.supervisor.middleware]]\nname = 'guard'\ngrpc_endpoint = 'http://127.0.0.1:50051'\nallow_insecure_transport = true\nmax_payload_bytes = 1024\ntimeout = 'invalid'\n", + ), + ]; + + for (name, contents) in cases { + let path = dir.path().join(format!("{name}.toml")); + std::fs::write(&path, contents).unwrap(); + let before = std::fs::read(&path).unwrap(); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let result = super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(path.clone()), + ..Default::default() + }, + run, + &matches, + ); + let Err(error) = result else { + panic!("{name}: invalid effective configuration passed preflight"); + }; + assert!(error.to_string().contains("category=malformed"), "{name}"); + assert!(error.to_string().contains("detected_version=2"), "{name}"); + assert!(!error.to_string().contains("secret-")); + assert!(!format!("{error:?}").contains("secret-")); + assert_eq!(std::fs::read(&path).unwrap(), before, "{name}"); + } + } + + #[test] + fn config_preflight_matches_effective_tls_environment_semantics() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let partial_external = dir.path().join("partial-external.toml"); + std::fs::write( + &partial_external, + "[openshell]\nversion = 2\n[openshell.gateway.tls]\ncert_path = '/tls/server.pem'\nkey_path = '/tls/server-key.pem'\nexternal_cert_path = '/tls/external.pem'\nexternal_server_names = ['external.example']\n", + ) + .unwrap(); + let disable_tls = EnvVarGuard::set("OPENSHELL_DISABLE_TLS", "true"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(partial_external), + ..Default::default() + }, + run, + &matches, + ) + .expect("inactive TLS table must not block an effective plaintext gateway"); + drop(disable_tls); + + let config = dir.path().join("client-ca-only.toml"); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); + let _disable_tls = EnvVarGuard::remove("OPENSHELL_DISABLE_TLS"); + let _client_ca = EnvVarGuard::set("OPENSHELL_TLS_CLIENT_CA", "/tls/ca.pem"); + let _cert = EnvVarGuard::remove("OPENSHELL_TLS_CERT"); + let _key = EnvVarGuard::remove("OPENSHELL_TLS_KEY"); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + let error = super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(config), + ..Default::default() + }, + run, + &matches, + ) + .expect_err("client CA without an explicit server pair must fail before cert generation"); + assert!(error.to_string().contains("category=malformed")); + } + + #[test] + fn config_preflight_allows_complete_future_generated_tls_paths() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _config_env = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("gateway.toml"); + std::fs::write( + &path, + "[openshell]\nversion = 2\n[openshell.gateway]\nguest_tls_ca = '/future/ca.pem'\nguest_tls_cert = '/future/client.pem'\nguest_tls_key = '/future/client-key.pem'\n", + ) + .unwrap(); + let (run, matches) = parse_with_args(&["openshell-gateway"]); + + super::run_config_preflight( + super::ConfigPreflightArgs { + path: Some(path), + ..Default::default() + }, + run, + &matches, + ) + .expect("complete package-generated TLS paths may not exist before certificate generation"); + } + #[test] fn bare_invocation_with_no_db_url_parses_for_runtime_defaults() { // db_url is Option at the clap level so subcommand parsing @@ -1249,6 +2289,14 @@ mod tests { toml::from_str(toml).expect("valid TOML in test fixture") } + #[test] + fn rejects_legacy_drivers_flag() { + let error = command() + .try_get_matches_from(["openshell-gateway", "--drivers", "docker"]) + .expect_err("legacy --drivers flag must be rejected"); + assert!(error.to_string().contains("--drivers")); + } + #[test] fn default_config_path_is_loaded_only_when_present() { let _lock = ENV_LOCK @@ -1263,7 +2311,7 @@ mod tests { let config = tmp.path().join("openshell").join("gateway.toml"); std::fs::create_dir_all(config.parent().unwrap()).unwrap(); - std::fs::write(&config, "[openshell]\nversion = 1\n").unwrap(); + std::fs::write(&config, "[openshell]\nversion = 2\n").unwrap(); assert_eq!(super::resolve_config_path(&args).unwrap(), Some(config)); } @@ -1346,6 +2394,46 @@ mod tests { assert_eq!(local_tls.client_cert, tls.path().join("client/tls.crt")); } + #[test] + fn tls_client_certificate_requirement_is_derived_from_ca_and_oidc() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let config_home = tempfile::tempdir().unwrap(); + let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap()); + let _config_path = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG"); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let registry = test_registry("shared", false, false); + + for (oidc_issuer, expected) in [(None, true), (Some("https://idp.example.com"), false)] { + let mut startup_args = vec![ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "shared", + "--tls-cert", + "/tls/server.crt", + "--tls-key", + "/tls/server.key", + "--tls-client-ca", + "/tls/ca.crt", + ]; + if let Some(issuer) = oidc_issuer { + startup_args.extend(["--oidc-issuer", issuer]); + } + let (mut args, matches) = parse_with_args(&startup_args); + let prepared = + super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); + + assert_eq!( + prepared.config.tls.as_ref().unwrap().require_client_auth, + expected, + "oidc issuer: {oidc_issuer:?}" + ); + } + } + #[test] fn mtls_auth_auto_defaults_for_local_tls_driver() { let _lock = ENV_LOCK @@ -1357,7 +2445,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1385,7 +2473,7 @@ mod tests { let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); - let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _drivers = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); let (mut args, matches) = parse_with_args(&[ @@ -1405,7 +2493,7 @@ mod tests { super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); assert_eq!(prepared.compute_driver.name(), "local"); - assert!(prepared.config.compute_drivers.is_empty()); + assert!(prepared.config.compute_driver.is_none()); assert!(prepared.config.mtls_auth.enabled); assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } @@ -1421,7 +2509,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "shared", "--tls-cert", "/tmp/server.crt", @@ -1450,7 +2538,7 @@ mod tests { "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "local", "--tls-cert", "/tmp/server.crt", @@ -1558,6 +2646,65 @@ log_level = "debug" assert_eq!(args.name, "env-gateway"); } + #[test] + fn compute_driver_file_value_and_cli_environment_precedence_are_explicit() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let file = config_file_from_toml( + r#" +[openshell.gateway] +compute_driver = "podman" +"#, + ); + + let canonical_guard = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let (mut file_args, file_matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut file_args, &file.openshell.gateway, &file_matches); + assert_eq!(file_args.compute_driver.as_deref(), Some("podman")); + + let (mut cli_args, cli_matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver", + "docker", + ]); + merge_file_into_args(&mut cli_args, &file.openshell.gateway, &cli_matches); + assert_eq!(cli_args.compute_driver.as_deref(), Some("docker")); + drop(canonical_guard); + + let _canonical = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "vm"); + let (mut env_args, env_matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut env_args, &file.openshell.gateway, &env_matches); + assert_eq!(env_args.compute_driver.as_deref(), Some("vm")); + } + + #[test] + fn legacy_compute_driver_environment_conflicts_with_file_selection() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _canonical = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); + let _legacy = EnvVarGuard::set("OPENSHELL_DRIVERS", "docker"); + let file = config_file_from_toml( + r#" +[openshell.gateway] +compute_driver = "podman" +"#, + ); + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + let error = super::resolve_legacy_driver_selector_env(&mut args) + .expect_err("different file and legacy selectors must conflict"); + assert!(error.to_string().contains("conflicts")); + } + #[test] fn file_oidc_block_populates_oidc_args() { let _lock = ENV_LOCK @@ -1724,23 +2871,23 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "Kyma", "--compute-driver-socket", "/run/openshell/kyma.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!( args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1749,19 +2896,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", "--compute-driver-socket", "/run/openshell/kyma.sock", ]); - let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); + let err = super::normalize_compute_driver_socket_args(&mut args).unwrap_err(); assert!( - err.to_string().contains("requires --drivers "), + err.to_string().contains("requires --compute-driver "), "unexpected error: {err}" ); } @@ -1772,19 +2919,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "docker", "--compute-driver-socket", "/run/openshell/extension.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["docker"]); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); + assert_eq!(args.compute_driver.as_deref(), Some("docker")); } #[test] @@ -1793,19 +2940,19 @@ ssh_session_ttl_secs = 1234 .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); - let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + let _g2 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER"); - let (mut args, matches) = parse_with_args(&[ + let (mut args, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "vm", "--compute-driver-socket", "/run/openshell/vm.sock", ]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); - assert_eq!(args.drivers, ["vm"]); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); + assert_eq!(args.compute_driver.as_deref(), Some("vm")); } #[test] @@ -1817,16 +2964,15 @@ ssh_session_ttl_secs = 1234 "OPENSHELL_COMPUTE_DRIVER_SOCKET", "/var/run/openshell/kyma.sock", ); - let _g2 = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + let _g2 = EnvVarGuard::set("OPENSHELL_COMPUTE_DRIVER", "kyma"); - let (mut args, matches) = - parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); - super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + let (mut args, _) = parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + super::normalize_compute_driver_socket_args(&mut args).unwrap(); assert_eq!( args.compute_driver_socket.as_deref(), Some(std::path::Path::new("/var/run/openshell/kyma.sock")) ); - assert_eq!(args.drivers, ["kyma"]); + assert_eq!(args.compute_driver.as_deref(), Some("kyma")); } #[test] @@ -1881,6 +3027,17 @@ enable_loopback_service_http = false ); } + #[test] + fn canonical_file_driver_selector_populates_cli_args() { + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml("[openshell.gateway]\ncompute_driver = \"podman\"\n"); + + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + assert_eq!(args.compute_driver.as_deref(), Some("podman")); + } + #[test] fn server_config_preparation_ignores_unselected_driver_tables() { let _lock = ENV_LOCK @@ -1897,6 +3054,9 @@ enable_loopback_service_http = false std::fs::write( &config_path, r#" +[openshell] +version = 2 + [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" @@ -1915,7 +3075,7 @@ mem_mib = "not-a-number" config_path.to_str().unwrap(), "--db-url", "sqlite::memory:", - "--drivers", + "--compute-driver", "podman", "--disable-tls", ]); @@ -1923,7 +3083,7 @@ mem_mib = "not-a-number" let prepared = super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); - assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); + assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index d06e6fbc8f..0c892dc373 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -27,13 +27,81 @@ impl GuestTlsPaths { } } -impl From<&LocalTlsPaths> for GuestTlsPaths { - fn from(paths: &LocalTlsPaths) -> Self { - Self { +impl GuestTlsPaths { + fn configured_paths( + gateway: &config_file::GatewayFileSection, + ) -> (Option<&PathBuf>, Option<&PathBuf>, Option<&PathBuf>) { + ( + gateway.guest_tls_ca.as_ref(), + gateway.guest_tls_cert.as_ref(), + gateway.guest_tls_key.as_ref(), + ) + } + + /// Validate guest TLS relationships without reading certificate files. + pub(crate) fn validate_configuration( + gateway: Option<&config_file::GatewayFileSection>, + tls_disabled: bool, + ) -> std::result::Result<(), String> { + let configured = gateway.map(Self::configured_paths); + let provided = configured + .is_some_and(|(ca, cert, key)| ca.is_some() || cert.is_some() || key.is_some()); + if tls_disabled && provided { + return Err( + "guest_tls_ca, guest_tls_cert, and guest_tls_key require gateway TLS; remove them or omit --disable-tls" + .to_string(), + ); + } + if let Some((ca, cert, key)) = configured + && (ca.is_some() || cert.is_some() || key.is_some()) + && (ca.is_none() || cert.is_none() || key.is_none()) + { + return Err( + "guest TLS requires one complete bundle: guest_tls_ca, guest_tls_cert, and guest_tls_key" + .to_string(), + ); + } + Ok(()) + } + + /// Resolve gateway-owned guest TLS inputs. Explicit TOML values take + /// precedence over the package-managed local bundle; partial bundles are + /// rejected before any driver is deserialized or constructed. + pub(crate) fn resolve( + gateway: Option<&config_file::GatewayFileSection>, + local: Option<&LocalTlsPaths>, + tls_disabled: bool, + ) -> std::result::Result, String> { + Self::validate_configuration(gateway, tls_disabled)?; + if tls_disabled { + return Ok(None); + } + + if let Some((Some(ca), Some(cert), Some(key))) = gateway.map(Self::configured_paths) { + for (field, path) in [ + ("guest_tls_ca", ca), + ("guest_tls_cert", cert), + ("guest_tls_key", key), + ] { + if !path.is_file() { + return Err(format!( + "{field} '{}' does not exist or is not a file", + path.display() + )); + } + } + return Ok(Some(Self { + ca: ca.clone(), + cert: cert.clone(), + key: key.clone(), + })); + } + + Ok(local.map(|paths| Self { ca: paths.ca.clone(), cert: paths.client_cert.clone(), key: paths.client_key.clone(), - } + })) } } @@ -57,6 +125,7 @@ pub fn remote_driver_config_from_context( &file.openshell.gateway, file.openshell.drivers.get(name), ); + reject_driver_owned_guest_tls_fields(&merged)?; if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { cfg.socket_path = PathBuf::from(socket_path); } @@ -100,6 +169,7 @@ where file.openshell.drivers.get(driver_name), inherited_config_keys, ); + reject_driver_owned_guest_tls_fields(&merged)?; merged.try_into().map_err(|e| { Error::config(format!( "invalid [openshell.drivers.{driver_name}] table: {e}" @@ -107,6 +177,23 @@ where }) } +/// Reject TLS paths in gateway driver tables. These credentials are gateway +/// inputs and are injected only into the selected local driver after the +/// gateway has validated the complete bundle. +fn reject_driver_owned_guest_tls_fields(table: &toml::Value) -> Result<()> { + let Some(table) = table.as_table() else { + return Ok(()); + }; + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + if table.contains_key(field) { + return Err(Error::config(format!( + "{field} belongs in [openshell.gateway], not a [openshell.drivers.*] table" + ))); + } + } + Ok(()) +} + fn apply_remote_driver_overrides( cfg: &mut RemoteDriverConfig, context: DriverStartupContext<'_>, @@ -130,6 +217,7 @@ fn validate_remote_driver_config(cfg: &RemoteDriverConfig, name: &str) -> Result mod tests { use super::*; use std::collections::BTreeMap; + use std::path::Path; fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = @@ -150,6 +238,210 @@ mod tests { } } + #[test] + fn gateway_guest_tls_resolves_explicit_complete_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let ca = dir.path().join("ca.pem"); + let cert = dir.path().join("cert.pem"); + let key = dir.path().join("key.pem"); + for path in [&ca, &cert, &key] { + std::fs::write(path, b"test").expect("write TLS fixture"); + } + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(ca.clone()), + guest_tls_cert: Some(cert.clone()), + guest_tls_key: Some(key.clone()), + ..Default::default() + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect("complete guest TLS should resolve") + .expect("guest TLS bundle"); + + assert_eq!( + resolved.as_paths(), + (ca.as_path(), cert.as_path(), key.as_path()) + ); + } + + #[test] + fn gateway_guest_tls_rejects_every_partial_bundle() { + let path = PathBuf::from("/tmp/guest-tls.pem"); + for (ca, cert, key) in [ + (Some(path.clone()), None, None), + (None, Some(path.clone()), None), + (None, None, Some(path.clone())), + (Some(path.clone()), Some(path.clone()), None), + (Some(path.clone()), None, Some(path.clone())), + (None, Some(path.clone()), Some(path)), + ] { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: ca, + guest_tls_cert: cert, + guest_tls_key: key, + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("partial guest TLS must fail"); + assert!(error.contains("one complete bundle")); + } + } + + #[test] + fn gateway_guest_tls_rejects_missing_explicit_file() { + let dir = tempfile::tempdir().expect("temp dir"); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(dir.path().join("missing-ca.pem")), + guest_tls_cert: Some(dir.path().join("missing-cert.pem")), + guest_tls_key: Some(dir.path().join("missing-key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("missing explicit file must fail"); + assert!(error.contains("guest_tls_ca")); + assert!(error.contains("does not exist")); + } + + #[test] + fn gateway_guest_tls_uses_package_managed_bundle() { + let local = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + let resolved = GuestTlsPaths::resolve(None, Some(&local), false) + .expect("managed bundle should resolve") + .expect("guest TLS bundle"); + assert_eq!( + resolved.as_paths(), + ( + Path::new("/managed/ca.pem"), + Path::new("/managed/client-cert.pem"), + Path::new("/managed/client-key.pem"), + ) + ); + } + + #[test] + fn gateway_guest_tls_can_be_absent() { + assert!(GuestTlsPaths::resolve(None, None, false).unwrap().is_none()); + assert!(GuestTlsPaths::resolve(None, None, true).unwrap().is_none()); + } + + #[test] + fn gateway_guest_tls_rejects_plaintext_gateway() { + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(PathBuf::from("/tmp/ca.pem")), + guest_tls_cert: Some(PathBuf::from("/tmp/cert.pem")), + guest_tls_key: Some(PathBuf::from("/tmp/key.pem")), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, true) + .expect_err("guest TLS and plaintext gateway conflict"); + assert!(error.contains("require gateway TLS")); + } + + #[derive(Debug, Default, Deserialize)] + struct EmptyDriverConfig {} + + #[test] + fn driver_owned_guest_tls_fields_are_rejected_for_local_and_remote_drivers() { + for field in ["guest_tls_ca", "guest_tls_cert", "guest_tls_key"] { + let source = format!( + r#" +[openshell] +version = 2 + +[openshell.drivers.kyma] +socket_path = "/run/openshell/kyma.sock" +{field} = "/run/openshell/guest.pem" +"# + ); + let file: config_file::ConfigFile = toml::from_str(&source).expect("valid TOML"); + + let local_error = driver_config_from_context::( + test_context(Some(&file)), + "kyma", + &[], + ) + .expect_err("local driver TLS field must be rejected"); + assert!(local_error.to_string().contains(field)); + assert!(local_error.to_string().contains("[openshell.gateway]")); + + let remote_error = remote_driver_config_from_context(test_context(Some(&file)), "kyma") + .expect_err("remote driver TLS field must be rejected"); + assert!(remote_error.to_string().contains(field)); + assert!(remote_error.to_string().contains("[openshell.gateway]")); + } + } + + #[test] + fn explicit_gateway_guest_tls_takes_precedence_over_package_bundle() { + let dir = tempfile::tempdir().expect("temp dir"); + let explicit = [ + dir.path().join("explicit-ca.pem"), + dir.path().join("explicit-cert.pem"), + dir.path().join("explicit-key.pem"), + ]; + for path in &explicit { + std::fs::write(path, b"explicit").expect("write explicit TLS fixture"); + } + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(explicit[0].clone()), + guest_tls_cert: Some(explicit[1].clone()), + guest_tls_key: Some(explicit[2].clone()), + ..Default::default() + }; + let package = LocalTlsPaths { + ca: PathBuf::from("/managed/ca.pem"), + server_cert: PathBuf::from("/managed/server-cert.pem"), + server_key: PathBuf::from("/managed/server-key.pem"), + client_cert: PathBuf::from("/managed/client-cert.pem"), + client_key: PathBuf::from("/managed/client-key.pem"), + }; + + let resolved = GuestTlsPaths::resolve(Some(&gateway), Some(&package), false) + .expect("explicit bundle resolves") + .expect("guest bundle"); + assert_eq!( + resolved.as_paths(), + ( + explicit[0].as_path(), + explicit[1].as_path(), + explicit[2].as_path() + ) + ); + } + + #[test] + fn gateway_guest_tls_rejects_directories_for_every_bundle_member() { + let dir = tempfile::tempdir().expect("temp dir"); + let files = [ + dir.path().join("ca.pem"), + dir.path().join("cert.pem"), + dir.path().join("key.pem"), + ]; + for path in &files { + std::fs::write(path, b"fixture").expect("write TLS fixture"); + } + + for index in 0..files.len() { + let mut paths = files.clone(); + paths[index] = dir.path().to_path_buf(); + let gateway = config_file::GatewayFileSection { + guest_tls_ca: Some(paths[0].clone()), + guest_tls_cert: Some(paths[1].clone()), + guest_tls_key: Some(paths[2].clone()), + ..Default::default() + }; + let error = GuestTlsPaths::resolve(Some(&gateway), None, false) + .expect_err("directory TLS input must be rejected"); + assert!(error.contains("not a file"), "{error}"); + } + } + #[test] fn remote_driver_config_reads_socket_path_from_named_table() { let file: config_file::ConfigFile = toml::from_str( @@ -167,11 +459,11 @@ socket_path = "/run/openshell/kyma.sock" } #[test] - fn remote_driver_config_ignores_in_process_driver_fields() { + fn remote_driver_config_reads_only_socket_path() { let file: config_file::ConfigFile = toml::from_str( r#" -[openshell.gateway] -sandbox_namespace = "sandboxes" +[openshell] +version = 2 [openshell.drivers.kubernetes] socket_path = "/run/openshell/kubernetes.sock" diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 64ac953624..189b4b5b4e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -6,9 +6,8 @@ //! See `rfc/0003-gateway-configuration/README.md` for the file format. This //! module parses the file into [`ConfigFile`], rejects fields that must be //! supplied via env/CLI (database URL), and provides -//! [`driver_table`] which overlays shared `[openshell.gateway]` defaults onto -//! a `[openshell.drivers.]` table so each driver crate's -//! `Deserialize` impl sees a fully-populated table. +//! [`driver_table`] which returns a driver-owned +//! `[openshell.drivers.]` table without gateway-level inheritance. //! //! The merge precedence for gateway process settings is: //! ```text @@ -21,7 +20,8 @@ //! values. use std::collections::BTreeMap; -use std::io::Cursor; +use std::fs::OpenOptions; +use std::io::{Cursor, Read as _}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -29,12 +29,12 @@ use base64::Engine as _; use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, TlsConfig, + GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, }; use serde::{Deserialize, Serialize}; -/// Latest schema version this build understands. -pub const SCHEMA_VERSION: u32 = 1; +/// Gateway configuration schema version supported by this build. +pub const SCHEMA_VERSION: u32 = 2; /// Root of the gateway TOML config file. /// @@ -52,8 +52,8 @@ pub struct ConfigFile { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OpenShellRoot { - /// Reserved for future schema migrations. Versions greater than - /// [`SCHEMA_VERSION`] are rejected at load time. + /// Gateway configuration schema version. Loaded files must set this to + /// [`SCHEMA_VERSION`]. #[serde(default)] pub version: Option, @@ -64,8 +64,8 @@ pub struct OpenShellRoot { pub supervisor: SupervisorFileSection, /// `[openshell.drivers.]` tables — passed verbatim to each driver - /// crate's `Deserialize` impl after the gateway-side inheritance merge. - /// Stored as raw [`toml::Value`] so each driver can evolve its schema + /// crate's `Deserialize` impl. Stored as raw [`toml::Value`] so each + /// driver can evolve its schema /// independently of this crate. #[serde(default)] pub drivers: BTreeMap, @@ -80,9 +80,8 @@ pub struct OpenShellRoot { /// /// All fields are `Option` so the loader can tell whether a key was set /// in the file (`Some`) or not (`None` — value is taken from CLI/env/default). -/// -/// The fields under "Shared driver defaults" are inherited into -/// `[openshell.drivers.]` tables per [`inheritable_keys`]. +/// Driver-specific settings belong exclusively in +/// `[openshell.drivers.]` tables. #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayFileSection { @@ -104,8 +103,9 @@ pub struct GatewayFileSection { pub log_level: Option, // ── Drivers ────────────────────────────────────────────────────────── - #[serde(default)] - pub compute_drivers: Option>, + /// Explicit compute driver selection. `None` enables auto-detection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compute_driver: Option, #[serde(default)] pub credential_drivers: Option>, #[serde(default)] @@ -115,8 +115,6 @@ pub struct GatewayFileSection { // ── Sandbox / SSH ──────────────────────────────────────────────────── #[serde(default)] - pub sandbox_namespace: Option, - #[serde(default)] pub ssh_session_ttl_secs: Option, #[serde(default)] pub grpc_rate_limit_requests: Option, @@ -135,24 +133,7 @@ pub struct GatewayFileSection { #[serde(default)] pub enable_loopback_service_http: Option, - // ── Shared driver defaults (inherited into [openshell.drivers.]) ─ - #[serde(default)] - pub default_image: Option, - #[serde(default)] - pub supervisor_image: Option, - #[serde(default)] - pub client_tls_secret_name: Option, - #[serde(default)] - pub service_account_name: Option, - #[serde(default)] - pub host_gateway_ip: Option, - #[serde(default)] - pub enable_user_namespaces: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet - /// writes for the `IssueSandboxToken` bootstrap exchange. Driver - /// clamps to `[600, 86400]`. - #[serde(default)] - pub sa_token_ttl_secs: Option, + // ── Sandbox client TLS ─────────────────────────────────────────────── #[serde(default)] pub guest_tls_ca: Option, #[serde(default)] @@ -168,7 +149,7 @@ pub struct GatewayFileSection { // ── Nested tables ──────────────────────────────────────────────────── #[serde(default)] - pub tls: Option, + pub tls: Option, #[serde(default)] pub oidc: Option, #[serde(default)] @@ -193,6 +174,27 @@ pub struct GatewayFileSection { pub database_url: Option, } +/// Gateway listener TLS fields accepted in `gateway.toml`. +/// +/// Client-certificate handshake policy is derived from the presence of a +/// client CA and OIDC configuration. It is intentionally not operator-settable +/// in TOML because changing it can either weaken CA-only gateways or require a +/// second credential from OIDC clients. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GatewayTlsFileConfig { + pub cert_path: PathBuf, + pub key_path: PathBuf, + #[serde(default)] + pub client_ca_path: Option, + #[serde(default)] + pub external_cert_path: Option, + #[serde(default)] + pub external_key_path: Option, + #[serde(default)] + pub external_server_names: Vec, +} + /// `[openshell.gateway.otlp]` section. /// /// Presence of this table enables OTLP export; there is no `enabled` flag. @@ -328,6 +330,8 @@ pub enum ConfigFileError { #[source] source: std::io::Error, }, + #[error("gateway config path '{}' is not a regular file ({kind})", path.display())] + NonRegular { path: PathBuf, kind: &'static str }, #[error("failed to parse gateway config file '{}': {source}", path.display())] Parse { path: PathBuf, @@ -335,7 +339,11 @@ pub enum ConfigFileError { source: toml::de::Error, }, #[error( - "unsupported gateway config version {version}; this build only supports version {SCHEMA_VERSION}" + "gateway config schema version is required; add `[openshell]` and `version = {SCHEMA_VERSION}`" + )] + MissingVersion, + #[error( + "unsupported gateway config version {version}; this build requires version {SCHEMA_VERSION}; migrate legacy fields to the version {SCHEMA_VERSION} schema" )] UnsupportedVersion { version: u32 }, #[error( @@ -351,6 +359,8 @@ pub enum ConfigFileError { field: &'static str, message: &'static str, }, + #[error("invalid gateway config field `openshell.drivers.{name}`: expected a TOML table")] + InvalidDriverTable { name: String }, #[error( "failed to read TLS CA certificate for supervisor middleware '{name}' from '{}': {source}", path.display() @@ -372,28 +382,111 @@ pub enum ConfigFileError { }, } -/// Load and validate a TOML config file. -/// -/// Returns `Ok(ConfigFile::default())` for an empty file (the gateway then -/// falls back entirely to CLI/env/built-in defaults). -#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] -pub fn load(path: &Path) -> Result { - let contents = std::fs::read_to_string(path).map_err(|source| ConfigFileError::Io { - path: path.to_path_buf(), +const CONFIG_MIGRATION_URL: &str = + "https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2"; + +/// Stable package-preflight failure with no configuration contents attached. +#[derive(Debug, thiserror::Error)] +#[error( + "gateway config preflight failed: path='{}' category={} detected_version={}; the file was preserved unchanged; migrate it before restarting the gateway: {CONFIG_MIGRATION_URL}", + path.display(), + category, + detected_version +)] +pub struct ConfigPreflightError { + path: PathBuf, + category: &'static str, + detected_version: String, +} + +impl ConfigPreflightError { + pub(crate) fn invalid_current(path: &Path) -> Self { + Self { + path: path.to_path_buf(), + category: "malformed", + detected_version: SCHEMA_VERSION.to_string(), + } + } + + #[cfg(test)] + fn category(&self) -> &'static str { + self.category + } + + #[cfg(test)] + fn detected_version(&self) -> &str { + &self.detected_version + } +} + +fn non_regular_kind(metadata: &std::fs::Metadata) -> &'static str { + let file_type = metadata.file_type(); + if file_type.is_symlink() { + "symlink" + } else if file_type.is_dir() { + "directory" + } else { + "non-regular" + } +} + +fn read_config_contents(path: &Path) -> Result { + let path_buf = path.to_path_buf(); + let metadata = std::fs::symlink_metadata(path).map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), + source, + })?; + if !metadata.file_type().is_file() { + return Err(ConfigFileError::NonRegular { + path: path_buf, + kind: non_regular_kind(&metadata), + }); + } + + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let mut file = options.open(path).map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), + source, + })?; + let opened_metadata = file.metadata().map_err(|source| ConfigFileError::Io { + path: path_buf.clone(), source, })?; + if !opened_metadata.file_type().is_file() { + return Err(ConfigFileError::NonRegular { + path: path_buf, + kind: non_regular_kind(&opened_metadata), + }); + } + + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|source| ConfigFileError::Io { + path: path.to_path_buf(), + source, + })?; + Ok(contents) +} + +fn parse_and_validate(path: &Path, contents: &str) -> Result { if contents.trim().is_empty() { - return Ok(ConfigFile::default()); + return Err(ConfigFileError::MissingVersion); } - let file: ConfigFile = toml::from_str(&contents).map_err(|source| ConfigFileError::Parse { + let file: ConfigFile = toml::from_str(contents).map_err(|source| ConfigFileError::Parse { path: path.to_path_buf(), source, })?; - if let Some(version) = file.openshell.version - && version > SCHEMA_VERSION - { - return Err(ConfigFileError::UnsupportedVersion { version }); + match file.openshell.version { + Some(SCHEMA_VERSION) => {} + Some(version) => return Err(ConfigFileError::UnsupportedVersion { version }), + None => return Err(ConfigFileError::MissingVersion), } if file.openshell.gateway.database_url.is_some() { @@ -415,72 +508,130 @@ pub fn load(path: &Path) -> Result { message: "omit the field to use default encrypted gateway credential storage, or specify exactly one external credential driver", }); } + if let Some((name, _)) = file + .openshell + .drivers + .iter() + .find(|(_, value)| !value.is_table()) + { + return Err(ConfigFileError::InvalidDriverTable { name: name.clone() }); + } Ok(file) } -/// Build the merged TOML table for `driver` by overlaying inheritable -/// `[openshell.gateway]` defaults onto `[openshell.drivers.]`. -/// -/// The returned [`toml::Value`] is a Table ready to feed into the driver's -/// `Deserialize` impl — keys present in `raw` win over the gateway defaults. -/// Keys outside [`inheritable_keys`] for this driver are never copied from -/// the gateway section, which keeps each driver's `deny_unknown_fields` -/// invariant intact. -pub fn driver_table( - driver_name: &str, - gateway: &GatewayFileSection, - raw: Option<&toml::Value>, -) -> toml::Value { - driver_table_with_inherited_keys(driver_name, gateway, raw, &[]) -} - -pub(crate) fn driver_table_with_inherited_keys( - _driver_name: &str, - gateway: &GatewayFileSection, - raw: Option<&toml::Value>, - inheritable_keys: &[&str], -) -> toml::Value { - let mut merged = match raw { - Some(toml::Value::Table(table)) => table.clone(), - _ => toml::Table::new(), +fn detected_version(contents: &str) -> String { + let Ok(value) = contents.parse::() else { + return "unavailable".to_string(); + }; + let Some(openshell) = value.get("openshell") else { + return "missing".to_string(); }; + let Some(openshell) = openshell.as_table() else { + return "unavailable".to_string(); + }; + openshell.get("version").map_or_else( + || "missing".to_string(), + |version| { + version + .as_integer() + .map_or_else(|| "unavailable".to_string(), |version| version.to_string()) + }, + ) +} - for key in inheritable_keys { - if merged.contains_key(*key) { - continue; +fn preflight_error( + path: &Path, + detected_version: String, + source: ConfigFileError, +) -> ConfigPreflightError { + let category = match &source { + ConfigFileError::NonRegular { .. } => "non_regular_path", + ConfigFileError::Io { source, .. } if source.kind() == std::io::ErrorKind::NotFound => { + "missing_path" } - if let Some(value) = gateway_inherited_value(gateway, key) { - merged.insert((*key).to_string(), value); + ConfigFileError::Io { source, .. } if source.kind() == std::io::ErrorKind::InvalidData => { + "malformed" } + ConfigFileError::Io { .. } => "unreadable_path", + ConfigFileError::MissingVersion => "missing_version", + ConfigFileError::UnsupportedVersion { version: 1 } => "legacy_schema_v1", + ConfigFileError::UnsupportedVersion { version } if *version > SCHEMA_VERSION => { + "future_version" + } + ConfigFileError::UnsupportedVersion { .. } => "unsupported_version", + ConfigFileError::Parse { .. } + | ConfigFileError::SecretInFile { .. } + | ConfigFileError::InvalidValue { .. } + | ConfigFileError::InvalidDriverTable { .. } + | ConfigFileError::MiddlewareTlsCaRead { .. } + | ConfigFileError::MiddlewareTlsCaInvalid { .. } => "malformed", + }; + ConfigPreflightError { + path: path.to_path_buf(), + category, + detected_version, } - - toml::Value::Table(merged) } -fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { - match key { - "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), - "default_image" => g.default_image.as_deref().map(string_value), - "supervisor_image" => g.supervisor_image.as_deref().map(string_value), - "client_tls_secret_name" => g.client_tls_secret_name.as_deref().map(string_value), - "service_account_name" => g.service_account_name.as_deref().map(string_value), - "host_gateway_ip" => g.host_gateway_ip.as_deref().map(string_value), - "enable_user_namespaces" => g.enable_user_namespaces.map(toml::Value::Boolean), - "sa_token_ttl_secs" => g.sa_token_ttl_secs.map(toml::Value::Integer), - "guest_tls_ca" => g.guest_tls_ca.as_deref().map(path_value), - "guest_tls_cert" => g.guest_tls_cert.as_deref().map(path_value), - "guest_tls_key" => g.guest_tls_key.as_deref().map(path_value), - _ => None, +/// Validate a gateway configuration without starting the gateway or writing state. +#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] +pub fn preflight(path: &Path) -> Result { + let contents = read_config_contents(path) + .map_err(|source| preflight_error(path, "unavailable".to_string(), source))?; + let version = detected_version(&contents); + if version == "missing" { + return Err(preflight_error( + path, + version, + ConfigFileError::MissingVersion, + )); } + if let Ok(version_number) = version.parse::() + && version_number != SCHEMA_VERSION + { + return Err(preflight_error( + path, + version, + ConfigFileError::UnsupportedVersion { + version: version_number, + }, + )); + } + parse_and_validate(path, &contents).map_err(|source| preflight_error(path, version, source)) } -fn string_value(s: &str) -> toml::Value { - toml::Value::String(s.to_owned()) +/// Load and validate a TOML config file. +/// +/// Configuration files must declare exactly [`SCHEMA_VERSION`]. Running +/// without a config file still uses CLI, environment, and built-in defaults. +#[cfg_attr(target_os = "windows", allow(clippy::result_large_err))] +pub fn load(path: &Path) -> Result { + let contents = read_config_contents(path)?; + parse_and_validate(path, &contents) } -fn path_value(p: &Path) -> toml::Value { - toml::Value::String(p.display().to_string()) +/// Return a driver's table without gateway-level inheritance. +/// Driver-specific configuration belongs exclusively to +/// `[openshell.drivers.]` in schema version 2. +pub fn driver_table( + _driver_name: &str, + _gateway: &GatewayFileSection, + raw: Option<&toml::Value>, +) -> toml::Value { + match raw { + Some(toml::Value::Table(table)) => toml::Value::Table(table.clone()), + _ => toml::Value::Table(toml::Table::new()), + } +} + +pub(crate) fn driver_table_with_inherited_keys( + driver_name: &str, + gateway: &GatewayFileSection, + raw: Option<&toml::Value>, + _inherited_config_keys: &[&str], +) -> toml::Value { + driver_table(driver_name, gateway, raw) } #[cfg(test)] @@ -488,7 +639,7 @@ mod tests { use super::*; use std::io::Write; - fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + fn write_raw_tmp(contents: &str) -> tempfile::NamedTempFile { let mut tmp = tempfile::Builder::new() .suffix(".toml") .tempfile() @@ -497,35 +648,103 @@ mod tests { tmp } + fn write_tmp(contents: &str) -> tempfile::NamedTempFile { + if contents.contains("[openshell]") { + write_raw_tmp(contents) + } else { + write_raw_tmp(&format!("[openshell]\nversion = 2\n\n{contents}")) + } + } + + #[test] + fn empty_file_requires_schema_version() { + let tmp = write_raw_tmp(""); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); + } + + #[test] + fn compute_driver_entries_must_be_tables() { + for value in ["\"not-a-table\"", "[\"also\", \"not-a-table\"]", "42"] { + let tmp = write_raw_tmp(&format!( + "[openshell]\nversion = 2\n\n[openshell.drivers]\ndocker = {value}\n" + )); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::InvalidDriverTable { ref name }) if name == "docker" + )); + } + } + + #[test] + fn canonical_compute_driver_is_singular() { + let file: ConfigFile = toml::from_str( + r#" +[openshell.gateway] +compute_driver = "docker" +"#, + ) + .expect("canonical compute driver parses"); + + assert_eq!( + file.openshell.gateway.compute_driver.as_deref(), + Some("docker") + ); + } + + #[test] + fn legacy_compute_drivers_list_is_rejected() { + let error = + toml::from_str::("[openshell.gateway]\ncompute_drivers = [\"docker\"]\n") + .expect_err("legacy compute_drivers must be rejected"); + assert!(error.to_string().contains("compute_drivers")); + } + + #[test] + fn compute_driver_rejects_non_string_values() { + let error = toml::from_str::( + r" +[openshell.gateway] +compute_driver = 42 +", + ) + .expect_err("compute driver must be a string"); + assert!(error.to_string().contains("invalid type")); + } + #[test] - fn empty_file_yields_default_config() { - let tmp = write_tmp(""); - let file = load(tmp.path()).expect("empty file parses"); - assert!(file.openshell.version.is_none()); - assert!(file.openshell.gateway.bind_address.is_none()); - assert!(file.openshell.drivers.is_empty()); + fn compute_driver_serialization_uses_scalar_name() { + let file = ConfigFile { + openshell: OpenShellRoot { + gateway: GatewayFileSection { + compute_driver: Some("docker".to_string()), + ..Default::default() + }, + ..Default::default() + }, + }; + + let serialized = toml::to_string(&file).expect("config serializes"); + assert!(serialized.contains("compute_driver = \"docker\"")); } #[test] fn parses_full_example() { let toml = r#" [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 policy_validation_failure_mode = "retain_last_valid" -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -client_tls_secret_name = "openshell-sandbox-tls" -service_account_name = "openshell-sandbox" [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" @@ -538,6 +757,10 @@ audience = "openshell-cli" [openshell.drivers.kubernetes] namespace = "agents" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +client_tls_secret_name = "openshell-sandbox-tls" +service_account_name = "openshell-sandbox" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" [openshell.credential_drivers.kubernetes-secrets] @@ -547,10 +770,6 @@ namespace = "agents" let file = load(tmp.path()).expect("valid file parses"); let gw = &file.openshell.gateway; assert_eq!(gw.log_level.as_deref(), Some("info")); - assert_eq!( - gw.default_image.as_deref(), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); assert_eq!( @@ -564,7 +783,18 @@ namespace = "agents" Some(&["kubernetes-secrets".to_string()][..]) ); assert!(gw.default_credential_driver.is_none()); - assert!(file.openshell.drivers.contains_key("kubernetes")); + let kubernetes = file + .openshell + .drivers + .get("kubernetes") + .and_then(toml::Value::as_table) + .expect("kubernetes driver table"); + assert_eq!( + kubernetes + .get("default_image") + .and_then(toml::Value::as_str), + Some("ghcr.io/nvidia/openshell/sandbox:latest") + ); assert!( file.openshell .credential_drivers @@ -895,6 +1125,42 @@ nonsense = true assert!(matches!(err, ConfigFileError::Parse { .. })); } + #[test] + fn rejects_operator_selected_tls_client_auth_policy() { + let toml = r#" +[openshell.gateway.tls] +cert_path = "/tls/server.crt" +key_path = "/tls/server.key" +client_ca_path = "/tls/ca.crt" +require_client_auth = false +"#; + let tmp = write_tmp(toml); + let error = load(tmp.path()).expect_err("derived TLS client-auth policy must be rejected"); + assert!(matches!(error, ConfigFileError::Parse { .. })); + assert!(error.to_string().contains("require_client_auth")); + } + + #[test] + fn rejects_removed_driver_fields_at_gateway_scope() { + for field in [ + "sandbox_namespace = \"agents\"", + "default_image = \"sandbox:latest\"", + "supervisor_image = \"supervisor:latest\"", + "client_tls_secret_name = \"sandbox-tls\"", + "service_account_name = \"sandbox-sa\"", + "host_gateway_ip = \"10.0.0.1\"", + "enable_user_namespaces = true", + "sa_token_ttl_secs = 3600", + ] { + let tmp = write_tmp(&format!("[openshell.gateway]\n{field}\n")); + let err = load(tmp.path()).expect_err("gateway-scoped driver field must be rejected"); + assert!( + matches!(err, ConfigFileError::Parse { .. }), + "field: {field}" + ); + } + } + #[test] fn rejects_unknown_field_in_nested_gateway_jwt_table() { // Regression guard for the class of silent-misconfig bug fixed in @@ -927,170 +1193,158 @@ ssh_gateway_port = 8080 } #[test] - fn rejects_unsupported_version() { - let toml = r" -[openshell] -version = 2 -"; - let tmp = write_tmp(toml); - let err = load(tmp.path()).expect_err("version > 1 must be rejected"); + fn rejects_legacy_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 1\n"); + let err = load(tmp.path()).expect_err("version 1 must be rejected"); assert!(matches!( err, - ConfigFileError::UnsupportedVersion { version: 2 } + ConfigFileError::UnsupportedVersion { version: 1 } )); } #[test] - fn driver_table_inherits_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), - ), - supervisor_image: Some("ghcr.io/nvidia/openshell/supervisor:0.9".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - namespace = "agents" - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - Some(&toml::Value::Table(raw)), - &["default_image", "supervisor_image"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("supervisor_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell/supervisor:0.9") - ); + fn rejects_missing_version_in_nonempty_file() { + let tmp = write_raw_tmp("[openshell]\n\n[openshell.gateway]\nname = \"test\"\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::MissingVersion) + )); } #[test] - fn registered_driver_table_inherits_selected_gateway_defaults() { - let gateway = GatewayFileSection { - sandbox_namespace: Some("agents".to_string()), - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), - ), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "alpha", - &gateway, - None, - &["sandbox_namespace", "default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("sandbox_namespace").and_then(|v| v.as_str()), - Some("agents") - ); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("10.0.0.1") - ); + fn rejects_future_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 3\n"); + assert!(matches!( + load(tmp.path()), + Err(ConfigFileError::UnsupportedVersion { version: 3 }) + )); } #[test] - fn registered_driver_table_can_select_network_defaults() { - let gateway = GatewayFileSection { - default_image: Some( - "ghcr.io/nvidia/openshell-community/sandboxes/base:latest".to_string(), + fn accepts_current_version() { + let tmp = write_raw_tmp("[openshell]\nversion = 2\n"); + load(tmp.path()).expect("schema version 2 must be accepted"); + preflight(tmp.path()).expect("schema version 2 must pass preflight"); + } + + #[test] + fn preflight_classifies_versions_and_malformed_files_without_disclosing_contents() { + for (contents, category, version) in [ + ("[openshell]\nversion = 1\n", "legacy_schema_v1", "1"), + ("[openshell]\n", "missing_version", "missing"), + ("[openshell]\nversion = 3\n", "future_version", "3"), + ("[openshell]\nversion = 0\n", "unsupported_version", "0"), + ("[openshell]\nversion = 'two'\n", "malformed", "unavailable"), + ( + "[openshell]\nversion = 2\nsecret-value = [", + "malformed", + "unavailable", ), - host_gateway_ip: Some("192.168.127.254".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys( - "beta", - &gateway, - None, - &["default_image", "host_gateway_ip"], - ); - let table = merged.as_table().expect("table"); - assert_eq!( - table.get("default_image").and_then(|v| v.as_str()), - Some("ghcr.io/nvidia/openshell-community/sandboxes/base:latest") - ); - assert_eq!( - table.get("host_gateway_ip").and_then(|v| v.as_str()), - Some("192.168.127.254") - ); + ( + "[openshell]\nversion = 2\n[openshell.gateway]\nunknown = 'secret-value'\n", + "malformed", + "2", + ), + ] { + let tmp = write_raw_tmp(contents); + let error = preflight(tmp.path()).expect_err("preflight must reject fixture"); + assert_eq!(error.category(), category, "contents: {contents}"); + assert_eq!(error.detected_version(), version, "contents: {contents}"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains(&format!("category={category}"))); + assert!(diagnostic.contains(&format!("detected_version={version}"))); + assert!(diagnostic.contains("the file was preserved unchanged")); + assert!(diagnostic.contains(CONFIG_MIGRATION_URL)); + assert!(!diagnostic.contains("secret-value")); + assert!(!format!("{error:?}").contains("secret-value")); + assert!(std::error::Error::source(&error).is_none()); + } } #[test] - fn driver_table_specific_value_overrides_gateway_default() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default".to_string()), - ..Default::default() - }; + fn preflight_classifies_missing_path() { + let path = Path::new("/nonexistent/openshell-preflight.toml"); + let error = preflight(path).expect_err("missing explicit path must fail"); + assert_eq!(error.category(), "missing_path"); + assert_eq!(error.detected_version(), "unavailable"); + assert!(error.to_string().contains(&path.display().to_string())); + } + + #[test] + fn preflight_rejects_directory_as_non_regular() { + let tmp = tempfile::tempdir().unwrap(); + let error = preflight(tmp.path()).expect_err("directory must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + assert_eq!(error.detected_version(), "unavailable"); + } + + #[cfg(unix)] + #[test] + fn preflight_and_runtime_reject_symlinks_without_following_them() { + use std::os::unix::fs::symlink; + + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("target.toml"); + let link = dir.path().join("gateway.toml"); + std::fs::write(&target, "[openshell]\nversion = 2\n").unwrap(); + symlink(&target, &link).unwrap(); + + let error = preflight(&link).expect_err("symlink must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + assert!(matches!( + load(&link), + Err(ConfigFileError::NonRegular { .. }) + )); + } + + #[cfg(unix)] + #[test] + fn preflight_rejects_fifo_without_blocking() { + use nix::sys::stat::Mode; + use nix::unistd::mkfifo; + + let dir = tempfile::tempdir().unwrap(); + let fifo = dir.path().join("gateway.toml"); + mkfifo(&fifo, Mode::S_IRUSR | Mode::S_IWUSR).unwrap(); + let error = preflight(&fifo).expect_err("FIFO must fail preflight"); + assert_eq!(error.category(), "non_regular_path"); + } + + #[test] + fn preflight_classifies_non_utf8_as_malformed() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(tmp.path(), [0xff, 0xfe]).unwrap(); + let error = preflight(tmp.path()).expect_err("non-UTF-8 config must fail"); + assert_eq!(error.category(), "malformed"); + assert_eq!(error.detected_version(), "unavailable"); + } + + #[test] + fn driver_table_uses_only_driver_owned_values() { let raw = toml::toml! { default_image = "driver-specific" + socket_path = "/run/openshell/driver.sock" }; - let merged = driver_table_with_inherited_keys( + let table = driver_table( "alpha", - &gateway, + &GatewayFileSection::default(), Some(&toml::Value::Table(raw)), - &["default_image"], ); + let table = table.as_table().expect("driver table"); assert_eq!( - merged - .as_table() - .unwrap() - .get("default_image") - .and_then(|v| v.as_str()), + table.get("default_image").and_then(toml::Value::as_str), Some("driver-specific") ); - } - - #[test] - fn driver_table_does_not_leak_keys_outside_allowlist() { - // Fields not selected by the registration must remain gateway-only. - let gateway = GatewayFileSection { - client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), - ..Default::default() - }; - let merged = driver_table_with_inherited_keys("alpha", &gateway, None, &["default_image"]); - assert!( - !merged - .as_table() - .unwrap() - .contains_key("client_tls_secret_name") + assert_eq!( + table.get("socket_path").and_then(toml::Value::as_str), + Some("/run/openshell/driver.sock") ); } #[test] - fn remote_driver_table_does_not_inherit_gateway_defaults() { - let gateway = GatewayFileSection { - default_image: Some("gateway-default:1.0".to_string()), - host_gateway_ip: Some("10.0.0.1".to_string()), - ..Default::default() - }; - let raw = toml::toml! { - socket_path = "/run/openshell/kyma.sock" - }; - - let merged = driver_table("kyma", &gateway, Some(&toml::Value::Table(raw))); - let table = merged.as_table().expect("table"); - - assert_eq!( - table.get("socket_path").and_then(|v| v.as_str()), - Some("/run/openshell/kyma.sock") - ); - assert!(!table.contains_key("default_image")); - assert!(!table.contains_key("host_gateway_ip")); + fn driver_table_does_not_inject_gateway_values() { + let table = driver_table("alpha", &GatewayFileSection::default(), None); + assert!(table.as_table().expect("driver table").is_empty()); } #[test] @@ -1108,7 +1362,7 @@ version = 2 /// - template corruption or unknown fields (`deny_unknown_fields`) /// - schema drift (version bump or field renames) /// - accidental addition of a wildcard bind-address override - /// - accidental changes to the compute driver list + /// - accidental changes to the configured compute driver #[test] fn rpm_default_config_parses_and_has_podman_defaults() { let path = @@ -1124,15 +1378,24 @@ version = 2 ); } - let drivers = gw - .compute_drivers - .as_ref() - .expect("compute_drivers must be explicitly set in the RPM default config"); assert_eq!( - drivers, - &["podman".to_string()], - "RPM default must pin compute_drivers to [podman] to prevent unexpected \ + gw.compute_driver.as_deref(), + Some("podman"), + "RPM default must pin compute_driver to podman to prevent unexpected \ driver selection when Docker is also installed" ); + + let podman = driver_table( + "podman", + &config.openshell.gateway, + config.openshell.drivers.get("podman"), + ); + assert_eq!( + podman + .get("health_check_interval_secs") + .and_then(toml::Value::as_integer), + Some(10), + "RPM defaults must retain Podman's readiness health check" + ); } } diff --git a/crates/openshell-server/src/defaults.rs b/crates/openshell-server/src/defaults.rs index b5a5a5e924..21e66b02bd 100644 --- a/crates/openshell-server/src/defaults.rs +++ b/crates/openshell-server/src/defaults.rs @@ -104,7 +104,7 @@ pub fn complete_local_jwt_config() -> Result> { public_key_path: paths.public_key, kid_path: paths.kid, gateway_id: "openshell".to_string(), - ttl_secs: 0, + ttl_secs: None, })), _ => Err(miette::miette!( "partial local sandbox JWT state in {}: expected jwt/signing.pem, jwt/public.pem, and jwt/kid", @@ -237,6 +237,6 @@ mod tests { assert_eq!(config.public_key_path, tmp.path().join("jwt/public.pem")); assert_eq!(config.kid_path, tmp.path().join("jwt/kid")); assert_eq!(config.gateway_id, "openshell"); - assert_eq!(config.ttl_secs, 0); + assert_eq!(config.ttl_secs, None); } } diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index 104d639584..35325f3bbf 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -226,11 +226,11 @@ fn mint_extension_credentials( .iter() .map(|service| (service.name.as_str(), service)) .collect(); - let ttl = if issuer.ttl().is_zero() { - DEFAULT_EXTENSION_TOKEN_TTL - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - }; + let ttl = issuer + .sandbox_token_ttl() + .map_or(DEFAULT_EXTENSION_TOKEN_TTL, |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }); requested_names .iter() @@ -323,7 +323,7 @@ mod tests { mat.signing_key_pem.as_bytes(), mat.kid, "test-gateway", - Duration::from_secs(3600), + Some(Duration::from_secs(3600)), ) .unwrap(); state.sandbox_jwt_issuer = Some(Arc::new(issuer)); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8c8afdf08..5f09dd7631 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -97,11 +97,11 @@ struct GatewayExtensionCredential { } fn extension_token_ttl(issuer: &auth::sandbox_jwt::SandboxJwtIssuer) -> Duration { - if issuer.ttl().is_zero() { - Duration::from_secs(15 * 60) - } else { - issuer.ttl().min(MAX_EXTENSION_TOKEN_TTL) - } + issuer + .sandbox_token_ttl() + .map_or(Duration::from_secs(15 * 60), |ttl| { + ttl.min(MAX_EXTENSION_TOKEN_TTL) + }) } /// Mint the gateway-caller credential for one extension registration. @@ -249,6 +249,7 @@ pub(crate) struct ServerStartupConfig { pub config_file: Option, pub guest_tls: Option, pub compute_driver: ComputeDriverSelection, + pub legacy_compute_driver_env_seen: bool, } /// Server state shared across handlers. @@ -449,6 +450,7 @@ pub(crate) async fn run_server( config_file, guest_tls, compute_driver, + legacy_compute_driver_env_seen: _, } = startup; let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -496,7 +498,7 @@ pub(crate) async fn run_server( &signing_pem, kid.clone(), &jwt.gateway_id, - Duration::from_secs(jwt.ttl_secs), + jwt.sandbox_token_ttl(), ) .map_err(Error::config)?, ); @@ -506,7 +508,7 @@ pub(crate) async fn run_server( ); info!( gateway_id = %jwt.gateway_id, - ttl_secs = jwt.ttl_secs, + ttl_secs = jwt.ttl_secs.map(std::num::NonZeroU64::get), "gateway-minted sandbox JWT enabled" ); (Some(issuer), Some(authenticator)) @@ -1058,6 +1060,10 @@ pub enum ComputeDriverInstance { /// Factory for a compute driver linked into a gateway binary. #[async_trait::async_trait] pub trait ComputeDriverFactory: Send + Sync { + /// Validate selected-driver configuration without starting a driver, + /// connecting a transport, or modifying runtime state. + fn validate_config(&self, context: ComputeDriverConfigContext<'_>) -> Result<()>; + async fn build(&self, context: ComputeDriverBuildContext<'_>) -> Result; } @@ -1226,6 +1232,18 @@ impl ComputeDriverRegistry { self.drivers.keys().map(String::as_str) } + /// Names whose runtime registrations participate in auto-detection. + /// + /// This does not execute detection probes. Package preflight uses it to + /// validate configured candidate tables without connecting local sockets + /// or starting discovery commands. + pub(crate) fn auto_detectable_driver_names(&self) -> impl Iterator { + self.drivers + .values() + .filter(|registration| registration.detect.is_some()) + .map(|registration| registration.name.as_str()) + } + pub(crate) fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { self.drivers.get(name) } @@ -1262,45 +1280,46 @@ impl ComputeDriverRegistry { ComputeDriverDetection { available } } - pub(crate) fn select(&self, configured_drivers: &[String]) -> Result { - match configured_drivers { - [] => { + pub(crate) fn select(&self, configured_driver: Option<&str>) -> Result { + match configured_driver { + None => { let detection = self.detect(); if detection.selected().is_none() { return Err(Error::config( "no compute driver configured and auto-detection found no suitable installed \ - driver; set --drivers or OPENSHELL_DRIVERS=", + driver; set --compute-driver or OPENSHELL_COMPUTE_DRIVER=", )); } Ok(ComputeDriverSelection::AutoDetected(detection)) } - [driver] => { + Some(driver) => { let name = openshell_core::config::normalize_compute_driver_name(driver) .map_err(Error::config)?; Ok(ComputeDriverSelection::Configured { name }) } - drivers => Err(Error::config(format!( - "multiple compute drivers are not supported yet; configured drivers: {}", - drivers.join(",") - ))), } } } -pub struct ComputeDriverBuildContext<'a> { - driver_name: String, +/// Read-only inputs available while validating a selected compute driver. +/// +/// This context deliberately exposes no shutdown handle, runtime store, or +/// transport client. Implementations must remain deterministic and must not +/// start processes, connect sockets, or modify state. +#[derive(Clone, Copy)] +pub struct ComputeDriverConfigContext<'a> { + driver_name: &'a str, gateway_name: &'a str, gateway_bind_address: SocketAddr, gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, - shutdown_rx: watch::Receiver, inherited_config_keys: &'static [&'static str], } -impl ComputeDriverBuildContext<'_> { +impl ComputeDriverConfigContext<'_> { #[must_use] pub fn driver_name(&self) -> &str { - &self.driver_name + self.driver_name } #[must_use] @@ -1328,10 +1347,65 @@ impl ComputeDriverBuildContext<'_> { self.driver_startup.gateway_tls_enabled } + /// Deserialize the selected driver's merged TOML table. + pub fn driver_config(&self) -> Result + where + T: Default + serde::de::DeserializeOwned, + { + compute::driver_config::driver_config_from_context( + self.driver_startup, + self.driver_name, + self.inherited_config_keys, + ) + } +} + +pub struct ComputeDriverBuildContext<'a> { + config: ComputeDriverConfigContext<'a>, + shutdown_rx: watch::Receiver, +} + +impl ComputeDriverBuildContext<'_> { + #[must_use] + pub fn config_context(&self) -> ComputeDriverConfigContext<'_> { + self.config + } + + #[must_use] + pub fn driver_name(&self) -> &str { + self.config.driver_name() + } + + #[must_use] + pub fn gateway_name(&self) -> &str { + self.config.gateway_name() + } + + #[must_use] + pub fn gateway_bind_address(&self) -> SocketAddr { + self.config.gateway_bind_address() + } + + #[must_use] + pub fn gateway_log_level(&self) -> &str { + self.config.gateway_log_level() + } + + #[must_use] + pub fn gateway_port(&self) -> u16 { + self.config.gateway_port() + } + + #[must_use] + pub fn gateway_tls_enabled(&self) -> bool { + self.config.gateway_tls_enabled() + } + /// Gateway client credentials that a local driver may mount into guests. #[must_use] pub fn guest_tls_paths(&self) -> Option<(&Path, &Path, &Path)> { - self.driver_startup + self.config + .driver_startup .guest_tls .map(compute::driver_config::GuestTlsPaths::as_paths) } @@ -1341,11 +1415,7 @@ impl ComputeDriverBuildContext<'_> { where T: Default + serde::de::DeserializeOwned, { - compute::driver_config::driver_config_from_context( - self.driver_startup, - &self.driver_name, - self.inherited_config_keys, - ) + self.config.driver_config() } #[must_use] @@ -1355,7 +1425,8 @@ impl ComputeDriverBuildContext<'_> { #[must_use] pub fn otlp_config(&self) -> Option<&config_file::OtlpConfig> { - self.driver_startup + self.config + .driver_startup .file .and_then(|file| file.openshell.gateway.otlp.as_ref()) } @@ -1374,30 +1445,39 @@ async fn build_compute_runtime( supervisor_sessions: Arc, shutdown_rx: watch::Receiver, ) -> Result { - let driver = resolve_configured_compute_driver(registry, selection.name(), driver_startup)?; + let driver = validate_compute_driver_config( + registry, + selection.name(), + &config.name, + config.bind_address, + &config.log_level, + driver_startup, + )?; let telemetry_compute_driver = driver.telemetry_compute_driver(registry); info!(driver = %driver.name(), "Using compute driver"); if config .gateway_jwt .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) + .is_some_and(|jwt| jwt.sandbox_token_ttl().is_none()) && !driver.is_local_singleplayer(registry) { warn!( - "Gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" + "Gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs is omitted or zero); set gateway_jwt.ttl_secs > 0 for shared deployments" ); } let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { let build_context = ComputeDriverBuildContext { - driver_name: registration.name.clone(), - gateway_name: &config.name, - gateway_bind_address: config.bind_address, - gateway_log_level: &config.log_level, - driver_startup, + config: ComputeDriverConfigContext { + driver_name: ®istration.name, + gateway_name: &config.name, + gateway_bind_address: config.bind_address, + gateway_log_level: &config.log_level, + driver_startup, + inherited_config_keys: registration.inherited_config_keys, + }, shutdown_rx, - inherited_config_keys: registration.inherited_config_keys, }; let instance = registration.factory.build(build_context).await?; match instance { @@ -1500,10 +1580,40 @@ fn configured_compute_driver( config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { - let selection = registry.select(&config.compute_drivers)?; + let selection = registry.select(config.compute_driver.as_deref())?; resolve_configured_compute_driver(registry, selection.name(), driver_startup) } +#[allow(clippy::too_many_arguments)] +fn validate_compute_driver_config( + registry: &ComputeDriverRegistry, + driver_name: &str, + gateway_name: &str, + gateway_bind_address: SocketAddr, + gateway_log_level: &str, + driver_startup: compute::driver_config::DriverStartupContext<'_>, +) -> Result { + let driver = resolve_configured_compute_driver(registry, driver_name, driver_startup)?; + match &driver { + ConfiguredComputeDriver::Registered(registration) => { + registration + .factory + .validate_config(ComputeDriverConfigContext { + driver_name: ®istration.name, + gateway_name, + gateway_bind_address, + gateway_log_level, + driver_startup, + inherited_config_keys: registration.inherited_config_keys, + })?; + } + ConfiguredComputeDriver::Remote { name } => { + compute::driver_config::remote_driver_config_from_context(driver_startup, name)?; + } + } + Ok(driver) +} + fn resolve_configured_compute_driver( registry: &ComputeDriverRegistry, driver_name: &str, @@ -1590,7 +1700,7 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - configured_compute_driver, is_benign_tls_handshake_failure, + configured_compute_driver, extension_token_ttl, is_benign_tls_handshake_failure, mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ @@ -1636,18 +1746,39 @@ mod tests { } fn extension_test_issuer() -> Arc { + extension_test_issuer_with_ttl(Some(Duration::from_secs(900))) + } + + fn extension_test_issuer_with_ttl( + ttl: Option, + ) -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( crate::auth::sandbox_jwt::SandboxJwtIssuer::from_pem( material.signing_key_pem.as_bytes(), material.kid, "gateway-a", - Duration::from_secs(900), + ttl, ) .expect("issuer"), ) } + #[test] + fn non_expiring_sandbox_tokens_use_finite_extension_ttl() { + let issuer = extension_test_issuer_with_ttl(None); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(15 * 60)); + } + + #[test] + fn extension_token_ttl_is_capped_at_one_hour() { + let issuer = extension_test_issuer_with_ttl(Some(Duration::from_secs(24 * 60 * 60))); + assert_eq!(extension_token_ttl(&issuer), Duration::from_secs(60 * 60)); + + let short = extension_test_issuer_with_ttl(Some(Duration::from_secs(5 * 60))); + assert_eq!(extension_token_ttl(&short), Duration::from_secs(5 * 60)); + } + #[test] fn plaintext_extension_endpoint_is_rejected_unless_explicitly_opted_out() { let issuer = extension_test_issuer(); @@ -1764,6 +1895,13 @@ mod tests { #[async_trait::async_trait] impl super::ComputeDriverFactory for TestComputeDriverFactory { + fn validate_config( + &self, + _context: super::ComputeDriverConfigContext<'_>, + ) -> openshell_core::Result<()> { + Ok(()) + } + async fn build( &self, _context: super::ComputeDriverBuildContext<'_>, @@ -2097,7 +2235,7 @@ mod tests { .unwrap(), ) .unwrap(); - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let config = Config::new(None); let result = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) .unwrap(); @@ -2166,25 +2304,9 @@ mod tests { ); } - #[test] - fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None).with_compute_drivers(["alpha", "beta"]); - let err = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap_err(); - assert!( - err.to_string() - .contains("multiple compute drivers are not supported yet") - ); - assert!(err.to_string().contains("alpha,beta")); - } - #[test] fn configured_compute_driver_accepts_registered_name() { - let config = Config::new(None).with_compute_drivers(["beta"]); + let config = Config::new(None).with_compute_driver("beta"); let registry = test_compute_drivers(); let driver = configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) @@ -2201,7 +2323,7 @@ mod tests { #[test] fn configured_compute_driver_resolves_named_remote() { - let config = Config::new(None).with_compute_drivers(["kyma"]); + let config = Config::new(None).with_compute_driver("kyma"); let registry = test_compute_drivers(); let driver = @@ -2228,7 +2350,7 @@ mod tests { #[test] fn configured_compute_driver_uses_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["alpha"]) + .with_compute_driver("alpha") .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); let registry = test_compute_drivers(); @@ -2248,7 +2370,7 @@ mod tests { #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers(["beta"]) + .with_compute_driver("beta") .with_compute_driver_endpoint("beta", "/run/openshell/beta.sock"); let driver = configured_compute_driver( diff --git a/crates/openshell-server/src/tls.rs b/crates/openshell-server/src/tls.rs index aa627746ce..4231407c7a 100644 --- a/crates/openshell-server/src/tls.rs +++ b/crates/openshell-server/src/tls.rs @@ -329,41 +329,49 @@ fn load_certified_key(cert_path: &Path, key_path: &Path) -> Result, external_key_path: Option<&Path>, external_server_names: &[String], -) -> Result>> { +) -> Result<()> { match (external_cert_path, external_key_path) { - (None, None) => Ok(None), (Some(_), None) => Err(Error::tls( "external_cert_path is set but external_key_path is missing", )), (None, Some(_)) => Err(Error::tls( "external_key_path is set but external_cert_path is missing", )), - (Some(ext_cert_path), Some(ext_key_path)) => { - if external_server_names.is_empty() { - return Err(Error::tls( - "external certificate is configured but external_server_names is empty — \ - the external cert would never be served", - )); - } - let internal = load_certified_key(cert_path, key_path)?; - let external = load_certified_key(ext_cert_path, ext_key_path)?; - Ok(Some(Arc::new(DualCertResolver { - internal, - external, - external_names: external_server_names.to_vec(), - }))) - } + (Some(_), Some(_)) if external_server_names.is_empty() => Err(Error::tls( + "external certificate is configured but external_server_names is empty — \ + the external cert would never be served", + )), + (None, None) | (Some(_), Some(_)) => Ok(()), } } +/// Build an SNI-based cert resolver when an external certificate is configured. +/// Returns `None` when no external cert is configured (single-cert mode). +fn build_cert_resolver( + cert_path: &Path, + key_path: &Path, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], +) -> Result>> { + validate_external_cert_config(external_cert_path, external_key_path, external_server_names)?; + let (Some(ext_cert_path), Some(ext_key_path)) = (external_cert_path, external_key_path) else { + return Ok(None); + }; + let internal = load_certified_key(cert_path, key_path)?; + let external = load_certified_key(ext_cert_path, ext_key_path)?; + Ok(Some(Arc::new(DualCertResolver { + internal, + external, + external_names: external_server_names.to_vec(), + }))) +} + /// Build a `ServerConfig` from certificate, key, and optional client CA files. fn build_server_config( cert_path: &Path, diff --git a/crates/openshell-supervisor-middleware/src/lib.rs b/crates/openshell-supervisor-middleware/src/lib.rs index 1065c18faf..8755ddae61 100644 --- a/crates/openshell-supervisor-middleware/src/lib.rs +++ b/crates/openshell-supervisor-middleware/src/lib.rs @@ -747,6 +747,11 @@ impl Default for MiddlewareRegistry { } } +/// Validate one external middleware registration without opening its transport. +pub fn validate_registration_config(registration: &SupervisorMiddlewareService) -> Result<()> { + validate_registration(registration).map(|_| ()) +} + fn validate_registration(registration: &SupervisorMiddlewareService) -> Result { if !is_stable_identifier(®istration.name) { return Err(miette!( diff --git a/deploy/deb/openshell-gateway.service b/deploy/deb/openshell-gateway.service index 9724235ed6..c5f766a129 100644 --- a/deploy/deb/openshell-gateway.service +++ b/deploy/deb/openshell-gateway.service @@ -8,6 +8,7 @@ Type=simple StateDirectory=openshell/gateway Environment=OPENSHELL_LOCAL_TLS_DIR=%h/.local/state/openshell/tls EnvironmentFile=-%E/openshell/gateway.env +ExecStartPre=/usr/bin/openshell-gateway config preflight ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal ExecStart=/usr/bin/openshell-gateway Restart=on-failure diff --git a/deploy/docker/gateway.toml b/deploy/docker/gateway.toml index 4fe84d633a..9fbd574035 100644 --- a/deploy/docker/gateway.toml +++ b/deploy/docker/gateway.toml @@ -22,7 +22,7 @@ # - "host.openshell.internal:host-gateway" [openshell] -version = 1 +version = 2 [openshell.gateway] # Bind to loopback only. The Docker driver adds an extra listener on the @@ -30,7 +30,7 @@ version = 1 bind_address = "127.0.0.1:8080" health_bind_address = "127.0.0.1:8081" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.drivers.docker] @@ -40,12 +40,15 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" # first start. The binary is cached to XDG_DATA_HOME and reused on restart. supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # Only pull images that are not already cached locally. -image_pull_policy = "IfNotPresent" -# Prefix applied to sandbox container names. -sandbox_namespace = "openshell" +image_pull_policy = "if_not_present" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "openshell" # Address sandbox containers use to call back to the gateway. # The Docker driver replaces the host with host.openshell.internal and the # port with the gateway's own bind port (8080). Only the scheme survives. # The gateway must be published on port 8080 on the Docker host so that # host.openshell.internal:8080 resolves to the gateway container. grpc_endpoint = "http://host.openshell.internal:8080" +# Explicit supervisor-compatible Docker default. Set RuntimeDefault or +# Localhost/ only when the daemon host has AppArmor available. +app_armor_profile = "Unconfined" diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index fb3161a604..92bd2a9db8 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -276,7 +276,7 @@ discovery endpoint or its TLS CA. | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into gateway and sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | -| server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. | +| server.sandboxImagePullPolicy | string | `nil` | Pull policy for sandbox pods. Leave unset to use the Kubernetes image default (Always for :latest, IfNotPresent otherwise). Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | server.sandboxJwt.gatewayId | string | `""` | Stable gateway identity embedded in iss/aud of every minted token. Defaults to the release name so HA replicas share identity. | | server.sandboxJwt.k8sSaTokenTtlSecs | int | `3600` | Lifetime (seconds) of the projected ServiceAccount token kubelet writes into each sandbox pod for the IssueSandboxToken bootstrap exchange. Kubelet enforces a minimum of 600s; the driver clamps values outside [600, 86400]. Default 3600 — generous, since the supervisor consumes the token within seconds of pod start. | @@ -297,7 +297,7 @@ discovery endpoint or its TLS CA. | serviceAccount.annotations | object | `{}` | Annotations to add to the generated service account. | | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | -| supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | +| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index 15ff87554e..706df3eca5 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -3,7 +3,7 @@ # Merge with values.yaml for Skaffold-driven local image builds (see skaffold.yaml). server: - sandboxImagePullPolicy: IfNotPresent + sandboxImagePullPolicy: if_not_present otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 # Comment out to enforce mTLS (uses PKI secrets generated by pkiInitJob). @@ -13,4 +13,4 @@ server: supervisor: image: - pullPolicy: IfNotPresent + pullPolicy: if_not_present diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 25662518e8..626e12b140 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -89,8 +89,12 @@ spec: - name: openshell-data mountPath: /var/openshell {{- end }} + # ConfigMap directory mounts expose keys through atomic-writer symlinks, + # while the gateway intentionally rejects symlinked configuration. + # The checksum annotation above rolls pods when this subPath changes. - name: gateway-config - mountPath: /etc/openshell + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml readOnly: true - name: sandbox-jwt mountPath: /etc/openshell-jwt diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3d9f2f3e0b..98243627dd 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -252,6 +252,26 @@ database requires persistent per-pod storage. {{- default "statefulset" (get $workload "kind") | lower -}} {{- end }} +{{/* +Translate chart image pull policy values to the canonical gateway vocabulary. +The Kubernetes spellings remain accepted so existing values files continue to +work across the schema-v2 chart upgrade. +*/}} +{{- define "openshell.canonicalImagePullPolicy" -}} +{{- $policy := printf "%v" . -}} +{{- if eq $policy "Always" -}} +always +{{- else if eq $policy "IfNotPresent" -}} +if_not_present +{{- else if eq $policy "Never" -}} +never +{{- else if has $policy (list "always" "if_not_present" "never") -}} +{{- $policy -}} +{{- else -}} +{{- fail (printf "image pull policy %q must be one of: always, if_not_present, never, Always, IfNotPresent, Never" $policy) -}} +{{- end -}} +{{- end }} + {{/* Validate chart values that Helm would otherwise accept silently. */}} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 083748aee3..2c2b8216ee 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -32,7 +32,7 @@ metadata: data: gateway.toml: | [openshell] - version = 1 + version = 2 [openshell.gateway] name = {{ .Values.server.name | default (include "openshell.fullname" .) | quote }} @@ -44,29 +44,17 @@ data: metrics_bind_address = "0.0.0.0:{{ .Values.service.metricsPort }}" {{- end }} log_level = {{ .Values.server.logLevel | quote }} + compute_driver = "kubernetes" {{- if $credentialDrivers }} credential_drivers = [{{- range $i, $driver := $credentialDrivers }}{{ if $i }}, {{ end }}{{ $driver | quote }}{{- end }}] {{- end }} - sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} {{- end }} policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} - default_image = {{ .Values.server.sandboxImage | quote }} - {{- if include "openshell.supervisorImageOverrideEnabled" . }} - supervisor_image = {{ include "openshell.supervisorImage" . | quote }} - {{- end }} - {{- if .Values.server.hostGatewayIP }} - host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} - {{- end }} - {{- if .Values.server.enableUserNamespaces }} - enable_user_namespaces = true - {{- end }} {{- if .Values.server.disableTls }} disable_tls = true - {{- else }} - client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} {{- end }} enable_loopback_service_http = {{ .Values.server.enableLoopbackServiceHttp }} {{- $sans := list -}} @@ -148,10 +136,24 @@ data: {{- end }} [openshell.drivers.kubernetes] + namespace = {{ include "openshell.sandboxNamespace" . | quote }} + default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.supervisorImageOverrideEnabled" . }} + supervisor_image = {{ include "openshell.supervisorImage" . | quote }} + {{- end }} + {{- if .Values.server.hostGatewayIP }} + host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} + {{- end }} + {{- if not .Values.server.disableTls }} + client_tls_secret_name = {{ .Values.server.tls.clientTlsSecretName | quote }} + {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} + {{- if .Values.server.enableUserNamespaces }} + enable_user_namespaces = true + {{- end }} {{- if .Values.server.drivers.kubernetes.operatorNamespaceLabel }} operator_namespace_label = {{ .Values.server.drivers.kubernetes.operatorNamespaceLabel | quote }} {{- end }} @@ -183,7 +185,7 @@ data: provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} {{- end }} {{- if .Values.server.sandboxImagePullPolicy }} - image_pull_policy = {{ .Values.server.sandboxImagePullPolicy | quote }} + image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.server.sandboxImagePullPolicy | quote }} {{- end }} {{- $sandboxImagePullSecretNames := list -}} {{- range .Values.server.sandboxImagePullSecrets }} @@ -207,7 +209,7 @@ data: app_armor_profile = {{ .Values.server.appArmorProfile | quote }} {{- end }} {{- if .Values.supervisor.image.pullPolicy }} - supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} + supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }} {{- end }} [openshell.drivers.kubernetes.managed_ssh_ingress] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1380cb18c5..9805af8254 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -27,6 +27,16 @@ tests: path: data["gateway.toml"] pattern: '(?m)^name\s*=\s*"production-us-west"$' + - it: renders schema version 2 and the Kubernetes compute driver selector + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\]\s*version\s*=\s*2' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?compute_driver\s*=\s*"kubernetes"' + # Regression for Drew's P2: a ConfigMap-only mutation in `helm upgrade` # must roll the StatefulSet, otherwise pods keep running with stale config. - it: annotates the StatefulSet pod template with a ConfigMap checksum @@ -35,6 +45,23 @@ tests: - exists: path: spec.template.metadata.annotations["checksum/gateway-config"] + - it: mounts gateway.toml as a regular read-only subPath file + template: templates/statefulset.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-config + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: gateway-config + configMap: + name: openshell-config + - it: renders a StatefulSet by default template: templates/statefulset.yaml asserts: @@ -138,12 +165,77 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\][^\[]*?grpc_endpoint' - - it: renders the sandbox service account name under [openshell.drivers.kubernetes] + - it: omits pull policies by default so Kubernetes applies its own defaults + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^\s*(image_pull_policy|supervisor_image_pull_policy)\s*=' + + - it: renders canonical image pull policies in the Kubernetes driver table template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: never asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"if_not_present".*?supervisor_image_pull_policy\s*=\s*"never"' + + - it: translates legacy Kubernetes pull policy values to canonical gateway values + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_policy\s*=\s*"always".*?supervisor_image_pull_policy\s*=\s*"if_not_present"' + + - it: rejects unsupported sandbox image pull policies + template: templates/statefulset.yaml + set: + server.sandboxImagePullPolicy: Sometimes + asserts: + - failedTemplate: + errorMessage: 'image pull policy "Sometimes" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: rejects unsupported supervisor image pull policies + template: templates/statefulset.yaml + set: + supervisor.image.pullPolicy: newer + asserts: + - failedTemplate: + errorMessage: 'image pull policy "newer" must be one of: always, if_not_present, never, Always, IfNotPresent, Never' + + - it: renders driver-owned Kubernetes settings only in its driver table + template: templates/gateway-config.yaml + set: + server.hostGatewayIP: 10.0.0.1 + server.enableUserNamespaces: true + supervisor.image.tag: test + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace".*?default_image\s*=.*?supervisor_image\s*=.*?host_gateway_ip\s*=\s*"10\.0\.0\.1".*?client_tls_secret_name\s*=.*?service_account_name\s*=\s*"openshell-sandbox".*?enable_user_namespaces\s*=\s*true.*?sa_token_ttl_secs\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?(sandbox_namespace|default_image|supervisor_image|client_tls_secret_name|service_account_name|host_gateway_ip|enable_user_namespaces|sa_token_ttl_secs)\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'guest_tls_(ca|cert|key)\s*=' + + - it: renders user namespace enablement under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + server.enableUserNamespaces: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?enable_user_namespaces\s*=\s*true' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces' - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml @@ -571,6 +663,13 @@ tests: - equal: path: spec.template.spec.containers[0].name value: openshell-gateway + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: gateway-config + mountPath: /etc/openshell/gateway.toml + subPath: gateway.toml + readOnly: true - notContains: path: spec.template.spec.containers[0].volumeMounts content: @@ -732,3 +831,98 @@ tests: asserts: - failedTemplate: errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." + + - it: does not render the schema-v1 compute_drivers selector + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: "(?m)^\\s*compute_drivers\\s*=" + + - it: uses the release fullname and namespace for default Kubernetes settings + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)name\\s*=\\s*\\\"openshell\\\".*?\\[openshell\\.drivers\\.kubernetes\\].*?namespace\\s*=\\s*\\\"my-namespace\\\".*?grpc_endpoint\\s*=\\s*\\\"https://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + + - it: uses an explicit server grpc endpoint verbatim + template: templates/gateway-config.yaml + set: + server.grpcEndpoint: https://gateway.example.test:9443 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"https://gateway\\.example\\.test:9443\\\"" + + - it: uses HTTP callback and omits client TLS secret when TLS is disabled + template: templates/gateway-config.yaml + set: + server.disableTls: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?grpc_endpoint\\s*=\\s*\\\"http://openshell\\.my-namespace\\.svc\\.cluster\\.local:8080\\\"" + - notMatchRegex: + path: data["gateway.toml"] + pattern: "client_tls_secret_name\\s*=" + + - it: accepts Always pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Always + supervisor.image.pullPolicy: Always + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" + + - it: accepts IfNotPresent pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: IfNotPresent + supervisor.image.pullPolicy: IfNotPresent + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" + + - it: accepts Never pull policy spelling for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: Never + supervisor.image.pullPolicy: Never + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" + + - it: accepts canonical lowercase pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: always + supervisor.image.pullPolicy: always + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"always\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"always\\\"" + + - it: accepts canonical if_not_present pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: if_not_present + supervisor.image.pullPolicy: if_not_present + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"if_not_present\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"if_not_present\\\"" + + - it: accepts canonical never pull policies for sandbox and supervisor + template: templates/gateway-config.yaml + set: + server.sandboxImagePullPolicy: never + supervisor.image.pullPolicy: never + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: "(?ms)\\[openshell\\.drivers\\.kubernetes\\].*?image_pull_policy\\s*=\\s*\\\"never\\\".*?supervisor_image_pull_policy\\s*=\\s*\\\"never\\\"" diff --git a/deploy/helm/openshell/tests/grpc_endpoint_test.yaml b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml new file mode 100644 index 0000000000..f84442f605 --- /dev/null +++ b/deploy/helm/openshell/tests/grpc_endpoint_test.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: Kubernetes gateway callback endpoint + +templates: + - templates/gateway-config.yaml + +release: + name: team-a + namespace: gateway-system + +tests: + - it: derives callback from the gateway Service when sandbox namespace differs + set: + server.sandboxNamespace: agent-sandboxes + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^namespace\s*=\s*"agent-sandboxes"$' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=\s*"https://team-a-openshell\.gateway-system\.svc\.cluster\.local:8080"$' + - notMatchRegex: + path: data["gateway.toml"] + pattern: '(?m)^grpc_endpoint\s*=.*agent-sandboxes' diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 864e3a8512..337c991961 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -13,21 +13,27 @@ release: namespace: my-namespace tests: - - it: defaults sandbox_namespace to release namespace in the TOML config + - it: defaults the Kubernetes driver namespace to release namespace template: templates/gateway-config.yaml asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"my-namespace"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"my-namespace"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - - it: uses explicit sandboxNamespace when set + - it: uses explicit sandboxNamespace for the Kubernetes driver template: templates/gateway-config.yaml set: server.sandboxNamespace: other-ns asserts: - matchRegex: path: data["gateway.toml"] - pattern: 'sandbox_namespace\s*=\s*"other-ns"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?namespace\s*=\s*"other-ns"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'sandbox_namespace\s*=' - it: defaults NetworkPolicy namespace to release namespace template: templates/networkpolicy.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 453654f790..8f5bfa9959 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -33,8 +33,10 @@ supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor - # -- Supervisor image pull policy. Defaults to the gateway image pull policy when empty. - pullPolicy: "" + # -- Sandbox supervisor pull policy. Leave unset to use the Kubernetes + # image default. Prefer always, if_not_present, or never; the chart also + # accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. + pullPolicy: null # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" # -- How the supervisor binary is delivered into sandbox pods. @@ -224,10 +226,11 @@ server: externalDbSecret: "" # -- Default sandbox image used when requests do not specify one. sandboxImage: "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" - # -- Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default - # (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev - # clusters so new images are picked up without manual eviction. - sandboxImagePullPolicy: "" + # -- Pull policy for sandbox pods. Leave unset to use the Kubernetes image + # default (Always for :latest, IfNotPresent otherwise). Prefer always, + # if_not_present, or never; the chart also accepts legacy Kubernetes spellings + # Always, IfNotPresent, and Never. + sandboxImagePullPolicy: null # -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist # in the sandbox namespace. sandboxImagePullSecrets: [] @@ -252,8 +255,9 @@ server: # Override only when sandboxes must reach the gateway via a different # hostname (e.g. an external ingress or a host alias). grpcEndpoint: "" - # TLS configuration for the server. The server always terminates mTLS - # directly and requires client certificates. + # The gateway terminates TLS directly. Its client CA authenticates sandbox + # callbacks; OIDC-enabled listeners permit bearer-only user clients while + # still validating any client certificate they present. # -- Host gateway IP for sandbox pod hostAliases. When set, sandbox pods get # hostAliases entries mapping host.docker.internal and host.openshell.internal # to this IP, allowing them to reach services running on the Docker host. diff --git a/deploy/man/openshell-gateway.8.md b/deploy/man/openshell-gateway.8.md index 2d584c4ba1..79c878172a 100644 --- a/deploy/man/openshell-gateway.8.md +++ b/deploy/man/openshell-gateway.8.md @@ -14,6 +14,8 @@ openshell-gateway - OpenShell gateway server daemon **openshell-gateway** \[*OPTIONS*\] +**openshell-gateway** **config preflight** [**--path** *PATH* | **--** *GATEWAY_ARGS*...] + # DESCRIPTION **openshell-gateway** is the control-plane server for OpenShell. It @@ -58,12 +60,11 @@ TLS. stores SQLite state under *~/.local/state/openshell/gateway/*. Environment: **OPENSHELL_DB_URL**. -**--drivers** *DRIVER*\[,*DRIVER*\] -: Compute driver. Accepts a comma-delimited list. The gateway - currently requires exactly one driver. Options: **podman**, +**--compute-driver** *DRIVER* +: Compute driver. Selects exactly one driver. Options: **podman**, **docker**, **kubernetes**, **vm**. When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. VM is opt-in. - Environment: **OPENSHELL_DRIVERS**. + Environment: **OPENSHELL_COMPUTE_DRIVER**. **--tls-cert** *PATH* : Path to server TLS certificate file. Defaults to the local generated @@ -111,6 +112,36 @@ Compute driver settings such as sandbox image, callback endpoint, image pull policy, network name, VM state directory, and guest TLS material are configured in the TOML file passed with **--config**. +# CONFIGURATION PREFLIGHT + +Validate a gateway configuration before starting the daemon: + + openshell-gateway config preflight [--path PATH | -- GATEWAY_ARGS...] + +With no path, preflight validates a nonempty OPENSHELL_GATEWAY_CONFIG. If that +variable is unset, it optionally validates an auto-discovered XDG config. The +absence of either config succeeds. An explicit missing path, legacy schema-v1 +file, invalid TOML, symlink, or nonregular file fails with a nonzero status. +Preflight merges file and environment values and applies read-only startup checks +for selector and socket normalization, registered compute-driver configuration, +rate limits, TLS and mTLS, interceptors, and middleware. When a selected file +omits the selector, it validates configured tables for auto-detectable drivers +without running socket or process-based detection probes. It does not construct a +compute driver or connect to a transport. Preflight never changes the file and +reports that failed input was preserved. + +Arguments after **--** replace **--path** mode and are parsed as the exact gateway +daemon invocation. Package wrappers use this form so command-line overrides are +validated before the same arguments reach startup. + +The Debian and Ubuntu systemd user unit runs preflight before certificate +generation, while retaining its EnvironmentFile and bare ExecStart behavior. The +Snap wrapper replays its effective daemon arguments through preflight. It first +uses a nonempty OPENSHELL_GATEWAY_CONFIG. Otherwise it passes the canonical +SNAP_COMMON/gateway.toml path whenever it exists or is a symlink. A broken symlink +fails preflight before the gateway is started. Correct or manually migrate an +operator-owned v1 file, then run preflight again before restarting the service. + # SYSTEMD INTEGRATION The package installs a systemd user unit at @@ -127,8 +158,9 @@ View logs: journalctl --user -u openshell-gateway journalctl --user -u openshell-gateway -f -The unit runs **openshell-gateway generate-certs** as an **ExecStartPre** -step on first start. This generates a self-signed PKI bundle for mTLS +The unit runs **openshell-gateway config preflight** and then +**openshell-gateway generate-certs** as **ExecStartPre** steps. Certificate +generation creates a self-signed PKI bundle for mTLS and sandbox JWT signing material, adding missing JWT files to older TLS-only installs when needed. The packaged unit sets **OPENSHELL_LOCAL_TLS_DIR** to *~/.local/state/openshell/tls* and uses that diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index 4fc18e6215..f6889e4fb9 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -17,10 +17,10 @@ The defaults are tuned for rootless Podman use: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] +compute_driver = "podman" ``` The RPM does not override `bind_address`. The primary listener uses the @@ -28,15 +28,17 @@ built-in `127.0.0.1:17670` default. The Podman driver reports the callback interface it needs, and the gateway adds a separate listener scoped to that interface. This keeps the general API off unrelated host interfaces. -`compute_drivers = ["podman"]` pins the compute driver to Podman. Without +`compute_driver = "podman"` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver selection if Docker is also installed on the host. ### Customizing the configuration -Edit `~/.config/openshell/gateway.toml` directly. The template at -`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime -and is not overwritten by RPM upgrades. +Edit `~/.config/openshell/gateway.toml` directly. The package-owned template at +`/usr/share/openshell-gateway/gateway.toml.default` is not read at runtime and +may change during an RPM upgrade. The active user copy is preserved. During a +schema-v2 upgrade, the service replaces only an exact package-generated v1 +copy; it never rewrites an edited configuration. To apply environment variable overrides that persist across upgrades without editing the TOML file, add them to `~/.config/openshell/gateway.env`: @@ -215,10 +217,10 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| | `bind_address` | `127.0.0.1:17670` (gateway default) | Address for the primary gRPC/HTTP API listener. | -| `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | -| `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | -| `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | -| `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | +| `compute_driver` | `"podman"` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman; legacy `compute_drivers` lists are rejected. | +| `[openshell.drivers.podman].default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | +| `[openshell.drivers.podman].supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | +| `[openshell.gateway].guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Gateway-owned client TLS material injected into the selected local driver and mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | @@ -232,14 +234,15 @@ settings: ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +compute_driver = "podman" [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell" stop_timeout_secs = 10 ``` @@ -247,7 +250,7 @@ stop_timeout_secs = 10 ### Image management The gateway pulls container images automatically on first sandbox -creation. The default pull policy is `missing`, which means images are +creation. The default pull policy is `if_not_present`, which means images are pulled once and then cached by Podman. To update cached images: @@ -260,9 +263,10 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest Or set `image_pull_policy = "always"` in `[openshell.drivers.podman]` to pull on every sandbox creation. -To pin specific image versions instead of `:latest`: +To pin specific image versions instead of `:latest`, set these values in +`[openshell.drivers.podman]`: -```shell +```toml supervisor_image = "ghcr.io/nvidia/openshell/supervisor:v0.0.37" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:v0.0.37" ``` diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 103ce3bf9d..58de9e7856 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -182,7 +182,7 @@ podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ### Images not updating -The default image pull policy is `missing` -- images are pulled once +The default image pull policy is `if_not_present` -- images are pulled once and cached. To update: ```shell @@ -227,9 +227,14 @@ non-functional until restarted, causing the gateway to fail with a connection error on `/run/user//podman/podman.sock`. The gateway retries briefly on startup, but a stale socket will not recover on its own. -Package upgrades do not overwrite `~/.config/openshell/gateway.toml` when you -create one. New gateway process options can be added manually by referencing -CONFIGURATION.md or running `openshell-gateway --help`. +Package upgrades preserve edited `~/.config/openshell/gateway.toml` files. On +the schema-v2 upgrade, the user service replaces only an exact copy of the v1 +file previously seeded by the RPM. If you edited that file, migrate it manually +before restarting the service; direct `dnf` or `rpm` upgrades do not use the +breaking-upgrade guard in `install.sh`. See the +[Gateway Configuration File](https://docs.nvidia.com/openshell/latest/reference/gateway-config#migrate-to-schema-version-2) +for the field-by-field migration steps. New gateway process options are listed +in CONFIGURATION.md and `openshell-gateway --help`. To pick up new container images after an upgrade: @@ -238,6 +243,17 @@ podman pull ghcr.io/nvidia/openshell/supervisor:latest podman pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest ``` +### Migrating a TLS-enabled local driver to schema version 2 + +Docker, Podman, and VM sandboxes connect back to the gateway with a guest TLS +bundle. Package-managed installs use the complete bundle generated under +`~/.local/state/openshell/tls`, so the RPM default requires no additional TOML. +If you override the listener with custom `--tls-cert` and `--tls-key` inputs and +do not use that managed bundle, configure all three `guest_tls_ca`, +`guest_tls_cert`, and `guest_tls_key` paths under `[openshell.gateway]`. The +gateway now fails at startup instead of allowing sandboxes to fail later. Omit +all three fields when TLS is disabled. + ### Migrating from gateway.env Previous releases generated `~/.config/openshell/gateway.env` on first @@ -255,7 +271,7 @@ and map the relevant variables: | Environment variable | TOML equivalent | |---|---| | `OPENSHELL_BIND_ADDRESS=A` + `OPENSHELL_SERVER_PORT=P` | `bind_address = "A:P"` under `[openshell.gateway]` | -| `OPENSHELL_DRIVERS=podman` | `compute_drivers = ["podman"]` under `[openshell.gateway]` | +| `OPENSHELL_COMPUTE_DRIVER=podman` | `compute_driver = "podman"` under `[openshell.gateway]` | | `OPENSHELL_DISABLE_TLS=true` | `disable_tls = true` under `[openshell.gateway]` | | `OPENSHELL_TLS_CERT=PATH` | `cert_path = "PATH"` under `[openshell.gateway.tls]` | | `OPENSHELL_TLS_KEY=PATH` | `key_path = "PATH"` under `[openshell.gateway.tls]` | diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index cd7e0d99c3..a0a6e296f0 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -15,7 +15,7 @@ # systemctl --user edit openshell-gateway [openshell] -version = 1 +version = 2 [openshell.gateway] # Keep the primary listener on the built-in 127.0.0.1:17670 default. The @@ -25,4 +25,9 @@ version = 1 # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver # selection if Docker is also installed on the host. -compute_drivers = ["podman"] +compute_driver = "podman" + +[openshell.drivers.podman] +# Keep the packaged local-gateway readiness behavior after health checks became +# opt-in in the Podman driver. Omit this setting only to disable health checks. +health_check_interval_secs = 10 diff --git a/deploy/rpm/gateway.toml.default.v1 b/deploy/rpm/gateway.toml.default.v1 new file mode 100644 index 0000000000..cd7e0d99c3 --- /dev/null +++ b/deploy/rpm/gateway.toml.default.v1 @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_drivers = ["podman"] diff --git a/deploy/rpm/migrate-gateway-config.sh b/deploy/rpm/migrate-gateway-config.sh new file mode 100755 index 0000000000..4794ffa454 --- /dev/null +++ b/deploy/rpm/migrate-gateway-config.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +if [ "$#" -ne 3 ]; then + echo "usage: $0 DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT" >&2 + exit 2 +fi + +destination=$1 +current_default=$2 +legacy_default=$3 + +for source in "$current_default" "$legacy_default"; do + # Package-owned defaults must be ordinary files; never follow a symlink. + if [ -L "$source" ] || [ ! -f "$source" ]; then + echo "gateway config migration source is not a regular file: $source" >&2 + exit 1 + fi +done + +if [ -L "$destination" ] || { [ -e "$destination" ] && [ ! -f "$destination" ]; }; then + echo "refusing to replace non-regular gateway config: $destination" >&2 + exit 1 +fi + +if [ ! -e "$destination" ]; then + install -Dm 0644 "$current_default" "$destination" + exit 0 +fi + +# Replace only the exact config seeded by the schema-v1 RPM. Any edit, +# including whitespace or comments, makes the operator-owned file authoritative. +if ! cmp -s "$legacy_default" "$destination"; then + exit 0 +fi + +destination_dir=$(dirname "$destination") +temporary=$(mktemp "$destination_dir/.gateway.toml.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM +install -m 0644 "$current_default" "$temporary" +mv -f "$temporary" "$destination" +trap - EXIT HUP INT TERM diff --git a/docs/about/container-gateway.mdx b/docs/about/container-gateway.mdx index 42fe2c7858..57be7d0142 100644 --- a/docs/about/container-gateway.mdx +++ b/docs/about/container-gateway.mdx @@ -59,7 +59,7 @@ docker run -d \ -v openshell-state:/var/openshell \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=http://host.openshell.internal:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ @@ -128,7 +128,7 @@ docker run -d \ -v "$HOME/.local/state/openshell:/home/openshell/.local/state/openshell" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ~/openshell/supervisor/openshell-sandbox:~/openshell/supervisor/openshell-sandbox:ro \ - -e OPENSHELL_DRIVERS=docker \ + -e OPENSHELL_COMPUTE_DRIVER=docker \ -e OPENSHELL_GRPC_ENDPOINT=https://127.0.0.1:8080 \ -e OPENSHELL_DOCKER_SUPERVISOR_BIN=~/openshell/supervisor/openshell-sandbox \ -e OPENSHELL_DB_URL=sqlite:/home/openshell/.local/state/openshell/openshell.db \ @@ -188,7 +188,7 @@ podman run -d \ -p 127.0.0.1:8080:8080 \ -v openshell-state:/var/openshell \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/podman.sock" \ - -e OPENSHELL_DRIVERS=podman \ + -e OPENSHELL_COMPUTE_DRIVER=podman \ -e OPENSHELL_PODMAN_SOCKET=/var/run/podman.sock \ -e OPENSHELL_DB_URL=sqlite:/var/openshell/openshell.db \ -e OPENSHELL_DISABLE_TLS=true \ diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index a733a1b881..a40d908fa3 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -30,7 +30,7 @@ Use `openshell status` to confirm the CLI can reach the gateway. ## Supported Compute Drivers -OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_drivers` in the gateway TOML when you need to pin a specific driver. +OpenShell supports several local compute drivers. Package-managed gateways leave the driver unset by default so the gateway can auto-detect an available driver. Set `compute_driver` in the gateway TOML when you need to pin a specific driver. | Compute Driver | How It Is Configured | System Requirements | |---|---|---| @@ -44,7 +44,7 @@ For detailed driver behavior, refer to [Sandbox Compute Drivers](/reference/sand On macOS, the install script uses Homebrew. The Homebrew package installs the `openshell` CLI, the gateway binary, and a Homebrew-managed gateway service. -The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew preserves user-edited prefix and user configs during upgrades; it removes the IPv6 bind only from an unchanged config generated by the affected formula. +The Homebrew service uses the gateway's built-in `127.0.0.1:17670` listener and generates a local mTLS bundle on install. The installer registers `https://localhost:17670` with the CLI so TLS uses a DNS name covered by the generated certificate. The formula creates a Homebrew prefix config, such as `/opt/homebrew/var/openshell/gateway.toml`, without overriding `bind_address`. Docker Desktop and Podman Machine reuse the primary listener for sandbox callbacks when they can reach it. The gateway reads `~/.config/openshell/gateway.toml` instead when that file exists. Homebrew upgrades migrate exact package-generated schema-v1 prefix configs, including the affected IPv6 variant. They preserve edited prefix configs and all user configs. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) for an edited v1 file. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. @@ -63,7 +63,7 @@ On Debian and Ubuntu, the install script uses a Debian package. The Debian packa Linux packages require glibc 2.28 or newer. The installer checks libc before downloading packages and exits with an error on older glibc versions, Alpine, musl-based distributions, or unknown libc environments. -The Linux user service listens on `https://127.0.0.1:17670`, starts from built-in defaults, and generates a local mTLS bundle before the gateway starts. Create `~/.config/openshell/gateway.toml` only when you need to override those defaults. +The Linux user service listens on `https://127.0.0.1:17670` and generates a local mTLS bundle before the gateway starts. Debian uses built-in gateway defaults unless you create a config. RPM seeds `~/.config/openshell/gateway.toml` from its packaged Podman template on first start. RPM upgrades migrate only an unchanged package-generated schema-v1 file; they preserve edited files. Follow the [schema version 2 migration steps](/reference/gateway-config#migrate-to-schema-version-2) when upgrading an edited v1 configuration. The CLI reads the client bundle from `~/.config/openshell/gateways/openshell/mtls/`. @@ -155,3 +155,30 @@ Kubernetes deployments use the OpenShell Helm chart. For step-by-step installati - To register, select, and inspect gateways, refer to [Gateways](/sandboxes/manage-gateways). - To supply API keys or tokens, refer to [Manage Providers](/sandboxes/manage-providers). - To control what the agent can access, refer to [Policies](/sandboxes/policies). + +## Validate a package-managed gateway configuration + +Debian and Ubuntu packages validate the selected gateway configuration before +generating local certificates or starting the service. Snap does the same before +its gateway daemon starts. The validation never changes the file. If startup reports a legacy schema, +malformed TOML, a symlink, or a nonregular configuration path, fix or manually +migrate the operator-owned file instead of deleting it. + +Check the selected file before restarting a service: + +```shell +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +``` + +Without `--path`, preflight checks a nonempty `OPENSHELL_GATEWAY_CONFIG` or an +auto-discovered XDG config; no config is also a successful result. It applies +read-only startup validation to the effective selector, registered compute driver, +driver configuration, socket, rate limits, TLS, interceptors, and middleware. If +a selected file omits the selector, preflight validates configured tables for +auto-detectable drivers without running socket or process-based detection probes. +Wrappers can pass daemon arguments after `--` to validate the exact startup +invocation. Debian keeps its bare service invocation and `gateway.env` semantics. +Snap replays its effective daemon arguments through preflight and gives a nonempty +`OPENSHELL_GATEWAY_CONFIG` precedence over `SNAP_COMMON/gateway.toml`. +See [Gateway Configuration](/reference/gateway-config#gateway-config-preflight) +for preflight details and manual schema-v1 migration steps. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 14cdb05ece..2d2fb9ad55 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -22,26 +22,28 @@ Gateway CLI flag > gateway OPENSHELL_* env var > TOML file > built-in defa ## Package-Managed Locations -Package-managed gateways do not require a TOML file. Create one at the package's optional config location when you need to override built-in defaults. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. +Package-managed gateways use either built-in defaults or a package-seeded TOML file. Set `OPENSHELL_GATEWAY_CONFIG` in the launch environment to use a different file. -| Package | Optional Gateway TOML location | +| Package | Gateway TOML location | |---|---| | Homebrew | `$XDG_CONFIG_HOME/openshell/gateway.toml` when it exists, otherwise the Homebrew prefix config such as `/opt/homebrew/var/openshell/gateway.toml`. | | Debian/Ubuntu | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | -| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml` for the systemd user service. | +| Fedora/RHEL RPM | `$XDG_CONFIG_HOME/openshell/gateway.toml`, usually `~/.config/openshell/gateway.toml`; the systemd user service seeds this file from the packaged template on first start. | | Snap | `$SNAP_COMMON/gateway.toml`, usually `/var/snap/openshell/common/gateway.toml`. | The Fedora/RHEL RPM template leaves `[openshell.gateway].bind_address` unset. The gateway therefore uses its built-in `127.0.0.1:17670` primary listener. The Podman driver negotiates separate, restricted listeners for sandbox callbacks, so the primary listener does not need a wildcard address. Set `bind_address` explicitly only when clients must reach the primary multiplexed API through another interface. -The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. Upgrades preserve user-edited configs and migrate only an unchanged prefix config generated with the affected IPv6-loopback default. +The Homebrew formula creates its prefix config without setting `bind_address`, so the gateway uses its built-in `127.0.0.1:17670` primary listener. Docker Desktop and Podman Machine reuse that listener for sandbox callbacks. A user config takes precedence. + +Homebrew and RPM upgrades migrate only exact package-generated schema-v1 defaults. Homebrew recognizes both its empty v1 prefix config and the affected IPv6-loopback variant. RPM recognizes the v1 file seeded by its systemd user service. Package upgrades never rewrite an edited file; migrate an edited v1 file manually with the steps below. ## Layout -The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Shared compute-driver keys set at gateway scope are inherited into compute driver tables when not overridden. +The file is rooted at `[openshell]`. Gateway-wide settings live under `[openshell.gateway]`. Each compute driver owns its own `[openshell.drivers.]` table. Credential drivers own `[openshell.credential_drivers.]` tables. Driver-specific values are never inherited from gateway scope. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] # ... gateway-wide settings ... @@ -59,6 +61,52 @@ version = 1 # ... credential-driver-specific settings ... ``` +The gateway selector is `compute_driver = ""`. It accepts one scalar driver name. Omit it to retain auto-detection. + +## Migrate to schema version 2 + +Schema version 2 is an intentional breaking cutover. The gateway rejects files +that omit `[openshell] version`, declare version 1, or declare an unsupported +future version. To migrate an existing file: + +1. Set `[openshell] version = 2`. +2. Replace `compute_drivers = [""]` with the scalar + `compute_driver = ""`. Replace `--drivers` and `OPENSHELL_DRIVERS` + with `--compute-driver` and `OPENSHELL_COMPUTE_DRIVER`. Package upgrades + temporarily accept one non-empty `OPENSHELL_DRIVERS` value as a deprecated + environment-only alias when it agrees with any canonical selection. + Comma-delimited, invalid, or conflicting legacy values fail startup. The + removed `--driver` and `--drivers` flags remain unsupported. +3. Move every compute-driver option into `[openshell.drivers.]`. Schema + version 2 does not inherit driver defaults from `[openshell.gateway]`. + Keep only `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` at gateway + scope. A TLS-enabled Docker, Podman, or VM gateway requires one complete + guest bundle. Set all three paths unless the package-managed local TLS + bundle supplies them. When TLS is disabled, omit all three. Kubernetes + projects sandbox TLS through `client_tls_secret_name` instead. +4. Rename Docker `sandbox_namespace` to `sandbox_label`, Podman + `sandbox_ssh_socket_path` to `ssh_socket_path`, and VM + `openshell_endpoint` to `grpc_endpoint`. +5. Use canonical image pull policies: `always`, `if_not_present`, `never`, or + Podman-only `newer`. Kubernetes-style capitalization and Podman's `missing` + spelling are rejected. +6. Remove zero sentinels. Omit `gateway_jwt.ttl_secs` for a non-expiring token, + omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit + of 2048, and omit Podman `health_check_interval_secs` to disable health + checks. Explicit zero values are invalid. +7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived + callback is correct, or retain it as an explicit override. Kubernetes raw + TOML requires an explicit endpoint; Helm derives one from the release's + gateway Service. New VM root filesystems use an image-provided `sandbox` + account when present and otherwise use UID/GID 1000. Existing persisted VM + state retains its recorded or recoverable identity, including legacy 10001; + the driver does not assign 10001 to an overlay without supporting state. + +Every `[openshell.drivers.]` entry must be a TOML table. The gateway +validates driver-specific fields when it selects and constructs that driver; +it does not deserialize unselected driver tables. Unknown or misplaced fields +in the selected table fail startup instead of being silently ignored. + ## Full Example A complete gateway configuration covering every section. Trim to the fields you need. @@ -68,7 +116,7 @@ A complete gateway configuration covering every section. Trim to the fields you # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] name = "production-us-west" @@ -78,15 +126,14 @@ metrics_bind_address = "0.0.0.0:9090" log_level = "info" -# When empty, the gateway auto-detects Kubernetes, then Podman, then Docker. +# When omitted, the gateway auto-detects Kubernetes, then Podman, then Docker. # VM is never auto-detected and requires an explicit entry here. -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" # Optional external provider credential storage backend. Omit this key to use # the gateway's default encrypted database credential storage. credential_drivers = ["kubernetes-secrets"] -sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 # Reject invalid policy generations securely by default. Set @@ -103,16 +150,11 @@ enable_loopback_service_http = true # Set true only for local plaintext gateways or trusted TLS termination. disable_tls = false -# Shared driver defaults. These inherit into [openshell.drivers.] tables -# when the driver-specific table does not override them. -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -# Defaults to the gateway version; override to pin a specific build. -# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -client_tls_secret_name = "openshell-client-tls" -service_account_name = "openshell-sandbox" -host_gateway_ip = "10.0.0.1" -enable_user_namespaces = false -sa_token_ttl_secs = 3600 +# Guest TLS paths remain gateway settings. TLS-enabled Docker, Podman, and VM +# gateways require a complete bundle unless package-managed local TLS supplies +# it automatically. Omit all three when TLS is disabled. Kubernetes projects +# sandbox TLS from client_tls_secret_name instead. Driver tables must not repeat +# these fields. guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -146,7 +188,6 @@ timeout = "500ms" cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" client_ca_path = "/etc/openshell/certs/client-ca.pem" -require_client_auth = false # Optional: SNI-based dual certificate for external (e.g. ACME) TLS. # external_cert_path = "/etc/openshell/certs/external.pem" # external_key_path = "/etc/openshell/certs/external-key.pem" @@ -157,7 +198,7 @@ signing_key_path = "/etc/openshell/jwt/signing.pem" public_key_path = "/etc/openshell/jwt/public.pem" kid_path = "/etc/openshell/jwt/kid" gateway_id = "openshell" -# Omit or set to 0 only for local single-player Docker, Podman, or VM gateways. +# Omit only for local single-player Docker, Podman, or VM gateways. ttl_secs = 3600 [openshell.gateway.auth] @@ -200,18 +241,33 @@ failure_policy = "fail_closed" rpc = "openshell.v1.OpenShell/UpdateConfig" phases = ["validate"] +[openshell.drivers.kubernetes] +namespace = "openshell" +# Required in raw TOML; Helm derives this from the gateway Service. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" +default_image = "ghcr.io/nvidia/openshell/sandbox:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" +client_tls_secret_name = "openshell-client-tls" +service_account_name = "openshell-sandbox" +host_gateway_ip = "10.0.0.1" +enable_user_namespaces = false +sa_token_ttl_secs = 3600 + [openshell.credential_drivers.kubernetes-secrets] namespace = "openshell" allow_reference_namespace = false ``` -Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to map a verified client certificate to a CLI user identity. This application-layer identity switch does not control the TLS handshake. When `client_ca_path` is set without OIDC, the listener requires a valid client certificate. When OIDC is configured, bearer-only clients may connect; the listener still validates any client certificate they present against the configured CA. Kubernetes deployments must leave `mtls_auth.enabled` unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. + +The client-certificate handshake policy is derived and has no `require_client_auth` TOML field. This preserves bearer-only OIDC clients and prevents a file setting from silently weakening CA-only gateways. `[openshell.gateway.tls]` supports optional SNI-based dual-certificate mode for deployments that need separate internal and external server certificates. Set `external_cert_path` and `external_key_path` to point at the external (e.g. ACME/publicly-trusted) certificate and key. List the hostnames that should be served with the external certificate in `external_server_names`. Connections whose TLS SNI hostname matches one of those names receive the external certificate; all other connections (including those with no SNI) receive the primary internal certificate from `cert_path`/`key_path`. Both fields must be set together — providing only one is a configuration error. On Kubernetes with the Helm chart, the external certificate is managed automatically when `certManager.serverIssuerRef.name` is set; the chart populates these fields from the cert-manager-issued external server certificate. `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. -`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. +`[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. Omit it for a non-expiring token: the token `exp` claim and `expires_at_ms` response field become `0`. Use this only for local single-player Docker, Podman, or VM gateways. Explicit `0` is invalid. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway omits the field. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. @@ -355,7 +411,7 @@ The gateway validates snapshot structure and provider-profile semantics. It trea `failure_policy` accepts `fail_closed` or `fail_open`. `timeout` accepts `ms` and `s` suffixes. In `dynamic` mode, binding overrides may select a manifest binding by `id`, `rpc`, or `service` plus `method`; they can disable a binding, narrow its phases, or override its failure policy. -`image_pull_policy` is intentionally not a shared gateway key. Kubernetes and Docker use `Always`, `IfNotPresent`, or `Never`. Podman uses `always`, `missing`, `never`, or `newer`. Set it inside the relevant driver table. +`image_pull_policy` is a shared driver setting with the canonical values `always`, `if_not_present`, `never`, and `newer`. Set it inside the relevant driver table. Drivers translate these values to their runtime APIs; `newer` is supported only by Podman and is rejected at Docker and Kubernetes startup. ## Credential Drivers @@ -444,7 +500,9 @@ args = [ ## Driver References -Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Driver-specific values override inherited gateway defaults. The gateway rejects unknown driver fields after inheritance is merged. +Each example is a complete TOML file for one compute driver. The examples repeat `[openshell]` and `[openshell.gateway]` so they stay copyable, and the driver tables list the accepted driver-specific keys. Drivers receive only their own tables, and the gateway rejects unknown gateway and driver fields. + +Kubernetes configurations set `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`. Docker configurations use `sandbox_label`; the legacy `sandbox_namespace` key is rejected. ### Kubernetes @@ -452,14 +510,14 @@ The gateway runs as a Pod and creates sandbox Pods in another namespace. mTLS ma ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "0.0.0.0:8080" health_bind_address = "0.0.0.0:8081" metrics_bind_address = "0.0.0.0:9090" log_level = "info" -compute_drivers = ["kubernetes"] +compute_driver = "kubernetes" [openshell.gateway.tls] cert_path = "/etc/openshell-tls/server/tls.crt" @@ -482,11 +540,11 @@ workspace_mode = "shared" namespace = "agents" service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. @@ -521,7 +579,10 @@ topology = "combined" # Last resort for hostname-filtering proxy ACLs. The proxy resolves the target, # so its ACL becomes part of the egress boundary for proxied connections. # proxy_connect_by_hostname = true -grpc_endpoint = "https://openshell-gateway.agents.svc:8080" +# Required in raw gateway TOML because `namespace` identifies sandbox +# placement, not the gateway Service. Helm renders this from the release's +# gateway Service name and namespace. +grpc_endpoint = "https://openshell-gateway.openshell.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" @@ -591,33 +652,36 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required). Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.docker] socket_path = "/var/run/docker.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -# Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. -image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" -# Empty auto-detects https://host.openshell.internal: when guest TLS is set. +# Canonical values: always | if_not_present | never. `newer` is Podman-only. +image_pull_policy = "if_not_present" +# Value assigned to the openshell.sandbox_namespace label on sandbox containers. +sandbox_label = "docker-dev" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for this topology. grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" ssh_socket_path = "/run/openshell/ssh.sock" @@ -625,22 +689,42 @@ ssh_socket_path = "/run/openshell/ssh.sock" # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 +# Explicit supervisor-compatible default. RuntimeDefault requires Docker to +# report AppArmor support; Localhost/ requires an operator-loaded profile. +app_armor_profile = "Unconfined" +# Corporate TLS egress proxy. These are supervisor argv settings, not workload +# environment variables. Do not embed credentials in the URL. +https_proxy = "https://proxy.corp.example:8443" +no_proxy = ".svc.cluster.local,10.0.0.0/8" +# Optional root-owned host file containing user:pass. An http:// proxy also +# requires proxy_auth_allow_insecure = true as an explicit acknowledgement. +proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# Project a host Unix Workload API socket into the supervisor for provider +# token exchange. The socket parent must be a dedicated absolute directory. +provider_spiffe_workload_api_socket = "/run/spire/agent.sock" ``` +Use `sandbox_label` for Docker configurations. The legacy +`sandbox_namespace` key is rejected. + ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount. Configure guest mTLS paths once under `[openshell.gateway]`; the gateway validates and injects the bundle into the selected local driver. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] # Rootless socket path. For root Podman use /run/podman/podman.sock. @@ -649,7 +733,8 @@ compute_drivers = ["podman"] # one. Set this to pin a specific Podman machine instead. socket_path = "/run/user/1000/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer +# Optional override. When omitted, the gateway derives this endpoint. grpc_endpoint = "https://host.containers.internal:17670" # The gateway overwrites gateway_port from bind_address at runtime. gateway_port = 17670 @@ -657,22 +742,19 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" -sandbox_ssh_socket_path = "/run/openshell/ssh.sock" +ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 45 # Defaults to the gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" # Unsafe operator override. Host bind mounts, including Podman local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. enable_bind_mounts = false -# Set to 0 to leave Podman's runtime default unchanged. +# Omit to use OpenShell's 2048-process default. Explicit 0 is invalid. sandbox_pids_limit = 2048 -# Health check interval in seconds. Lower values detect readiness faster -# but increase process churn (each check spawns a conmon subprocess). -# Set to 0 to disable health checks entirely. Default: 10. +# Health check interval in seconds. Omit to disable health checks; explicit 0 +# is invalid. Lower values detect readiness faster but increase process churn +# (each check spawns a conmon subprocess). health_check_interval_secs = 10 # User namespace mode for sandbox containers. Omit to use the default. # Supported modes: auto, host, keep-id, no-map, private. @@ -761,52 +843,72 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# Project a host Workload API Unix socket into the supervisor, or use an +# explicit container-reachable TCP endpoint, for provider token exchange. +# provider_spiffe_workload_api_socket = "/run/spire/agent.sock" +# provider_spiffe_workload_api_socket = "tcp:169.254.1.2:8081" +# Omit app_armor_profile to preserve Podman's runtime-selected profile. +# Set Unconfined only when the supervisor's mount setup requires it. +# Explicit RuntimeDefault and Localhost/ require Podman to report +# AppArmor support. +# app_armor_profile = "Unconfined" ``` +Use `ssh_socket_path` for Podman configurations. The legacy +`sandbox_ssh_socket_path` key is rejected. When `app_armor_profile` is omitted, +OpenShell sends no override and Podman applies its runtime-selected profile. +Set `Unconfined` explicitly only when the deployment requires the supervisor's +mount setup to bypass that profile. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" # VM is never auto-detected; an explicit entry here is required. -compute_drivers = ["vm"] +compute_driver = "vm" +# Gateway-owned bundle injected into the selected local driver. +guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" +guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -grpc_endpoint = "https://host.containers.internal:17670" +# Optional override. When omitted, the gateway derives +# https://host.openshell.internal: for the VM topology. +grpc_endpoint = "https://host.openshell.internal:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" -# Resolved sandbox UID/GID for the rootfs /etc/passwd entry. -# Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. -# Any non-root Linux UID/GID is valid. +# Resolved sandbox UID/GID for new rootfs /etc/passwd entries. +# Defaults to the image's sandbox account, or 1000 when the account is absent; +# matching GID is used if sandbox_gid is empty. Persisted overlays recover their +# recorded identity, including 10001, rather than receiving a legacy fallback. +# Values must fall within OpenShell's allowed non-root sandbox identity range. # sandbox_uid = 20001 +# sandbox_gid = 20001 # Corporate forward proxy for sandbox egress. The keys, their semantics, and # the fail-closed contract are identical to the Podman driver above: only TLS # (CONNECT) egress is chained, plain-HTTP destination requests always dial # directly, credentials must come from proxy_auth_file rather than the URL, # an http:// proxy with credentials requires proxy_auth_allow_insecure, and # any present-but-invalid value is rejected at gateway startup rather than -# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on -# the gateway host. +# degrading to a direct dial. proxy_auth_file is a path on the gateway host. # # The sandbox cannot select or override these settings. They reach the guest -# supervisor on its command line through a per-sandbox file the driver writes +# supervisor through a protected per-sandbox argument file the driver writes # into the overlay upperdir on every launch, so a sandbox image cannot supply # its own values or disable the operator's by baking a file at that path. # @@ -826,20 +928,29 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" # address routable from the guest's masqueraded egress. # # Because a microVM has no bind mounts or container secrets, the driver stages -# the credential and the CA into the per-sandbox overlay disk: the credential -# root-only inside the guest, and both removed with the sandbox. The +# the credential and optional CA bundle into the per-sandbox overlay disk and +# removes them with the sandbox. The credential is root-only; the non-secret CA +# bundle is mode 0644. Both are referenced from the protected supervisor +# argument file by fixed guest paths, never their gateway-host paths. The # credential is therefore at rest in that overlay image on the gateway host — -# the same delivery the per-sandbox gateway token already uses, and a -# difference from the Podman secret model worth noting when choosing where to -# keep proxy credentials. +# the same delivery the per-sandbox gateway token already uses, and a difference +# from the Podman secret model worth noting when choosing where to keep proxy +# credentials. # https_proxy = "http://host.openshell.internal:8080" # no_proxy = "10.0.0.0/8,.internal.example" # proxy_auth_file = "/etc/openshell/secrets/proxy-auth" +# An http:// proxy with proxy_auth_file requires this explicit acknowledgement: # proxy_auth_allow_insecure = true # Last resort for hostname-filtering proxy ACLs; see the Podman section above. # proxy_connect_by_hostname = true -# Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. +# Gateway-host PEM bundle trusted for an https:// proxy and for server +# certificates re-signed by a TLS-intercepting proxy. Requires https_proxy. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# VM guests cannot mount a host Workload API Unix socket. Configure only a +# separately operated guest-reachable TCP listener and explicitly acknowledge +# the exposure; host-only sockets are never exposed automatically. +# provider_spiffe_workload_api_tcp_endpoint = "tcp:192.0.2.10:8081" +# provider_spiffe_allow_guest_tcp = true # Where the gateway stages rootfs tar archives for `--from ./rootfs.tar`. # Defaults to /rootfs-tar-staging. The gateway creates one # request-scoped subdirectory per staging slot and removes it after use. @@ -867,13 +978,65 @@ key used for driver-owned sandbox config such as `template.driver_config.` ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:17670" log_level = "info" -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma-compute-driver.sock" ``` + +## Preflight package configuration {#gateway-config-preflight} + +Before starting a package-managed gateway, validate the selected file without +changing it: + +```shell +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +``` + +Without `--path`, the command validates a nonempty `OPENSHELL_GATEWAY_CONFIG`. +Otherwise, it validates an existing XDG gateway config when one is discovered. +When neither source selects a config, preflight succeeds. An explicit missing path, +a legacy schema-v1 file, invalid TOML, a symlink, or any nonregular file fails. +Preflight merges the selected file with the current `OPENSHELL_*` environment and +applies the daemon's read-only startup checks. These checks include selector and +socket normalization, registered-driver selection and configuration, rate-limit +pairs, TLS and mTLS relationships, interceptor registrations, and supervisor +middleware registrations. When a selected file omits `compute_driver`, preflight +validates each configured table for an auto-detectable driver without running the +runtime detection probes, which can connect local sockets or launch discovery +commands. It validates complete guest TLS path sets without requiring +package-generated certificates to exist before certificate generation. It does +not construct a compute driver or connect to a transport. A failed +preflight always preserves the file; it never migrates, replaces, or rewrites +configuration. + +To validate the exact daemon arguments that a wrapper will pass, place them after +`--` instead of using `--path`: + +```shell +openshell-gateway config preflight -- --config /etc/openshell/gateway.toml --grpc-rate-limit-requests 100 --grpc-rate-limit-window-seconds 60 +``` + +Debian and Ubuntu run preflight from the systemd user unit before local certificate +generation. The unit still loads the `gateway.env` environment file and starts the +gateway with no configuration arguments. Snap replays the exact effective daemon +arguments through preflight. It gives a nonempty `OPENSHELL_GATEWAY_CONFIG` +precedence; otherwise it validates and passes its canonical +`SNAP_COMMON/gateway.toml` only when that path exists in the filesystem. A broken +symlink is therefore rejected instead of being treated as absent. + +Package startup does not modify an operator-owned v1 file. Back it up, follow +[Migrate to schema version 2](#migrate-to-schema-version-2), then validate the +result explicitly before restarting the service: + +```shell +cp ~/.config/openshell/gateway.toml ~/.config/openshell/gateway.toml.v1.bak +$EDITOR ~/.config/openshell/gateway.toml +openshell-gateway config preflight --path ~/.config/openshell/gateway.toml +systemctl --user restart openshell-gateway +``` diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 5e9789f4fe..0670e43f6f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -42,11 +42,11 @@ with the exact exit code. Driver and supervisor failures remain `Error`. ## Configure a Compute Driver -Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set `compute_drivers` in the gateway TOML file: +Configure the compute driver on the gateway. Current releases accept one driver per gateway. Set the singular `compute_driver` key in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["docker"] +compute_driver = "docker" ``` Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. @@ -54,15 +54,17 @@ The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_driver` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_driver = "vm"` or set `OPENSHELL_COMPUTE_DRIVER=vm` in the launch environment. + +`compute_driver` accepts exactly one scalar driver name. The legacy `compute_drivers` list is rejected by schema version 2. Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | +| `compute_driver = ""` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | -Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Set driver-specific values such as sandbox images, callback endpoints, network names, and VM sizing in the gateway TOML file. A TLS-enabled gateway-managed Docker, Podman, or VM driver requires a complete `guest_tls_ca`, `guest_tls_cert`, and `guest_tls_key` bundle in `[openshell.gateway]`; package-managed local TLS supplies it automatically. Driver tables reject those gateway-owned fields. Kubernetes projects guest TLS through a Secret instead. See the [Gateway Configuration File](./gateway-config) reference for the full schema and migration steps. Extension drivers use the same `compute_driver.proto` gRPC surface as the managed VM driver. For an out-of-tree driver, choose a driver name and point @@ -70,7 +72,7 @@ the gateway at the Unix socket the operator has already provisioned: ```toml [openshell.gateway] -compute_drivers = ["kyma"] +compute_driver = "kyma" [openshell.drivers.kyma] socket_path = "/run/openshell/kyma.sock" @@ -81,8 +83,8 @@ socket path. The endpoint replaces normal driver construction for that name, including canonical built-in names: ```shell -openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock -openshell-gateway --drivers docker --compute-driver-socket /run/openshell/docker.sock +openshell-gateway --compute-driver kyma --compute-driver-socket /run/openshell/kyma.sock +openshell-gateway --compute-driver docker --compute-driver-socket /run/openshell/docker.sock ``` The gateway connects to the operator-provided endpoint; it does not provision @@ -175,7 +177,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_driver = "docker"` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `sandbox_label`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace @@ -253,7 +255,7 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_driver = "podman"` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `ssh_socket_path`, and `sandbox_pids_limit` in `[openshell.drivers.podman]`. Podman sandboxes default to a 45-second graceful stop window before Podman escalates from `SIGTERM` to `SIGKILL`. Set `stop_timeout_secs` in gateway config, or `OPENSHELL_STOP_TIMEOUT` for the standalone driver, when a local runtime needs a different teardown window. @@ -267,6 +269,12 @@ stopped sandboxes alone. For proxy-required networks, the Podman driver also accepts the corporate egress proxy keys `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, and `proxy_connect_by_hostname`. The supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior. +Podman preserves its runtime-selected AppArmor profile when +`app_armor_profile` is omitted. Set `Unconfined` explicitly only when the +supervisor's mount setup requires it. Explicit `RuntimeDefault` and +`Localhost/` selections fail startup when Podman reports that AppArmor +is unavailable. + On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. Direct local callbacks from rootless Podman require Podman to report the pasta network helper. Slirp4netns, other helpers, and Podman versions that do not report their helper require an explicitly remote `grpc_endpoint`; otherwise the gateway fails startup rather than leaving sandbox callbacks unreachable. Rootful Podman continues to use the configured network's bridge gateway address. ### Podman Driver Config Mounts @@ -347,16 +355,16 @@ For maintainer-level implementation details, refer to the [VM driver README](htt The VM driver is opt-in. Release packages can install `openshell-driver-vm`, but the gateway does not select it unless you configure the driver explicitly. -Enable VM by setting `compute_drivers = ["vm"]` in the gateway TOML file: +Enable VM by setting `compute_driver = "vm"` in the gateway TOML file: ```toml [openshell.gateway] -compute_drivers = ["vm"] +compute_driver = "vm" ``` -For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. +For a launch-time override, set `OPENSHELL_COMPUTE_DRIVER=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, and `krun_log_level` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. @@ -401,23 +409,24 @@ owner references or use the sandbox ServiceAccount. The operator namespace allowlist is a trust grant, not a tenant isolation mechanism. -Helm deployments set Kubernetes driver values through the chart. +Helm deployments set Kubernetes driver values through the chart. Canonical TOML places `namespace`, `service_account_name`, and `enable_user_namespaces` in `[openshell.drivers.kubernetes]`; schema version 2 rejects their historical `[openshell.gateway]` locations. For maintainer-level implementation details, refer to the [Kubernetes driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-kubernetes/README.md). | Gateway configuration | Helm value | Description | |---|---|---| -| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. | +| `compute_driver = "kubernetes"` | Not applicable | Select the Kubernetes compute driver. | | `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. | -| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the Kubernetes driver's TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. | +| `[openshell.drivers.kubernetes].enable_user_namespaces` | `server.enableUserNamespaces` | Enable Kubernetes user namespaces for sandbox pods. | | `default_image` | `server.sandboxImage` | Set the default sandbox image. | -| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. | +| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the canonical sandbox pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image-pull Secrets to sandbox pods. Managed mode copies these explicitly named Secrets from the configured source namespace into each workspace namespace. In shared and operator modes, the Secrets must already exist in the sandbox namespace. | | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | -| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | +| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. Raw TOML and the standalone Kubernetes driver require an explicit endpoint because the sandbox namespace does not identify the gateway Service. Helm derives it from the release's gateway Service when the value is empty. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | -| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | +| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the canonical supervisor pull policy: `always`, `if_not_present`, or `never`. `newer` is Podman-only. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | @@ -608,7 +617,7 @@ The resolved UID/GID appear in: ### VM Driver -The VM driver injects the sandbox UID into the rootfs guest's `/etc/passwd`, `/etc/group`, and `/etc/gshadow` during rootfs preparation. Default UID is `10001`; configure `sandbox_uid` in `[openshell.drivers.vm]` to use a different value. +The VM driver preserves an image-provided `sandbox` account when `sandbox_uid` and `sandbox_gid` are omitted. Images without that account use UID/GID `1000`. Explicit values in `[openshell.drivers.vm]` override the image account. Persisted overlays retain the UID/GID recorded when they were created. An unmarked overlay recovers identity from concrete overlay or prepared-image state, an explicit override, or the current image; the driver never assigns legacy `10001:10001` without persisted evidence. ### Custom Images diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 1c541f8f8e..53a68364e5 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -71,7 +71,7 @@ This provides defense-in-depth: even if a container escape vulnerability exists, | Aspect | Detail | |---|---| -| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in the gateway config to enable cluster-wide. | +| Default | Disabled. Set `server.enableUserNamespaces: true` in Helm values or `enable_user_namespaces = true` in `[openshell.drivers.kubernetes]` to enable cluster-wide. | | What you can change | Enable cluster-wide through Helm or gateway config. Override per-sandbox through the `user_namespaces` field on `SandboxTemplate` in the API. | | Prerequisites | Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), a container runtime that supports user namespaces (containerd 2.0+, CRI-O 1.25+), and Linux 5.12+ for ID-mapped mounts. | | Risk if enabled with GPU | NVIDIA device plugin compatibility with user namespaces is unverified. OpenShell logs a warning when both GPU and user namespaces are active on the same sandbox. | diff --git a/e2e/configs/gateway/docker.toml b/e2e/configs/gateway/docker.toml index 59baed1d7d..63e587ef62 100644 --- a/e2e/configs/gateway/docker.toml +++ b/e2e/configs/gateway/docker.toml @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["docker"] +compute_driver = "docker" disable_tls = true [openshell.gateway.auth] @@ -18,10 +18,10 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" -sandbox_namespace = "openshell-e2e" +image_pull_policy = "if_not_present" +sandbox_label = "openshell-e2e" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..6eff3b5615 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -2,12 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 [openshell] -version = 1 +version = 2 [openshell.gateway] bind_address = "127.0.0.1:8080" log_level = "info" -compute_drivers = ["podman"] +compute_driver = "podman" disable_tls = true [openshell.gateway.auth] @@ -18,11 +18,13 @@ signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" kid_path = ".cache/openshell-e2e/gateway-jwt/kid" gateway_id = "openshell-e2e" -ttl_secs = 0 [openshell.drivers.podman] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" +ssh_socket_path = "/run/openshell/ssh.sock" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +app_armor_profile = "Unconfined" diff --git a/e2e/configs/gateway/schema-v2-capability-parity.toml b/e2e/configs/gateway/schema-v2-capability-parity.toml new file mode 100644 index 0000000000..3927f81df1 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-capability-parity.toml @@ -0,0 +1,400 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Inventory for the schema-v2 live capability-parity campaign. Every entry is a +# test case contract, not a result. `status = "not_run"` deliberately means no +# live assertion has been made. Keep the oracle observable and put the runtime +# prerequisites in `required_environment`; this lets later waves select a lane +# without rediscovering the migration's intended behavior. +manifest_version = 1 +baseline_ref = "origin/main" +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +baseline_schema_version = 1 +candidate_ref = "HEAD" +candidate_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" +candidate_schema_version = 2 + +# Allowed lane names: deterministic, e2e-docker, e2e-podman, e2e-kubernetes, +# e2e-vm, windows-mxc, extension-driver, auth-oidc, observability, packaging. +# Allowed statuses: not_run, blocked, planned. A live pass is intentionally not +# a status in this first-wave manifest. + +[[capabilities]] +id = "configuration-source-precedence" +topics = ["configuration_producers"] +origin_main_access_paths = ["--config", "OPENSHELL_GATEWAY_CONFIG", "CLI > OPENSHELL_* > [openshell.gateway] > defaults"] +schema_v2_access_paths = ["--config", "OPENSHELL_GATEWAY_CONFIG", "CLI > OPENSHELL_* > [openshell.gateway] > defaults"] +behavioral_oracle = "A CLI or environment value wins over the corresponding file value; an unset value uses the file value." +required_environment = "none; parse and startup-config construction only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "schema-version-and-strict-layout" +topics = ["configuration_producers"] +origin_main_access_paths = ["[openshell] version = 1", "[openshell.gateway] inherited driver defaults"] +schema_v2_access_paths = ["[openshell] version = 2", "[openshell.gateway] gateway-only fields", "[openshell.drivers.] driver-owned fields"] +behavioral_oracle = "Missing, v1, future, unknown gateway, misplaced selected-driver, unknown selected-driver, and non-table driver values fail before runtime construction; unselected driver tables are validated when selected." +required_environment = "none; TOML parser only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "gateway-identity-and-logging" +topics = ["configuration_producers", "observability"] +origin_main_access_paths = ["--name / OPENSHELL_GATEWAY_NAME", "--log-level / OPENSHELL_LOG_LEVEL", "[openshell.gateway].name", "[openshell.gateway].log_level"] +schema_v2_access_paths = ["[openshell.gateway].name", "[openshell.gateway].log_level", "same CLI and environment variables"] +behavioral_oracle = "The configured name and log filter are retained after v2 file merge and identify emitted gateway telemetry." +required_environment = "gateway process with captured logs" +test_lane = "observability" +status = "not_run" + +[[capabilities]] +id = "primary-health-and-metrics-listeners" +topics = ["listeners"] +origin_main_access_paths = ["--bind-address / OPENSHELL_BIND_ADDRESS", "--port / OPENSHELL_SERVER_PORT", "--health-port / OPENSHELL_HEALTH_PORT", "--metrics-port / OPENSHELL_METRICS_PORT", "[openshell.gateway].{bind_address,health_bind_address,metrics_bind_address}"] +schema_v2_access_paths = ["[openshell.gateway].bind_address", "[openshell.gateway].health_bind_address", "[openshell.gateway].metrics_bind_address", "same CLI and environment variables"] +behavioral_oracle = "The primary multiplexed endpoint and optional health/metrics endpoints bind their configured addresses; a zero auxiliary port disables only that auxiliary listener." +required_environment = "loopback TCP ports" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "database-url-and-persistence-backends" +topics = ["database"] +origin_main_access_paths = ["--db-url / OPENSHELL_DB_URL", "[openshell.gateway].database_url (rejected)"] +schema_v2_access_paths = ["--db-url / OPENSHELL_DB_URL only", "[openshell.gateway].database_url (rejected)"] +behavioral_oracle = "A SQLite or Postgres URL supplied outside TOML opens the store; a URL embedded in TOML is rejected without exposing credentials." +required_environment = "SQLite temporary directory; Postgres service for backend variant" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "ssh-rate-limit-and-policy-posture" +topics = ["listeners"] +origin_main_access_paths = ["[openshell.gateway].ssh_session_ttl_secs", "[openshell.gateway].grpc_rate_limit_{requests,window_seconds}", "[openshell.gateway].policy_validation_failure_mode"] +schema_v2_access_paths = ["same [openshell.gateway] fields"] +behavioral_oracle = "SSH TTL, paired rate-limit behavior, and fail_closed versus retain_last_valid policy posture survive migration with their documented validation rules." +required_environment = "gateway with a controllable clock and policy fixture" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "sandbox-service-routing" +topics = ["listeners"] +origin_main_access_paths = ["--server-san / OPENSHELL_SERVER_SAN", "--enable-loopback-service-http / OPENSHELL_ENABLE_LOOPBACK_SERVICE_HTTP", "[openshell.gateway].{server_sans,enable_loopback_service_http}"] +schema_v2_access_paths = ["[openshell.gateway].server_sans", "[openshell.gateway].enable_loopback_service_http", "same CLI and environment variables"] +behavioral_oracle = "Wildcard SAN routing and loopback-only plaintext sandbox-service HTTP remain available while gateway APIs stay unavailable on that plaintext route." +required_environment = "TLS certificate with sandbox wildcard SAN and loopback HTTP client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "gateway-listener-tls-and-sni" +topics = ["auth_tls_jwt", "listeners"] +origin_main_access_paths = ["--tls-cert / OPENSHELL_TLS_CERT", "--tls-key / OPENSHELL_TLS_KEY", "--tls-client-ca / OPENSHELL_TLS_CLIENT_CA", "[openshell.gateway.tls]"] +schema_v2_access_paths = ["[openshell.gateway.tls].{cert_path,key_path,client_ca_path,external_cert_path,external_key_path,external_server_names}", "same CLI and environment variables for primary bundle"] +behavioral_oracle = "The listener presents the primary or configured SNI certificate, derives client-certificate requirements from client CA and OIDC presence, rejects the unsupported require_client_auth file field, and rejects incomplete TLS bundles." +required_environment = "test CA, primary and external certificates, TLS client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "plaintext-listener-mode" +topics = ["auth_tls_jwt", "listeners"] +origin_main_access_paths = ["--disable-tls / OPENSHELL_DISABLE_TLS", "[openshell.gateway].disable_tls"] +schema_v2_access_paths = ["[openshell.gateway].disable_tls", "same CLI and environment variable"] +behavioral_oracle = "An explicit plaintext listener starts without a listener TLS table and is usable behind a trusted TLS-terminating proxy." +required_environment = "loopback HTTP client" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "guest-callback-tls-ownership" +topics = ["auth_tls_jwt", "docker", "podman", "vm"] +origin_main_access_paths = ["[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by local drivers", "driver-local guest_tls_* overrides"] +schema_v2_access_paths = ["[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key} injected only into selected Docker, Podman, or VM driver", "driver tables reject guest_tls_*"] +behavioral_oracle = "A TLS-enabled selected local driver receives one complete guest bundle; partial, misplaced, and plaintext-incompatible bundles fail closed." +required_environment = "test CA and client certificate bundle" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "oidc-bearer-authentication" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["--oidc-* / OPENSHELL_OIDC_*", "[openshell.gateway.oidc].{issuer,audience,jwks_ttl_secs,roles_claim,admin_role,user_role,scopes_claim}"] +schema_v2_access_paths = ["same [openshell.gateway.oidc] fields", "same CLI and environment variables"] +behavioral_oracle = "A token from the configured issuer is accepted only with the expected audience, role, and optional scope; invalid JWTs are rejected." +required_environment = "OIDC issuer with JWKS and signed test tokens" +test_lane = "auth-oidc" +status = "not_run" + +[[capabilities]] +id = "mtls-user-authentication" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["--enable-mtls-auth / OPENSHELL_ENABLE_MTLS_AUTH", "[openshell.gateway.mtls_auth].enabled"] +schema_v2_access_paths = ["[openshell.gateway.mtls_auth].enabled", "same CLI and environment variable"] +behavioral_oracle = "A verified local client certificate maps to a user principal only when mTLS user auth is enabled and no stronger auth policy replaces it." +required_environment = "local driver, test CA, client certificate" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "unsafe-unauthenticated-user-mode" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["[openshell.gateway.auth].allow_unauthenticated_users"] +schema_v2_access_paths = ["same [openshell.gateway.auth].allow_unauthenticated_users"] +behavioral_oracle = "The explicit trusted-development switch admits user requests without a credential while sandbox callbacks retain sandbox authentication." +required_environment = "isolated gateway with no public exposure" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "gateway-minted-sandbox-jwt" +topics = ["auth_tls_jwt"] +origin_main_access_paths = ["[openshell.gateway.gateway_jwt].{signing_key_path,public_key_path,kid_path,gateway_id,ttl_secs}"] +schema_v2_access_paths = ["same [openshell.gateway.gateway_jwt] fields"] +behavioral_oracle = "The gateway mints sandbox and extension tokens with the configured identity, key ID, audience, and positive or omitted TTL semantics; explicit zero is rejected." +required_environment = "Ed25519 keypair and sandbox callback fixture" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "otlp-observability" +topics = ["observability"] +origin_main_access_paths = ["[openshell.gateway.otlp].{endpoint,service_name}", "OTEL_* SDK tuning environment"] +schema_v2_access_paths = ["same [openshell.gateway.otlp] fields", "same OTEL_* tuning environment"] +behavioral_oracle = "A configured OTLP/gRPC collector receives gateway and in-tree driver traces with gateway name and selected-driver resource attributes; collector failure does not prevent serving." +required_environment = "loopback OTLP/gRPC collector" +test_lane = "observability" +status = "not_run" + +[[capabilities]] +id = "gateway-interceptor-registration" +topics = ["interceptors", "auth_tls_jwt"] +origin_main_access_paths = ["[[openshell.gateway.interceptors]] including endpoint, TLS CA, audience, ordering, failure, timeout, limits, binding policy, and bindings"] +schema_v2_access_paths = ["same [[openshell.gateway.interceptors]] table and [[openshell.gateway.interceptors.bindings]]"] +behavioral_oracle = "Startup validates Describe metadata and configured binding policy; an allowed interceptor observes or modifies only its selected non-secret unary RPC phases." +required_environment = "test interceptor gRPC service; optional private CA or Unix socket" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "supervisor-middleware-registration" +topics = ["middleware", "auth_tls_jwt"] +origin_main_access_paths = ["[[openshell.supervisor.middleware]] including endpoint, TLS CA, audience, payload limit, timeout, and insecure transport"] +schema_v2_access_paths = ["same [[openshell.supervisor.middleware]] table"] +behavioral_oracle = "Gateway startup validates middleware Describe and policy configuration, applies payload/time limits, and distributes only validated registration data to supervisors." +required_environment = "test supervisor middleware gRPC service and sandbox supervisor" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "provider-profile-sources" +topics = ["inference", "interceptors"] +origin_main_access_paths = ["[openshell.gateway].provider_profile_sources"] +schema_v2_access_paths = ["same [openshell.gateway].provider_profile_sources"] +behavioral_oracle = "Configured builtin, user, and interceptor sources form one ordered validated catalog; duplicate normalized profile IDs or invalid interceptor source selections fail closed." +required_environment = "provider records and profile-capable interceptor fixture" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "inference-control-plane-configuration" +topics = ["inference"] +origin_main_access_paths = ["OpenShell inference configuration RPCs and persisted gateway settings; not a startup TOML field"] +schema_v2_access_paths = ["unchanged OpenShell inference configuration RPCs and persisted gateway settings; not a startup TOML field"] +behavioral_oracle = "A provider and route configured through the control plane resolves the same effective inference bundle after a schema-v2 gateway starts." +required_environment = "gateway, provider fixture, and sandbox supervisor" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "credential-driver-selection-and-kek" +topics = ["credentials"] +origin_main_access_paths = ["[openshell.gateway].credential_drivers", "[openshell.gateway].default_credential_driver", "[openshell.gateway.credential_storage]"] +schema_v2_access_paths = ["same [openshell.gateway] fields"] +behavioral_oracle = "Omission selects encrypted database storage; an explicit empty or multi-driver selection fails, and KEK path/environment configuration preserves credential confidentiality." +required_environment = "temporary gateway database and KEK fixture" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "credential-driver-backend-tables" +topics = ["credentials", "external_drivers"] +origin_main_access_paths = ["[openshell.credential_drivers.kubernetes-secrets]", "[openshell.credential_drivers.vault]", "[openshell.credential_drivers.]"] +schema_v2_access_paths = ["same credential-driver tables, including in_tree or uds transport, socket_path, command, args, and startup_timeout_secs"] +behavioral_oracle = "Built-in and remote credential driver tables validate their transport contract and store/retrieve opaque credential material without inline secrets in TOML." +required_environment = "credential-driver mock over UDS; Kubernetes or Vault for backend variants" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "docker-image-and-callback-configuration" +topics = ["docker"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,host_gateway_ip} inherited by Docker", "[openshell.drivers.docker].{socket_path,default_image,image_pull_policy,sandbox_namespace,grpc_endpoint,supervisor_bin,supervisor_image,network_name,host_gateway_ip,ssh_socket_path}"] +schema_v2_access_paths = ["[openshell.drivers.docker].{socket_path,default_image,image_pull_policy,sandbox_label,grpc_endpoint,supervisor_bin,supervisor_image,network_name,host_gateway_ip,ssh_socket_path}"] +behavioral_oracle = "Docker starts a sandbox using its driver table, maps the canonical sandbox label, honors image policy, and derives or honors a callback endpoint." +required_environment = "Linux Docker daemon, bridge network, sandbox and supervisor images" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "docker-security-and-provider-configuration" +topics = ["docker", "credentials"] +origin_main_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts}", "no origin/main Docker equivalent for proxy, SPIFFE, or AppArmor configuration"] +schema_v2_access_paths = ["[openshell.drivers.docker].{sandbox_pids_limit,enable_bind_mounts,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_socket,app_armor_profile}"] +behavioral_oracle = "Docker preserves PID and bind-mount behavior while schema v2 adds fail-closed proxy, SPIFFE, and AppArmor configuration whose material is mounted only into the supervisor." +required_environment = "Linux Docker daemon, proxy fixture, optional SPIFFE Unix socket and AppArmor support" +test_lane = "e2e-docker" +status = "not_run" + +[[capabilities]] +id = "podman-image-and-callback-configuration" +topics = ["podman"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,host_gateway_ip} inherited by Podman", "[openshell.drivers.podman].{socket_path,default_image,image_pull_policy,grpc_endpoint,gateway_port,network_name,host_gateway_ip,stop_timeout_secs,supervisor_image}"] +schema_v2_access_paths = ["same [openshell.drivers.podman] fields; gateway_port is runtime-derived"] +behavioral_oracle = "Podman starts a sandbox with its selected socket, image and policy, derives the callback route, and applies stop timeout without v1 gateway inheritance." +required_environment = "Podman service socket, user bridge network, sandbox and supervisor images" +test_lane = "e2e-podman" +status = "not_run" + +[[capabilities]] +id = "podman-runtime-security-and-health" +topics = ["podman", "credentials"] +origin_main_access_paths = ["[openshell.drivers.podman].{sandbox_ssh_socket_path,sandbox_pids_limit,enable_bind_mounts,provider_spiffe_workload_api_socket,app_armor_profile,health_check_interval_secs,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,proxy_ca_bundle,userns,uidmap,gidmap}"] +schema_v2_access_paths = ["[openshell.drivers.podman].{ssh_socket_path,sandbox_pids_limit,enable_bind_mounts,provider_spiffe_workload_api_socket,app_armor_profile,health_check_interval_secs,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,proxy_ca_bundle,userns,uidmap,gidmap}"] +behavioral_oracle = "Podman uses the renamed SSH path and validates PID, health, user namespace mappings, AppArmor, proxy, CA, and SPIFFE contracts before a sandbox can run." +required_environment = "Podman service socket, optional proxy/CA/SPIFFE fixture, rootless and rootful variants" +test_lane = "e2e-podman" +status = "not_run" + +[[capabilities]] +id = "kubernetes-core-placement-and-images" +topics = ["kubernetes"] +origin_main_access_paths = ["[openshell.gateway].{default_image,supervisor_image,client_tls_secret_name,service_account_name,host_gateway_ip,enable_user_namespaces,sa_token_ttl_secs} inherited by Kubernetes", "[openshell.drivers.kubernetes].{namespace,default_image,image_pull_policy,image_pull_secrets,service_account_name,supervisor_image,supervisor_image_pull_policy,grpc_endpoint,ssh_socket_path,client_tls_secret_name,host_gateway_ip,enable_user_namespaces,sa_token_ttl_secs}"] +schema_v2_access_paths = ["same fields exclusively in [openshell.drivers.kubernetes]"] +behavioral_oracle = "The Kubernetes driver creates a sandbox Pod with driver-owned namespace, service account, image, pull policy, callback endpoint, SSH socket, TLS Secret, and token TTL." +required_environment = "Kubernetes cluster, namespace, service account, image pull secret, and client TLS Secret" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-workspace-isolation" +topics = ["kubernetes"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{workspace_mode,gateway_id,operator_namespace_label,operator_namespace_file,workspace_default_storage_size,workspace_storage_class,default_runtime_class_name}"] +schema_v2_access_paths = ["same [openshell.drivers.kubernetes] fields"] +behavioral_oracle = "Shared, managed, and operator workspace modes select or create the expected namespace and workspace storage with configured discovery, class, and runtime class." +required_environment = "Kubernetes cluster with namespaces, PVC provisioning, and optional RuntimeClass" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-supervisor-topology" +topics = ["kubernetes", "middleware"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{supervisor_sideload_method,topology}", "[openshell.drivers.kubernetes.sidecar].{proxy_uid,process_binary_aware_network_policy}", "[openshell.drivers.kubernetes.managed_ssh_ingress].{enabled,gateway_namespace,gateway_pod_selector}"] +schema_v2_access_paths = ["same Kubernetes driver subtables and fields"] +behavioral_oracle = "The rendered sandbox Pod matches the selected combined or sidecar supervisor topology, sideload method, ingress selector, and sidecar security posture." +required_environment = "Kubernetes cluster supporting selected image-volume or init-container method" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "kubernetes-egress-spiffe-and-security" +topics = ["kubernetes", "credentials"] +origin_main_access_paths = ["[openshell.drivers.kubernetes].{https_proxy,no_proxy,proxy_auth_secret_name,proxy_auth_secret_key,proxy_auth_allow_insecure,proxy_connect_by_hostname,app_armor_profile,provider_spiffe_workload_api_socket_path,sandbox_uid,sandbox_gid}"] +schema_v2_access_paths = ["same [openshell.drivers.kubernetes] fields"] +behavioral_oracle = "Kubernetes validates proxy/Secret and sidecar-topology relationships, projects SPIFFE only from an allowed path, and applies valid non-root identity and AppArmor settings." +required_environment = "Kubernetes cluster, proxy credential Secret, optional SPIFFE socket, AppArmor-capable node" +test_lane = "e2e-kubernetes" +status = "not_run" + +[[capabilities]] +id = "vm-launch-and-resource-configuration" +topics = ["vm"] +origin_main_access_paths = ["[openshell.gateway].{default_image,guest_tls_ca,guest_tls_cert,guest_tls_key} inherited by VM", "[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "standalone openshell-driver-vm --openshell-endpoint / OPENSHELL_GRPC_ENDPOINT"] +schema_v2_access_paths = ["[openshell.drivers.vm].{grpc_endpoint,state_dir,driver_dir,default_image,bootstrap_image,krun_log_level,vcpus,mem_mib,overlay_disk_mib,sandbox_uid,sandbox_gid}", "[openshell.gateway].{guest_tls_ca,guest_tls_cert,guest_tls_key}", "standalone openshell-driver-vm --grpc-endpoint / OPENSHELL_GRPC_ENDPOINT"] +behavioral_oracle = "The gateway finds and launches the VM driver from driver_dir, forwards the TOML grpc_endpoint through the schema-appropriate standalone-driver flag, and launches a guest with the selected state, images, resources, and identity." +required_environment = "Linux libkrun/KVM host, VM driver binary, and OCI image" +test_lane = "e2e-vm" +status = "not_run" + +[[capabilities]] +id = "vm-guest-security-and-spiffe" +topics = ["vm", "credentials"] +origin_main_access_paths = ["[openshell.drivers.vm].{sandbox_uid,sandbox_gid}"] +schema_v2_access_paths = ["[openshell.drivers.vm].{sandbox_uid,sandbox_gid,https_proxy,no_proxy,proxy_auth_file,proxy_auth_allow_insecure,proxy_connect_by_hostname,provider_spiffe_workload_api_tcp_endpoint,provider_spiffe_allow_guest_tcp}", "gateway-owned guest_tls_* bundle"] +behavioral_oracle = "VM validates non-root guest ownership, callback TLS, proxy safety, and requires an explicit opt-in before exposing a guest-reachable SPIFFE TCP endpoint." +required_environment = "Linux libkrun/KVM host, TLS and optional proxy/SPIFFE TCP fixture" +test_lane = "e2e-vm" +status = "not_run" + +[[capabilities]] +id = "mxc-windows-driver-configuration" +topics = ["mxc"] +origin_main_access_paths = ["[openshell.drivers.mxc].{wxc_exec_path,backend,pc_least_privilege,pc_capabilities,default_configuration_id,debug}"] +schema_v2_access_paths = ["same [openshell.drivers.mxc] fields"] +behavioral_oracle = "The Windows gateway selects MXC and passes the configured wxc-exec path, backend, AppContainer capabilities, isolation configuration, and debug flag to MXC." +required_environment = "Windows host with MXC/wxc-exec; mock fixture for deterministic smoke variant" +test_lane = "windows-mxc" +status = "not_run" + +[[capabilities]] +id = "external-compute-driver-socket" +topics = ["external_drivers"] +origin_main_access_paths = ["--drivers / --driver / OPENSHELL_DRIVERS plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.] remote socket table"] +schema_v2_access_paths = ["--compute-driver / OPENSHELL_COMPUTE_DRIVER plus --compute-driver-socket / OPENSHELL_COMPUTE_DRIVER_SOCKET", "[openshell.drivers.].socket_path"] +behavioral_oracle = "A selected non-reserved external driver connects through its configured Unix socket; legacy plural selector flags are rejected, while one OPENSHELL_DRIVERS environment value remains a deprecated upgrade alias when it does not conflict with canonical selection." +required_environment = "external compute-driver gRPC fixture over Unix socket" +test_lane = "extension-driver" +status = "not_run" + +[[capabilities]] +id = "helm-configuration-producer" +topics = ["configuration_producers", "kubernetes"] +origin_main_access_paths = ["Helm rendered schema-v1 gateway TOML and environment-backed overrides"] +schema_v2_access_paths = ["deploy/helm/openshell/templates/gateway-config.yaml renders [openshell] version = 2 and Kubernetes driver table", "Secret-backed OPENSHELL_DB_URL and certificate inputs"] +behavioral_oracle = "helm template renders schema-v2 TOML, derives the Kubernetes callback endpoint, and keeps database and secret material outside the ConfigMap." +required_environment = "Helm and chart test plugin; Kubernetes cluster only for live install variant" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "local-launch-script-producers" +topics = ["configuration_producers", "docker", "podman", "vm", "kubernetes"] +origin_main_access_paths = ["tasks/scripts/gateway{,-docker,-podman,-vm}.sh generated schema-v1 TOML"] +schema_v2_access_paths = ["tasks/scripts/gateway{,-docker,-podman,-vm}.sh generated schema-v2 TOML with per-driver tables"] +behavioral_oracle = "Each local launch script emits parseable v2 TOML with a scalar driver and its canonical driver-owned values before it executes the gateway binary." +required_environment = "shell, uv, and fake gateway executable; no runtime daemon required" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "e2e-fixture-producers" +topics = ["configuration_producers", "docker", "podman"] +origin_main_access_paths = ["e2e/configs/gateway/docker.toml and podman.toml schema-v1 fixtures"] +schema_v2_access_paths = ["e2e/configs/gateway/docker.toml and podman.toml schema-v2 fixtures"] +behavioral_oracle = "Docker and Podman E2E fixtures parse as v2, select one scalar driver, and contain canonical renamed policy and callback fields." +required_environment = "none; TOML parser only" +test_lane = "deterministic" +status = "not_run" + +[[capabilities]] +id = "rpm-schema-upgrade" +topics = ["packaging_upgrades", "configuration_producers"] +origin_main_access_paths = ["deploy/rpm/gateway.toml.default version 1", "systemd user config seeded from package default"] +schema_v2_access_paths = ["deploy/rpm/gateway.toml.default version 2", "deploy/rpm/migrate-gateway-config.sh exact-v1 replacement"] +behavioral_oracle = "RPM first start seeds the v2 default and upgrade replaces only an exact package-generated v1 file, preserving edited files, modes, and symlink safety." +required_environment = "temporary filesystem and shell; RPM install test host for package variant" +test_lane = "packaging" +status = "not_run" + +[[capabilities]] +id = "homebrew-debian-and-snap-upgrades" +topics = ["packaging_upgrades"] +origin_main_access_paths = ["Homebrew prefix config and migration helper", "Debian XDG user-service config", "Snap $SNAP_COMMON/gateway.toml"] +schema_v2_access_paths = ["package-managed schema-v2 defaults and documented exact-v1/manual migration paths"] +behavioral_oracle = "Each package resolves its documented config location, starts from v2 defaults, and never overwrites operator-edited legacy configuration during upgrade." +required_environment = "macOS Homebrew, Debian/Ubuntu user-service, and Snap test environments" +test_lane = "packaging" +status = "not_run" diff --git a/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json b/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json new file mode 100644 index 0000000000..783bb00932 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-compute-boundary-comparison.json @@ -0,0 +1,376 @@ +{ + "manifest_version": 2, + "baseline_commit": "74960ebfaeec4673885089ed995fad902459749f", + "candidate_commit": "4a39da510e4d278a24dd60291149519c9a570b46", + "lane": "local-linux-x86_64-rootless-podman-5.8.2", + "retained_evidence_bundles": { + "in_tree": "target/parity/step10-intree-4a39da51", + "external_uds": "target/parity/step10-external-4a39da51" + }, + "oracle": { + "status": true, + "create": true, + "ready": true, + "list_visible": true, + "callback_exec_exact_marker": true, + "delete": true, + "list_empty": true + }, + "in_tree": { + "baseline": { + "schema_version": 1, + "source_sha": "74960ebfaeec4673885089ed995fad902459749f", + "gateway_profile": "in-tree", + "gateway_cargo_features": "default", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "not_applicable", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "264cf3d809bd1d54633bce9251ec1a5540b567b8150640091df85bdfbe61797a", + "cli": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "supervisor": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 1, + "gateway_port": 32893, + "external_compute_driver": false, + "compute_driver_transport": "in_tree", + "external_driver_pull_policy": "missing", + "supervisor_image": "localhost/openshell/supervisor:parity-baseline-74960ebfaeec", + "supervisor_image_id": "b5bcaae227a1d6ea90fd146fc2904d4166725c8eb1b9590bdfd85b14c7b9caef", + "supervisor_image_digest": "sha256:0f11b6ca65ff33c99d03d6ebd8af239ac9edc148f849201a692a06d722853030", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:0f11b6ca65ff33c99d03d6ebd8af239ac9edc148f849201a692a06d722853030", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "264cf3d809bd1d54633bce9251ec1a5540b567b8150640091df85bdfbe61797a", + "cli_sha256_before_execution": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance_sha256_before_execution": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "external_driver_sha256_before_execution": "", + "supervisor_sha256_before_execution": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": null, + "external_driver_host_gateway_ip": null, + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": null + }, + "raw_evidence_sha256": { + "baseline.json": "d7e7f18607d1eab7fab65b1643febe46499aeaaa00c100b66f92b11101d798a1", + "baseline.launch.json": "a56fe7df2c454137cebcea17bc022adcb9b00f3a2e7403611f61fede62a9b329", + "baseline.log": "e44a39cf19296908e99c58b661bd1c162f0aedd2fafb9c70a1a875d91e712cbe", + "baseline.gateway.toml": "9d11c7c8455308c21354f2a41b546068f06e4605f9127e0156467c321bac550c", + "baseline.exec.stdout": "f3a45a1114b92419a9c4c91364d584de4161ec171a1a9a73dae5b4daca89f8a9" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "candidate": { + "schema_version": 2, + "source_sha": "4a39da510e4d278a24dd60291149519c9a570b46", + "gateway_profile": "in-tree", + "gateway_cargo_features": "default", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "not_applicable", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "2bd42b145f0438f48a579b010537f501e036035b0e22a4ff70e37db733a6e6ff", + "cli": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "supervisor": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 2, + "gateway_port": 55987, + "external_compute_driver": false, + "compute_driver_transport": "in_tree", + "external_driver_pull_policy": "if_not_present", + "supervisor_image": "localhost/openshell/supervisor:parity-candidate-4a39da510e4d", + "supervisor_image_id": "489ba15df40c47957b400e54633c250b4322dc37f2798a59717b57c75421330a", + "supervisor_image_digest": "sha256:6de7479f138e88da131741e64c170aa75ae14b4e7b9b49de2d11f51af7d1b6d7", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:6de7479f138e88da131741e64c170aa75ae14b4e7b9b49de2d11f51af7d1b6d7", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "2bd42b145f0438f48a579b010537f501e036035b0e22a4ff70e37db733a6e6ff", + "cli_sha256_before_execution": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance_sha256_before_execution": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "external_driver_sha256_before_execution": "", + "supervisor_sha256_before_execution": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": null, + "external_driver_host_gateway_ip": null, + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": null + }, + "raw_evidence_sha256": { + "candidate.json": "ed7d6334cccb40488291e35e9b7ad4fb45dd47b55b7e27925384d22907bedefd", + "candidate.launch.json": "384ee89066973cacdb43160a39542096bc3d02da051ac0417fefc0be41c2f619", + "candidate.log": "719aaedc019492ccdc2f1e003af6b12af5bf6d2507fc2573135fca529a9f398f", + "candidate.gateway.toml": "504bbea27b56cc630884ba3eb17eb518f7c55cb7886b10ff353cc777e3d08362", + "candidate.exec.stdout": "d903d8a1f84caca8daa2a5a047c3fe90e2501690fba7c0b62e14d3892b5aadc3" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "comparison_sha256": "e97312770ed8cbba6325b901885d083c7f82ef979b5b89e4e0ed2bd32478d9fb", + "classification": "pass", + "parity": true, + "accepted": true + }, + "external_uds": { + "baseline": { + "schema_version": 1, + "source_sha": "74960ebfaeec4673885089ed995fad902459749f", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "built_by_harness", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "5cc2f700d93dde5dd09060d9c9190ae44a99440b1399530a2b23941ec4e88f85", + "cli": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "supervisor": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external-driver": "d976be0580ab5a2e006ca9e1bc1556b84fa810481f6e1c2132793f8e38c7194c", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 1, + "gateway_port": 50871, + "external_compute_driver": true, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": "missing", + "supervisor_image": "localhost/openshell/supervisor:parity-baseline-74960ebfaeec", + "supervisor_image_id": "b2e8f8dae82296a54e7500beb0b2b55d1f8d4ac4afe6ed8de5fba4b3b65ba748", + "supervisor_image_digest": "sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "5cc2f700d93dde5dd09060d9c9190ae44a99440b1399530a2b23941ec4e88f85", + "cli_sha256_before_execution": "3ad0ec143bf6850af780bf643c303242956d6ef1066d4a9372bc62fef33ada20", + "conformance_sha256_before_execution": "d669f960333f9bcd777f4fac92e5208fa66d1b51578838e5aba9b1f58bc8dc6e", + "external_driver_sha256_before_execution": "d976be0580ab5a2e006ca9e1bc1556b84fa810481f6e1c2132793f8e38c7194c", + "supervisor_sha256_before_execution": "ae2e854936a15a5b952187fd4488a746a8a6e31e9f919967e3ead7a80d4b1077", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": "https://host.containers.internal:50871", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": "/tmp/openshell-e2e-podman.H8oMaO/compute-driver.sock", + "OPENSHELL_PODMAN_SOCKET": "/tmp/openshell-e2e-podman.H8oMaO/podman/podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": "missing", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:50871", + "OPENSHELL_GATEWAY_PORT": 50871, + "OPENSHELL_NETWORK_NAME": "e2e-podman-659291-50871", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": "localhost/openshell/supervisor@sha256:b576159e1c7ca3258b9428411977a4fa7eddf2d46aa1a1b2238de7a0081c7e60", + "OPENSHELL_PODMAN_TLS_CA": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/ca.crt", + "sha256": "1a77771ebd83be7ed988263ea4a8e42d6b2af94c46861693911c4d5bff6a3fdd" + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/client/tls.crt", + "sha256": "0e9ad610227c223ebca348599a113e07f46ab0c6ff6f4ea5598e4c15673e25fd" + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": "/tmp/openshell-e2e-podman.H8oMaO/pki/client/tls.key", + "sha256": "d5bbf26f73a8fdfe60e16783bb99275bf35c617b1f229b7c48afb0a3edbacbc0" + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": true + } + }, + "raw_evidence_sha256": { + "baseline.json": "2b48dd376fd02d46c93d6ae0831d3a81d7337893794ca97c6559676e1b8302de", + "baseline.launch.json": "babe872dd643f024da5e4d0697a3a946ba21432d16e7b7abb2e343e82459ec79", + "baseline.log": "1718593d49cd26f76b7ce1d3be5cfca7c70fae7c44976aa1485796b52048ab41", + "baseline.gateway.toml": "8d4f2579c58ba140e95dc1151216cf9962ac211272ce26d2f811b6f3a1a659d0", + "baseline.exec.stdout": "f740ac1e211850739407334863f73311e7ff508bb4e507b7d7fcaf8d8dc2de32", + "baseline.driver.log": "877ab38e85532d3b772ec080adc727ef517203b1b553e75c9b6f29ad06e3374a" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "candidate": { + "schema_version": 2, + "source_sha": "4a39da510e4d278a24dd60291149519c9a570b46", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "artifact_origins": { + "gateway": "built_by_harness", + "cli": "built_by_harness", + "conformance": "built_by_harness", + "external_driver": "built_by_harness", + "supervisor": "built_by_harness" + }, + "artifact_sha256": { + "gateway": "fdefc047c1b675c311b1796db9f1b1a944456fc34b76547efc0352c6a75a7707", + "cli": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "supervisor": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor.Dockerfile": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli-trace-wrapper": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external-driver": "9accb17523a33e8fc4df93b3b3dec619f1f5ad6f5ff3f51abb2a8a5718278f8c", + "supervisor.packages.txt": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9" + }, + "launch_attestation": { + "schema_version": 2, + "gateway_port": 32901, + "external_compute_driver": true, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": "if_not_present", + "supervisor_image": "localhost/openshell/supervisor:parity-candidate-4a39da510e4d", + "supervisor_image_id": "9c7f51faec648947fe1515ec4bcbab52234e4e143a53bd1bf0a358eadd11440e", + "supervisor_image_digest": "sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "supervisor_runtime_image": "localhost/openshell/supervisor@sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "supervisor_base_image": "alpine:3.22", + "supervisor_base_image_id": "b66e0ce64844f5c6435b0c4bfd965558199ab0f53270846861c979cb1ac29365", + "supervisor_base_image_digest": "sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6", + "supervisor_base_runtime_image": "docker.io/library/alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce", + "supervisor_package_manifest_sha256": "6966057ffaf1ef0d4617d413dad53646bcd049a4b08f572796392fb0ffbb01b9", + "sandbox_image_request": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_image_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "sandbox_image_digest": "sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_runtime_image": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest", + "sandbox_client_image_alias_id": "65fa5d3d598a07d385ddbea41bf593be15af306337b76255b40705287cebcbbf", + "gateway_sha256_before_execution": "fdefc047c1b675c311b1796db9f1b1a944456fc34b76547efc0352c6a75a7707", + "cli_sha256_before_execution": "82827d9068f3e9c75681a804f8fb48274ecf8acb179a46f179fb9fbfe828f69a", + "conformance_sha256_before_execution": "cf6866bdd9755881bf8e2fe0c60a72b41037c6e31725fe43f3328b5c8435cc6d", + "external_driver_sha256_before_execution": "9accb17523a33e8fc4df93b3b3dec619f1f5ad6f5ff3f51abb2a8a5718278f8c", + "supervisor_sha256_before_execution": "38e1afd251898593619864557ed948ef76370ec0c4618850474a67d1d0508b40", + "supervisor_dockerfile_sha256_before_execution": "ec8b25f0a674c0c7ae123c978f6857436af987faea43051e67d3a8d4b52831cd", + "cli_trace_wrapper_sha256_before_execution": "0691d47d9eae7e9fe847a58994e170a480c0e61d1d9013193d6ca9c93476777f", + "external_driver_grpc_endpoint": "https://host.containers.internal:32901", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": null, + "external_driver_spiffe": false, + "external_driver_proxy": false, + "external_driver_app_armor": false, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": "/tmp/openshell-e2e-podman.M8xsgQ/compute-driver.sock", + "OPENSHELL_PODMAN_SOCKET": "/tmp/openshell-e2e-podman.M8xsgQ/podman/podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:c2a43bb0d765774e2790b3babfb20997bb2eac7b4bf4c6d7d8661e99817bf904", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": "if_not_present", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:32901", + "OPENSHELL_GATEWAY_PORT": 32901, + "OPENSHELL_NETWORK_NAME": "e2e-podman-660516-32901", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": "localhost/openshell/supervisor@sha256:b4cc939fcf3a95ee7cbfa1560acda5ab8c234b041a21d31ac958810e6b748798", + "OPENSHELL_PODMAN_TLS_CA": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/ca.crt", + "sha256": "90b268e741240a39838b7074b66ffa30576c053f5c8f5659e1e2e3c0e22809df" + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/client/tls.crt", + "sha256": "a89f555a5e53cf155f122bd2c8b58e0856019468cdf1cf80cafb1c832979f40f" + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": "/tmp/openshell-e2e-podman.M8xsgQ/pki/client/tls.key", + "sha256": "a53d8aaea929dec1b355a5e6ae45eb5001f9c3845d3d2cbbff9ca5a7e468be06" + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": true + } + }, + "raw_evidence_sha256": { + "candidate.json": "a68a9a3f78a8ba0fbf33c81e9960e705e554e82077fd4e0e8565a3bc8598790b", + "candidate.launch.json": "5be63290c313302ec2fa35cdd55ec641742d2d7009f7c2746842c42072425f1a", + "candidate.log": "d19a44064c8d90b078e4d34d711c599ae424834de433b355805105ae3f272cf2", + "candidate.gateway.toml": "862322125b3bc9438b87b35448e107d87850f4f3cd61c9c36f045560c963385e", + "candidate.exec.stdout": "fc1085830f6824809d3fdec6ecf518d54a7dde92ec4441e134f13abd5a6a6522", + "candidate.driver.log": "0b977d9c2b69a3e9665d457dd88fe231b3da7a55d74d4ace4c1a467e7cf716d1" + }, + "artifacts_verified": true, + "raw_output_verified": true, + "success": true + }, + "comparison_sha256": "6b30e24a1cf0f9b7d8b5a5de206cfbbfb01c19a7921d6a449900af8448abec4c", + "classification": "pass", + "parity": true, + "accepted": true + }, + "callback_listener": { + "in_tree_baseline_exec": true, + "in_tree_candidate_exec": true, + "external_baseline_exec": true, + "external_candidate_exec": true, + "classification": "pass" + }, + "classification": "pass", + "accepted": true, + "verification": { + "retained_artifact_hashes_recomputed": true, + "raw_lifecycle_output_inspected": true, + "authenticated_preflight_verified": true, + "exact_callback_exec_stdout_verified": true, + "external_driver_allowlist_verified": true, + "external_driver_logs_retained": true, + "external_uds_isolation_verified": true, + "digest_pinned_supervisor_runtime_verified": true, + "same_immutable_sandbox_verified": true, + "supervisor_dependency_provenance_matched": true + } +} diff --git a/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml b/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml new file mode 100644 index 0000000000..813da3bc61 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Step 11 dispositions for the cross-cutting gateway capabilities that are not +# exercised by the compute-driver waves. The paired deterministic preflight is +# regression evidence only: it does not replace a candidate-owned oracle that +# drives both gateway processes and observes the same live behavior. +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_commit = "363d8540830b2ea294d43198daa2b7a283a2face" +overall_status = "platform_blocked" + +[deterministic_preflight] +status = "pass" +baseline_attestation = "e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt" +baseline_attestation_sha256 = "df3c5e1dcb073eb8db0cb28377516523b331d44fc6f5a5fefab2e92273efaf51" +candidate_attestation = "e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt" +candidate_attestation_sha256 = "2562bff73eba8042199597e814b618b12b93994d15e4ae36e902347347d95fed" +commands = [ + "cargo test -q -p openshell-server --lib", + "cargo test -q -p openshell-gateway-interceptors", + "cargo test -q -p openshell-supervisor-middleware", + "cargo test -q -p openshell-otel-test-support", + "cargo test -q -p openshell-server --test multiplex_tls_integration", + "cargo test -q -p openshell-server --test edge_tunnel_auth", +] +evidence = [ + "The frozen baseline passed 1,455 openshell-server library tests with 8 ignored; the candidate passed 1,481 with 8 ignored.", + "Both revisions passed 49 gateway-interceptor tests, 85 supervisor-middleware tests, 5 multiplex TLS integration tests, and 5 edge-tunnel authentication tests.", + "The openshell-otel-test-support crate compiled successfully on both revisions but currently defines no tests; OTLP behavior is exercised by openshell-server unit tests and still requires a collector-backed live lane.", + "Both runs used checkout-local target directories with sccache disabled after a prior ENOSPC event corrupted the shared cache.", +] + +[[capabilities]] +id = "oidc-bearer-authentication" +status = "platform_blocked" +owner = "OpenShell auth and OIDC CI lane" +lane = "linux-oidc-paired-keycloak-jwks" +blocker = "The host has no isolated paired issuer fixture that presents equivalent signed tokens to both exact-source gateways. Existing Keycloak and OIDC suites are candidate-oriented and cannot establish schema-v1/schema-v2 audience, role, scope, expiry, signature, and key-rotation parity without a shared candidate-owned oracle." + +[[capabilities]] +id = "mtls-user-authentication" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-mtls-user-principal-paired" +blocker = "The paired TLS integration tests validate handshake policy but do not drive an authenticated user RPC through both gateways or prove certificate-to-principal mapping. A live lane must use isolated client PKI, equivalent schema files, and the same authorization oracle for enabled and disabled mTLS user authentication." + +[[capabilities]] +id = "unsafe-unauthenticated-user-mode" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-auth-chain-paired-isolated" +blocker = "No paired live fixture currently proves that the trusted-development switch admits user requests while sandbox callbacks continue to require sandbox credentials. Candidate-only extension examples use this switch for setup, but that is not an authentication-boundary parity test." + +[[capabilities]] +id = "gateway-minted-sandbox-jwt" +status = "platform_blocked" +owner = "OpenShell gateway JWT CI lane" +lane = "linux-gateway-jwt-claims-paired" +blocker = "Step 10 proved authenticated callback connectivity but did not capture and compare token claims. A controlled callback and extension fixture must verify issuer, gateway identity, key ID, audience, subject, token type, and expiry semantics for both gateways; explicit zero versus omitted TTL remains the ledgered gateway-jwt-zero-sentinel-removed intentional change." + +[[capabilities]] +id = "otlp-observability" +status = "platform_blocked" +owner = "OpenShell observability CI lane" +lane = "linux-otlp-grpc-collector-paired" +blocker = "The deterministic exporter tests passed, but no retained paired collector capture proves emitted gateway and in-tree-driver spans, gateway name, service name, selected-driver resource attributes, or continued serving after collector failure. A loopback OTLP/gRPC collector lane must observe those outcomes from both exact-source processes." + +[[capabilities]] +id = "gateway-interceptor-registration" +status = "platform_blocked" +owner = "OpenShell gateway interceptor CI lane" +lane = "linux-gateway-interceptor-paired" +blocker = "The interceptor library suite passed on both revisions, while the governance-interceptor smoke test remains candidate-only. A paired service must compare Describe validation, binding selection, unary request and response mutation, failure policy, timeout and size limits, and secret exclusion against both gateway schema variants." + +[[capabilities]] +id = "supervisor-middleware-registration" +status = "platform_blocked" +owner = "OpenShell supervisor middleware CI lane" +lane = "linux-supervisor-middleware-paired" +blocker = "The middleware library suite passed on both revisions, while the content-guard smoke test remains candidate-only. A paired gateway, sandbox, and middleware fixture must compare Describe negotiation, policy distribution, guarded and unguarded traffic, failure policy, timeout, and payload limits; the field-name normalization remains the ledgered middleware-payload-name-normalized intentional change." + +[[capabilities]] +id = "provider-profile-sources" +status = "platform_blocked" +owner = "OpenShell provider profile CI lane" +lane = "linux-provider-profiles-interceptor-paired" +blocker = "Server tests cover source construction and duplicate rejection, but no shared live interceptor catalog has been queried through both gateways. The assigned lane must compare builtin, user, and interceptor source ordering, normalized profile identities, duplicate rejection, discovery output, and failure handling with isolated persistent state." + +[[capabilities]] +id = "inference-control-plane-configuration" +status = "platform_blocked" +owner = "OpenShell inference E2E lane" +lane = "linux-inference-provider-routing-paired" +blocker = "Existing inference routing tests are candidate-oriented and this host has no assigned paired provider and model-service fixture. The lane must configure providers and routes through each gateway control plane, observe the effective supervisor bundle, and compare model discovery and an inference request without reusing database state." + +[[capabilities]] +id = "credential-driver-selection-and-kek" +status = "platform_blocked" +owner = "OpenShell credential security CI lane" +lane = "linux-credential-kek-paired" +blocker = "Both server suites passed deterministic selection, KEK, and credential tests, but they are revision-local tests rather than one candidate-owned live oracle. A paired lane must start isolated gateways with independent databases and KEKs, compare default encrypted storage and invalid selections, store and resolve opaque credentials, and verify that logs and configuration do not disclose secrets." + +[[capabilities]] +id = "credential-driver-backend-tables" +status = "platform_blocked" +owner = "OpenShell credential driver E2E lane" +lane = "kubernetes-vault-uds-credential-drivers-paired" +blocker = "The host has no assigned disposable Kubernetes Secrets or Vault backend and no retained paired UDS credential-driver execution. Existing backend E2E coverage is candidate-oriented. The assigned lane must compare in-tree and remote transport validation plus opaque store and retrieve behavior using isolated namespaces, Vault state, sockets, databases, and credentials." diff --git a/e2e/configs/gateway/schema-v2-intentional-changes.toml b/e2e/configs/gateway/schema-v2-intentional-changes.toml new file mode 100644 index 0000000000..d0707b8a54 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-intentional-changes.toml @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Explicit compatibility exceptions for the schema-v2 parity campaign. These +# entries document behavior that is not expected to match schema v1 byte for +# byte. They do not waive the requirement that the replacement behavior work. +ledger_version = 1 +issue = 2792 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +[[intentional_changes]] +id = "schema-version-cutover" +category = "schema_cutover" +origin_main_contract = "Gateway configuration uses schema version 1; omitted versions were accepted in some paths." +schema_v2_contract = "Every loaded gateway configuration must declare [openshell] version = 2; missing, v1, and future versions fail." +migration = "Set version = 2 and apply every field relocation, rename, and type migration in this ledger before restart." +rationale = "An explicit version boundary prevents a partially migrated security-sensitive configuration from being interpreted with mixed ownership rules." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "rpm-schema-upgrade", "homebrew-debian-and-snap-upgrades"] + +[[intentional_changes]] +id = "singular-compute-driver-selector" +category = "cardinality" +origin_main_contract = "Configuration spells the selector compute_drivers as a list even though runtime selection rejects more than one configured driver." +schema_v2_contract = "Configuration uses one optional scalar compute_driver; omission retains built-in auto-detection." +migration = "Replace compute_drivers = [\"name\"] with compute_driver = \"name\"." +rationale = "The schema now represents the existing one-driver runtime invariant instead of implying unsupported multi-driver operation." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "external-compute-driver-socket"] + +[[intentional_changes]] +id = "driver-table-exclusive-ownership" +category = "ownership" +origin_main_contract = "Selected driver tables inherit an allowlisted set of values from [openshell.gateway]." +schema_v2_contract = "Driver-specific values are read only from [openshell.drivers.]; gateway scope contains only gateway-owned values." +migration = "Move every driver option to the selected driver table instead of relying on gateway inheritance." +rationale = "Only one compute driver is active, so inheritance adds ambiguity and can map one gateway key to different backend meanings." +parity_disposition = "intentional_change" +validation_capability_ids = ["schema-version-and-strict-layout", "docker-image-and-callback-configuration", "podman-image-and-callback-configuration", "kubernetes-core-placement-and-images", "vm-launch-and-resource-configuration"] + +[[intentional_changes]] +id = "kubernetes-fields-relocated" +category = "relocation" +origin_main_contract = "Kubernetes namespace, image, client TLS Secret, ServiceAccount, host gateway, user namespace, and token TTL values may be inherited from gateway scope." +schema_v2_contract = "Those values exist exclusively under [openshell.drivers.kubernetes]." +migration = "Move namespace, default_image, supervisor_image, client_tls_secret_name, service_account_name, host_gateway_ip, enable_user_namespaces, and sa_token_ttl_secs into the Kubernetes driver table." +rationale = "Kubernetes-only deployment controls are not gateway-wide concerns." +parity_disposition = "intentional_change" +validation_capability_ids = ["kubernetes-core-placement-and-images"] + +[[intentional_changes]] +id = "docker-sandbox-label-rename" +category = "rename" +origin_main_contract = "Docker identifies OpenShell containers with sandbox_namespace." +schema_v2_contract = "Docker uses sandbox_label." +migration = "Rename [openshell.drivers.docker].sandbox_namespace to sandbox_label." +rationale = "The value is a Docker container label, not a Kubernetes-style namespace." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-image-and-callback-configuration"] + +[[intentional_changes]] +id = "podman-ssh-socket-rename" +category = "rename" +origin_main_contract = "Podman uses sandbox_ssh_socket_path in gateway TOML." +schema_v2_contract = "Podman uses ssh_socket_path in gateway TOML." +migration = "Rename [openshell.drivers.podman].sandbox_ssh_socket_path to ssh_socket_path." +rationale = "The canonical name now matches the Docker and Kubernetes driver tables." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "vm-grpc-endpoint-rename" +category = "rename" +origin_main_contract = "The VM gateway TOML callback override is grpc_endpoint, while the spawned standalone driver uses the internal field openshell_endpoint and flag --openshell-endpoint; OPENSHELL_GRPC_ENDPOINT is the environment override." +schema_v2_contract = "The VM gateway TOML callback override remains grpc_endpoint, and the spawned standalone driver now uses the same internal field name plus --grpc-endpoint; OPENSHELL_GRPC_ENDPOINT is unchanged." +migration = "No gateway TOML key changes. Operators invoking openshell-driver-vm directly must replace --openshell-endpoint with --grpc-endpoint." +rationale = "All compute drivers use the same name for the gateway gRPC callback endpoint." +parity_disposition = "intentional_change" +validation_capability_ids = ["vm-launch-and-resource-configuration", "external-compute-driver-socket"] + +[[intentional_changes]] +id = "canonical-image-pull-policy" +category = "type_normalization" +origin_main_contract = "Drivers accept different pull-policy types and spellings, including Podman missing and Kubernetes-style capitalization." +schema_v2_contract = "Gateway TOML uses always, if_not_present, never, and Podman-only newer through the shared typed policy." +migration = "Translate legacy or runtime-native spellings to canonical lowercase values." +rationale = "A shared validated type rejects typos before contacting a container runtime while preserving Podman's additional newer behavior." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-image-and-callback-configuration", "podman-image-and-callback-configuration", "kubernetes-core-placement-and-images", "local-launch-script-producers"] + +[[intentional_changes]] +id = "gateway-jwt-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "gateway_jwt.ttl_secs = 0 means that sandbox tokens do not expire." +schema_v2_contract = "Omitting gateway_jwt.ttl_secs means non-expiring sandbox tokens; an explicit zero is invalid." +migration = "Remove ttl_secs when non-expiring local tokens are intended, or set a positive duration." +rationale = "Omission distinguishes a deliberate absence of expiry from an invalid duration." +parity_disposition = "intentional_change" +validation_capability_ids = ["gateway-minted-sandbox-jwt"] + +[[intentional_changes]] +id = "sandbox-pid-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "Docker and Podman sandbox_pids_limit use zero as a backend-default sentinel." +schema_v2_contract = "Omitting sandbox_pids_limit uses the OpenShell default of 2048; configured values must be positive." +migration = "Remove a zero sandbox_pids_limit or replace it with a positive explicit limit." +rationale = "A typed optional non-zero limit removes ambiguous zero handling and gives both local container drivers one default." +parity_disposition = "intentional_change" +validation_capability_ids = ["docker-security-and-provider-configuration", "podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "podman-pid-limit-restored" +category = "bug_fix" +origin_main_contract = "Podman accepts a positive sandbox_pids_limit, but serializes it with Docker's PidsLimit shape; libpod ignores that field and applies its default limit of 2048." +schema_v2_contract = "Podman serializes a positive sandbox_pids_limit as OCI resource_limits.pids.limit, and the configured value is applied to the container." +migration = "No configuration migration is required; existing positive values begin taking effect after upgrading." +rationale = "Schema-v2 live validation exposed that the frozen baseline accepted this security control without enforcing it at runtime. Preserving that defect would make configuration parity unsafe." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "podman-health-zero-sentinel-removed" +category = "sentinel_removal" +origin_main_contract = "Podman health_check_interval_secs = 0 disables health checks." +schema_v2_contract = "Omitting health_check_interval_secs disables gateway-managed Podman health checks; configured values must be positive." +migration = "Remove a zero health_check_interval_secs or set a positive interval." +rationale = "Optional non-zero duration expresses enabled and disabled states without a sentinel." +parity_disposition = "intentional_change" +validation_capability_ids = ["podman-runtime-security-and-health"] + +[[intentional_changes]] +id = "guest-tls-centralized" +category = "ownership" +origin_main_contract = "Local driver tables may override guest_tls_ca, guest_tls_cert, and guest_tls_key inherited from gateway scope." +schema_v2_contract = "One complete guest TLS bundle is gateway-owned and injected into the selected Docker, Podman, or VM driver; driver-local fields are rejected." +migration = "Keep guest_tls_ca, guest_tls_cert, and guest_tls_key under [openshell.gateway] only." +rationale = "A single active local driver needs one callback client identity, and centralized validation prevents partial or conflicting bundles." +parity_disposition = "intentional_change" +validation_capability_ids = ["guest-callback-tls-ownership"] + +[[intentional_changes]] +id = "middleware-payload-name-normalized" +category = "rename_with_alias" +origin_main_contract = "Supervisor middleware uses max_body_bytes while gateway interceptors use max_response_bytes." +schema_v2_contract = "Supervisor middleware uses max_payload_bytes and continues accepting max_body_bytes as a compatibility alias." +migration = "Prefer max_payload_bytes in new configuration; existing max_body_bytes remains accepted." +rationale = "Payload describes middleware data more accurately without forcing an immediate compatibility break." +parity_disposition = "intentional_change" +validation_capability_ids = ["supervisor-middleware-registration"] + +[[intentional_changes]] +id = "vm-sandbox-identity-selection" +category = "default_behavior" +origin_main_contract = "New VM sandboxes default to UID and GID 10001 when no explicit identity is configured." +schema_v2_contract = "New VM root filesystems use the image sandbox account when present and otherwise UID/GID 1000; persisted overlays retain recorded or recoverable identity and ambiguous state fails closed." +migration = "Set sandbox_uid and sandbox_gid for a fixed identity, or retain the generated owner marker with persistent VM state." +rationale = "Image-derived identity preserves filesystem ownership, while explicit state evidence avoids silently reassigning legacy overlays." +parity_disposition = "intentional_change" +validation_capability_ids = ["vm-launch-and-resource-configuration", "vm-guest-security-and-spiffe"] + +[[intentional_changes]] +id = "package-default-only-auto-migration" +category = "migration_policy" +origin_main_contract = "Package-managed and operator-edited schema-v1 configuration may exist at upgrade time." +schema_v2_contract = "RPM and Homebrew automatically replace only recognized byte-identical package defaults; Debian and Snap preserve every legacy file and fail closed through read-only package preflight with explicit manual-migration guidance because their historical generated defaults have no safe provenance marker." +migration = "Automatically migrate recognized defaults, preserve every edited file, and require manual schema-v2 conversion when a package cannot prove that a legacy file is an untouched default." +rationale = "An upgrade must not overwrite operator intent merely because the old schema no longer parses." +parity_disposition = "intentional_change" +validation_capability_ids = ["rpm-schema-upgrade", "homebrew-debian-and-snap-upgrades"] diff --git a/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json b/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json new file mode 100644 index 0000000000..344343adf1 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json @@ -0,0 +1,9 @@ +{ + "accepted": true, + "baseline_commit": "74960ebfaeec4673885089ed995fad902459749f", + "baseline_success": true, + "candidate_commit": "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd", + "candidate_success": true, + "classification": "pass", + "parity": true +} diff --git a/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml b/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml new file mode 100644 index 0000000000..75e2f35b03 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-kubernetes-option-dispositions.toml @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Step 8 records field-level Kubernetes evidence separately from the broader +# capability manifest. A core pass means both frozen schema v1 and schema v2 +# successfully produced and executed the same observed Kubernetes resources. +# A platform-blocked field or value still requires its assigned qualifying lane. +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd" +core_comparison = "e2e/configs/gateway/schema-v2-kubernetes-core-comparison.json" + +[[coverage]] +id = "shared-combined-core" +status = "pass" +lane = "kind-v1.36.1-rootless-podman-host-gateway" +fields = [ + "namespace", + "default_image", + "image_pull_policy", + "image_pull_secrets", + "service_account_name", + "supervisor_image", + "supervisor_image_pull_policy", + "grpc_endpoint", + "ssh_socket_path", + "client_tls_secret_name", + "host_gateway_ip", + "sa_token_ttl_secs", + "workspace_mode", + "gateway_id", + "workspace_default_storage_size", + "workspace_storage_class", + "default_runtime_class_name", + "supervisor_sideload_method", + "topology", + "app_armor_profile", + "sandbox_uid", + "sandbox_gid", +] +evidence = [ + "Both variants reached Ready through ServiceAccount-token exchange and gateway-minted JWT authentication, then completed callback exec.", + "The API oracle matched images, Kubernetes pull-policy spelling, pull Secret, ServiceAccount, callback and SSH environment, client TLS mount, host aliases, 600-second token projection, runc RuntimeClass, Unconfined AppArmor request, UID/GID environment, CPU/memory, managed metadata, Bound 64Mi standard PVC, and deletion.", + "The normalized baseline and candidate results are byte-identical; comparison.json records both successes, parity=true, classification=pass, and accepted=true.", +] + +[[coverage]] +id = "operator-namespace-discovery" +status = "platform_blocked" +owner = "OpenShell Kubernetes workspace-mode CI" +lane = "managed-and-operator-workspace-matrix" +fields = ["operator_namespace_label", "operator_namespace_file"] +requirement = "Run paired managed and operator workspace lifecycles with isolated namespace RBAC, label discovery, allowlist hot reload, positive placement, and negative rejection probes." + +[[coverage]] +id = "managed-ssh-ingress" +status = "platform_blocked" +owner = "OpenShell Kubernetes network-policy CI" +lane = "managed-workspace-network-policy" +fields = [ + "managed_ssh_ingress.enabled", + "managed_ssh_ingress.gateway_namespace", + "managed_ssh_ingress.gateway_pod_selector", +] +requirement = "Run paired managed-workspace creation and inspect the generated NetworkPolicy while proving the configured gateway selector can connect and a nonmatching pod cannot." + +[[coverage]] +id = "sidecar-topology" +status = "platform_blocked" +owner = "OpenShell Kubernetes sidecar CI" +lane = "kubernetes-sidecar-network-enforcement" +fields = ["sidecar.proxy_uid", "sidecar.process_binary_aware_network_policy"] +requirement = "Run paired sidecar topology with relaxed and process-aware policy profiles and verify pod security context, proxy UID, network init, and enforcement behavior." + +[[coverage]] +id = "authenticated-upstream-proxy" +status = "platform_blocked" +owner = "OpenShell Kubernetes proxy CI" +lane = "kubernetes-authenticated-connect-proxy" +fields = [ + "https_proxy", + "no_proxy", + "proxy_auth_secret_name", + "proxy_auth_secret_key", + "proxy_auth_allow_insecure", + "proxy_connect_by_hostname", +] +requirement = "Provide an authenticated HTTP CONNECT proxy and Secret, then run paired allow, deny, no-proxy, credential, insecure-opt-in, and hostname-resolution probes." + +[[coverage]] +id = "spiffe-workload-api" +status = "platform_blocked" +owner = "OpenShell SPIRE Kubernetes CI" +lane = "kubernetes-spire-csi" +fields = ["provider_spiffe_workload_api_socket_path"] +requirement = "Provide SPIRE and its CSI driver, then run paired gateway and sandbox JWT-SVID acquisition from the allowed Workload API socket path and reject a disallowed path." + +[[coverage]] +id = "user-namespace-isolation" +status = "platform_blocked" +owner = "OpenShell Kubernetes userns CI" +lane = "kubernetes-userns-supported-runtime" +fields = ["enable_user_namespaces"] +requirement = "Run paired hostUsers=false lifecycle and host-ID mapping probes on a node whose kernel, kubelet, and runtime support Kubernetes user namespaces." + +[[qualified_value]] +id = "managed-and-operator-workspace-values" +field = "workspace_mode" +status = "platform_blocked" +owner = "OpenShell Kubernetes workspace-mode CI" +lane = "managed-and-operator-workspace-matrix" +requirement = "The local core pass exercised shared mode; managed namespace creation and operator placement/rejection remain assigned live checks." + +[[qualified_value]] +id = "image-volume-sideload" +field = "supervisor_sideload_method" +status = "platform_blocked" +owner = "OpenShell Kubernetes version matrix" +lane = "kubernetes-image-volume" +requirement = "The local core pass exercised init-container; run paired image-volume lifecycle on a cluster with the ImageVolume feature enabled." + +[[qualified_value]] +id = "sidecar-topology-value" +field = "topology" +status = "platform_blocked" +owner = "OpenShell Kubernetes sidecar CI" +lane = "kubernetes-sidecar-network-enforcement" +requirement = "The local core pass exercised combined topology; sidecar requires its enforcement fixture and positive/negative network probes." + +[[qualified_value]] +id = "apparmor-enforcement" +field = "app_armor_profile" +status = "platform_blocked" +owner = "OpenShell Kubernetes AppArmor CI" +lane = "kubernetes-apparmor-runtime-default-localhost" +requirement = "The local core pass proved Unconfined API projection; an AppArmor-enabled node with an installed localhost profile must prove RuntimeDefault and Localhost enforcement." + +[[qualified_value]] +id = "runtime-class-isolation" +field = "default_runtime_class_name" +status = "platform_blocked" +owner = "OpenShell Kubernetes confidential-runtime CI" +lane = "kubernetes-kata-runtimeclass" +requirement = "The local core pass proved runc RuntimeClass placement; a configured Kata or equivalent runtime must prove isolation-specific lifecycle behavior." + +[[qualified_value]] +id = "gpu-resource-combinations" +field = "create-request.resources.gpu" +status = "platform_blocked" +owner = "OpenShell Kubernetes GPU CI" +lane = "kubernetes-nvidia-device-plugin" +requirement = "Provide NVIDIA GPU nodes and the device plugin, then run paired GPU count/resource-name combinations and verify scheduling, limits, execution, and cleanup." diff --git a/e2e/configs/gateway/schema-v2-live-results.toml b/e2e/configs/gateway/schema-v2-live-results.toml new file mode 100644 index 0000000000..3860ac77f1 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-live-results.toml @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +# Step 5 establishes the portable status/create/Ready/list/exec/delete/list-empty +# contract. A platform-blocked result is not parity evidence; each blocked row +# names the lane that must replace it before the final release gate can pass. + +[[result]] +id = "portable-lifecycle-podman" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Podman compute driver" +driver = "podman" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "a3860084d019ed2ac979e3eaa1ddf085a96b773c" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Paired conformance smoke connected with authenticated mTLS on both variants.", + "Both variants passed status, create, Ready inspection, paginated list visibility, exact-output exec, delete, and eventual empty-list checks.", + "Normalized comparison recorded baseline_success=true, candidate_success=true, and parity=true; equal failures are rejected by the harness.", +] + +[[result]] +id = "portable-lifecycle-docker" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Docker compute driver" +driver = "docker" +status = "platform_blocked" +owner = "OpenShell Linux Docker CI lane" +lane = "linux-x86_64-docker" +blocker = "The validation host has no Docker CLI or Docker daemon socket. Step 7 and Step 13 must execute and assign this lane." + +[[result]] +id = "portable-lifecycle-kubernetes" +step = 5 +capability = "Portable sandbox lifecycle on the in-tree Kubernetes compute driver" +driver = "kubernetes" +status = "platform_blocked" +owner = "OpenShell Kubernetes CI lane" +lane = "ephemeral-kind-or-managed-kubernetes" +blocker = "The active kubectl context is an external OpenShift cluster that this validation must not modify. The installed kind cluster belongs to another project; Step 8 and Step 13 must use a dedicated OpenShell cluster." + +[[result]] +id = "portable-lifecycle-vm" +step = 5 +capability = "Portable sandbox lifecycle on the standalone VM compute driver" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM CI lane" +lane = "linux-x86_64-kvm-libkrun" +blocker = "The validation host has no prepared VM runtime bundle or built openshell-driver-vm executable. Step 9 and Step 13 must execute and assign this lane." + +[[result]] +id = "portable-lifecycle-mxc" +step = 5 +capability = "Portable sandbox lifecycle on the Windows MXC compute driver" +driver = "mxc" +status = "platform_blocked" +owner = "OpenShell Windows MXC CI lane" +lane = "windows-x64-and-windows-arm64-mxc" +blocker = "The validation host is Linux and cannot execute the Windows MXC runtime. Step 13 must assign native Windows validation." + +[[result]] +id = "gateway-wide-process-options" +step = 6 +capability = "Gateway listeners, configuration precedence, persistence, and legacy singleton selector" +driver = "gateway" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh binaries from both recorded commits started against isolated schema-v1 and schema-v2 files, SQLite databases, ports, state directories, and a rootless Podman socket.", + "Both variants exposed primary, health, and metrics listeners; the CLI primary port beat conflicting environment and file values, and --health-port 0 disabled the file-configured auxiliary listener.", + "Both variants created and reopened the same isolated SQLite database, rejected database_url in TOML, identified the rejected field, and did not disclose its secret-bearing value.", + "Both variants accepted one legacy OPENSHELL_DRIVERS=podman selector when no canonical selector was present; the candidate emitted its deprecation warning without logging the value.", + "Normalized comparison recorded baseline_success=true, candidate_success=true, and parity=true.", +] + +[[result]] +id = "gateway-tls-client-auth-policy" +step = 6 +capability = "Gateway TLS client-certificate handshake policy" +driver = "gateway" +status = "intentional_change" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea" +lane = "local-linux-x86_64-deterministic-tls" +evidence = [ + "Security review confirmed that origin/main accepted require_client_auth in TOML but ignored it and instead derived runtime policy from client CA and OIDC presence.", + "Schema v2 now rejects the unsupported file field while preserving the established derived policy: CA without OIDC requires a client certificate, while CA with OIDC permits bearer-only clients and validates certificates that are presented.", + "Focused preparation tests cover both derived branches, and TLS integration tests pass required-certificate, optional-certificate, valid-CA, wrong-CA, and multiplexed HTTP/gRPC handshakes.", +] + +[[result]] +id = "docker-driver-option-parity" +step = 7 +capability = "Docker image, callback, pull-policy, resource, security, provider, and mount options" +driver = "docker" +status = "platform_blocked" +owner = "OpenShell Linux Docker CI lane" +lane = "linux-x86_64-docker-with-apparmor-sub-lane" +blocker = "The validation host has neither a Docker CLI nor a Docker daemon, and the Docker driver deliberately rejects the available Libpod socket. A dedicated Docker lane must run paired lifecycle and inspect-based resource, label, mount, callback, and pull-policy checks; candidate-only AppArmor, authenticated proxy, and SPIFFE checks must run on qualifying daemons." + +[[result]] +id = "podman-driver-option-parity" +step = 7 +capability = "Podman image, callback, pull-policy, resource, mount, bootstrap, SSH, and health options" +driver = "podman" +status = "intentional_change" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "e09070d4ba0b30f0ac278fbb9938c1520ecff696" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh binaries from both recorded commits ran the same candidate-owned oracle against isolated schema-v1 and schema-v2 gateway processes, state, PKI, sockets, networks, and container stores.", + "Both variants selected the same immutable sandbox image, mapped their schema-specific pull-policy spelling to Podman missing, applied 750m CPU and 384Mi memory limits, preserved managed labels, mounted read-only bind and tmpfs configuration, injected sandbox-token and guest-TLS bootstrap files, used the renamed SSH socket, became healthy at the configured seven-second interval, and completed callback exec.", + "The normalized results were identical except for pids_limit: the frozen baseline accepted 31 but sent Docker's ignored PidsLimit field and received Podman's 2048 default, while the candidate sent OCI resource_limits.pids.limit and applied 31.", + "comparison.json recorded baseline_success=true, candidate_success=true, parity=false, classification=intentional_change, intentional_change_id=podman-pid-limit-restored, and accepted=true.", +] + +[[result]] +id = "podman-qualified-security-option-parity" +step = 7 +capability = "Podman AppArmor, authenticated proxy, SPIFFE, and rootful user-namespace options" +driver = "podman" +status = "platform_blocked" +owner = "OpenShell Podman security matrix" +lane = "linux-podman-apparmor-proxy-spiffe-rootful-userns" +blocker = "The available daemon is rootless and reports AppArmor disabled, and this local run has no authenticated proxy or SPIFFE Workload API fixture. A qualifying rootful matrix must validate those environment-dependent options live; deterministic driver tests remain coverage, not parity evidence." + +[[result]] +id = "kubernetes-core-option-parity" +step = 8 +capability = "Kubernetes shared workspace, images, placement, bootstrap, security projection, resources, and combined supervisor options" +driver = "kubernetes" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd" +lane = "kind-v1.36.1-rootless-podman-host-gateway" +evidence = [ + "Fresh candidate gateway and CLI binaries were built from the recorded candidate commit and bound by SHA-256, with the frozen baseline binary, in the retained artifact manifest.", + "Both host gateway variants used isolated schema-v1/schema-v2 files, SQLite stores, JWT keys, ports, namespaces, ServiceAccounts, Secrets, and resource names against a guarded disposable kind cluster.", + "Both variants reached Ready through authenticated callback, completed exact-marker exec, and produced matching Sandbox, Pod, and PVC semantics for images and pull policies, pull Secret, ServiceAccount, TLS and token projections, host aliases, SSH endpoint, runc placement, AppArmor request, UID/GID, CPU/memory, shared storage, managed metadata, and deletion.", + "comparison.json recorded baseline_success=true, candidate_success=true, parity=true, classification=pass, and accepted=true; private keys and all per-run Kubernetes fixtures were removed.", +] + +[[result]] +id = "kubernetes-qualified-option-parity" +step = 8 +capability = "Kubernetes managed/operator workspaces, sidecar enforcement, proxy, SPIFFE, user namespaces, AppArmor enforcement, confidential runtimes, and GPUs" +driver = "kubernetes" +status = "platform_blocked" +owner = "OpenShell Kubernetes qualified option matrix" +lane = "kubernetes-managed-operator-sidecar-proxy-spire-userns-apparmor-kata-gpu" +blocker = "The disposable core lane executed shared, combined, init-container, runc, and Unconfined values but did not execute the infrastructure-qualified alternatives. Assigned lanes must run paired managed/operator namespace discovery, sidecar network enforcement, authenticated proxy, SPIRE CSI, hostUsers=false, AppArmor RuntimeDefault/Localhost, image-volume, Kata RuntimeClass, and NVIDIA device-plugin GPU probes." + +[[result]] +id = "vm-launch-and-resource-configuration" +step = 9 +capability = "VM launch, callback, persistent state, images, resources, and guest identity" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM CI lane" +lane = "linux-x86_64-kvm-libkrun" +blocker = "This host has no prepared VM runtime bundle and no frozen-baseline or candidate openshell-driver-vm executable linked to a runnable libkrun environment. Both source trees passed their deterministic VM library suites, but only a paired KVM/libkrun lane can prove Ready, exec, restart recovery, overlay persistence, resource sizing, callback equivalence, and deletion." + +[[result]] +id = "vm-guest-security-and-spiffe" +step = 9 +capability = "VM guest token containment, TLS, owner state, proxy credentials, and SPIFFE opt-in" +driver = "vm" +status = "platform_blocked" +owner = "OpenShell Linux VM security CI lane" +lane = "linux-x86_64-kvm-libkrun-proxy-spiffe" +blocker = "The guest JWT mode and visibility, private owner-marker and sandbox-state persistence, callback recovery, TLS key permissions, proxy credential containment, and guest-reachable SPIFFE TCP behavior require a booted VM. The assigned lane must add authenticated-proxy and Workload API TCP fixtures to the paired libkrun/KVM run." + +[[result]] +id = "compute-driver-boundary-parity" +step = 10 +capability = "In-tree and external-UDS compute-driver execution with sandbox callback connectivity" +driver = "podman" +status = "pass" +validated_baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +validated_candidate_commit = "4a39da510e4d278a24dd60291149519c9a570b46" +lane = "local-linux-x86_64-rootless-podman-5.8.2" +evidence = [ + "Fresh exact-source gateway, CLI, conformance, Podman driver, and supervisor artifacts were staged before execution, hashed, executed from retained staging directories, and verified unchanged after execution.", + "Driver-free baseline and candidate gateways completed the same authenticated status, create, Ready, list, callback-exec, delete, and list-empty oracle through distinct-content external-driver executables over separate UDS endpoints; in-tree gateways completed the same oracle.", + "External drivers ran under env -i with a complete allowlisted environment attestation covering compute and Podman sockets, callback endpoint and TLS-file hashes, pull policy, images, network, timeouts, and bind mounts; successful driver logs were retained and hashed.", + "The verifier bound each transport-only gateway socket_path to the driver process input and required distinct baseline and candidate compute sockets, Podman sockets, networks, and callback TLS paths.", + "All four runs executed one repository-digest-pinned sandbox image and matched its image ID and manifest digest. They also matched the supervisor base-image identity, digest-pinned base reference, source Dockerfile, installed-package manifest, and same-source runtime artifacts across topologies.", + "Each variant built its supervisor image from its own staged supervisor binary in a forced temporary Podman service and isolated store; all supervisor runtime image references were digest-pinned.", + "A staged and hashed transparent CLI wrapper retained exact exec stdout bytes; the verifier tied them to the unique conformance run ID, required authenticated preflight and every lifecycle exit 0, and hashed result, launch, driver, raw-log, exec-stdout, gateway-configuration, package, and staged-artifact evidence.", +] + +# Step 11 inventories the remaining cross-cutting capabilities. Paired +# deterministic suites passed, but they are regression preflight rather than +# live parity evidence. Every row therefore remains assigned to a live lane. + +[[result]] +id = "oidc-bearer-authentication" +step = 11 +capability = "OIDC bearer token validation, role and scope authorization, and JWKS refresh" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell auth and OIDC CI lane" +lane = "linux-oidc-paired-keycloak-jwks" +blocker = "The host has no isolated paired issuer fixture that presents equivalent signed tokens to both exact-source gateways. Existing Keycloak and OIDC suites are candidate-oriented and cannot establish schema-v1/schema-v2 audience, role, scope, expiry, signature, and key-rotation parity without a shared candidate-owned oracle." + +[[result]] +id = "mtls-user-authentication" +step = 11 +capability = "mTLS client-certificate user identity and authorization" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-mtls-user-principal-paired" +blocker = "The paired TLS integration tests validate handshake policy but do not drive an authenticated user RPC through both gateways or prove certificate-to-principal mapping. A live lane must use isolated client PKI, equivalent schema files, and the same authorization oracle for enabled and disabled mTLS user authentication." + +[[result]] +id = "unsafe-unauthenticated-user-mode" +step = 11 +capability = "Explicit trusted-development unauthenticated user mode with authenticated sandbox callbacks" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway authentication CI lane" +lane = "linux-auth-chain-paired-isolated" +blocker = "No paired live fixture currently proves that the trusted-development switch admits user requests while sandbox callbacks continue to require sandbox credentials. Candidate-only extension examples use this switch for setup, but that is not an authentication-boundary parity test." + +[[result]] +id = "gateway-minted-sandbox-jwt" +step = 11 +capability = "Gateway-minted sandbox and extension JWT claims and lifetime semantics" +driver = "auth" +status = "platform_blocked" +owner = "OpenShell gateway JWT CI lane" +lane = "linux-gateway-jwt-claims-paired" +blocker = "Step 10 proved authenticated callback connectivity but did not capture and compare token claims. A controlled callback and extension fixture must verify issuer, gateway identity, key ID, audience, subject, token type, and expiry semantics for both gateways; explicit zero versus omitted TTL remains the ledgered gateway-jwt-zero-sentinel-removed intentional change." + +[[result]] +id = "otlp-observability" +step = 11 +capability = "Gateway and in-tree driver OTLP traces and failure isolation" +driver = "observability" +status = "platform_blocked" +owner = "OpenShell observability CI lane" +lane = "linux-otlp-grpc-collector-paired" +blocker = "The deterministic exporter tests passed, but no retained paired collector capture proves emitted gateway and in-tree-driver spans, gateway name, service name, selected-driver resource attributes, or continued serving after collector failure. A loopback OTLP/gRPC collector lane must observe those outcomes from both exact-source processes." + +[[result]] +id = "gateway-interceptor-registration" +step = 11 +capability = "Gateway interceptor registration, binding, mutation, and failure behavior" +driver = "gateway-interceptor" +status = "platform_blocked" +owner = "OpenShell gateway interceptor CI lane" +lane = "linux-gateway-interceptor-paired" +blocker = "The interceptor library suite passed on both revisions, while the governance-interceptor smoke test remains candidate-only. A paired service must compare Describe validation, binding selection, unary request and response mutation, failure policy, timeout and size limits, and secret exclusion against both gateway schema variants." + +[[result]] +id = "supervisor-middleware-registration" +step = 11 +capability = "Supervisor middleware registration, policy distribution, and enforcement" +driver = "supervisor-middleware" +status = "platform_blocked" +owner = "OpenShell supervisor middleware CI lane" +lane = "linux-supervisor-middleware-paired" +blocker = "The middleware library suite passed on both revisions, while the content-guard smoke test remains candidate-only. A paired gateway, sandbox, and middleware fixture must compare Describe negotiation, policy distribution, guarded and unguarded traffic, failure policy, timeout, and payload limits; the field-name normalization remains the ledgered middleware-payload-name-normalized intentional change." + +[[result]] +id = "provider-profile-sources" +step = 11 +capability = "Builtin, user, and interceptor provider-profile source composition" +driver = "provider-profiles" +status = "platform_blocked" +owner = "OpenShell provider profile CI lane" +lane = "linux-provider-profiles-interceptor-paired" +blocker = "Server tests cover source construction and duplicate rejection, but no shared live interceptor catalog has been queried through both gateways. The assigned lane must compare builtin, user, and interceptor source ordering, normalized profile identities, duplicate rejection, discovery output, and failure handling with isolated persistent state." + +[[result]] +id = "inference-control-plane-configuration" +step = 11 +capability = "Persisted provider and route configuration delivered to sandbox inference" +driver = "inference" +status = "platform_blocked" +owner = "OpenShell inference E2E lane" +lane = "linux-inference-provider-routing-paired" +blocker = "Existing inference routing tests are candidate-oriented and this host has no assigned paired provider and model-service fixture. The lane must configure providers and routes through each gateway control plane, observe the effective supervisor bundle, and compare model discovery and an inference request without reusing database state." + +[[result]] +id = "credential-driver-selection-and-kek" +step = 11 +capability = "Credential-driver selection, KEK handling, encrypted storage, and secret containment" +driver = "credentials" +status = "platform_blocked" +owner = "OpenShell credential security CI lane" +lane = "linux-credential-kek-paired" +blocker = "Both server suites passed deterministic selection, KEK, and credential tests, but they are revision-local tests rather than one candidate-owned live oracle. A paired lane must start isolated gateways with independent databases and KEKs, compare default encrypted storage and invalid selections, store and resolve opaque credentials, and verify that logs and configuration do not disclose secrets." + +[[result]] +id = "credential-driver-backend-tables" +step = 11 +capability = "In-tree and remote credential-driver transport and opaque credential lifecycle" +driver = "credential-driver" +status = "platform_blocked" +owner = "OpenShell credential driver E2E lane" +lane = "kubernetes-vault-uds-credential-drivers-paired" +blocker = "The host has no assigned disposable Kubernetes Secrets or Vault backend and no retained paired UDS credential-driver execution. Existing backend E2E coverage is candidate-oriented. The assigned lane must compare in-tree and remote transport validation plus opaque store and retrieve behavior using isolated namespaces, Vault state, sockets, databases, and credentials." + +# Step 12 resolves deterministic package-startup blockers without claiming that +# source-tree tests substitute for installation, upgrade, or refresh evidence. + +[[result]] +id = "rpm-package-upgrade" +step = 12 +capability = "RPM schema-v1 default migration and operator-edited configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell RPM package upgrade CI lane" +lane = "fedora-rpm-prior-release-upgrade" +blocker = "Deterministic migration tests prove exact-default replacement, edited-file preservation, mode preservation, idempotence, and unsafe-path rejection, but no prior RPM was installed and upgraded on this host. The assigned lane must verify package ownership and permissions, service restart, exact and edited schema-v1 files, and rollback-safe failure behavior." + +[[result]] +id = "debian-package-upgrade" +step = 12 +capability = "Debian prior-artifact upgrade with schema-v1 configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell Debian package upgrade CI lane" +lane = "ubuntu-debian-prior-release-upgrade" +blocker = "The source-tree tests prove Debian preflight selection, source-free diagnostics, non-mutation, and ordering before certificate generation, but dpkg-deb and an installed prior Debian artifact are unavailable. The assigned lane must preserve generated and edited schema-v1 files, exercise manual conversion, and prove successful schema-v2 service restart after upgrade." + +[[result]] +id = "snap-package-refresh" +step = 12 +capability = "Snap prior-release refresh with schema-v1 configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell Snap refresh CI lane" +lane = "ubuntu-snap-prior-release-refresh" +blocker = "The source-tree tests prove Snap config precedence, preflight failure handling, and non-mutation, but this host cannot install and refresh confined Snap revisions. The assigned lane must refresh a prior Snap with generated and edited schema-v1 files, exercise manual conversion, and prove successful schema-v2 daemon startup." + +[[result]] +id = "homebrew-package-upgrade" +step = 12 +capability = "Homebrew prior-release upgrade with package-default and operator configuration preservation" +driver = "packaging" +status = "platform_blocked" +owner = "OpenShell macOS Homebrew package upgrade CI lane" +lane = "macos-homebrew-prior-release-upgrade" +blocker = "No Homebrew formula installation or prior-release upgrade ran on this Linux host. The assigned macOS lane must upgrade a prior formula with recognized package defaults and edited schema-v1 configuration, verify only provable defaults migrate automatically, and prove successful schema-v2 service restart." diff --git a/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml new file mode 100644 index 0000000000..473e092433 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Reviewed potential blockers for the schema-v2 parity campaign. A disposition +# records whether the branch must change before parity can be claimed; it does +# not record test execution results. +ledger_version = 1 +issue = 2792 +baseline_commit = "74960ebfaeec4673885089ed995fad902459749f" +candidate_start_commit = "8c868e430e9cd3284d7e274628419ab484ebcee0" + +[[gaps]] +id = "legacy-environment-selector-upgrade" +severity = "none" +parity_relation = "regression" +disposition = "resolved" +origin_main_behavior = "OPENSHELL_DRIVERS accepts one driver name and is loaded by RPM, Debian, and Homebrew gateway services through gateway.env." +candidate_behavior = "Schema v2 accepts one non-empty OPENSHELL_DRIVERS value as a deprecated environment-only alias, rejects ambiguity and conflicts, and keeps removed CLI flags rejected." +impact = "The package-upgrade regression is resolved while preserving singular selector semantics and actionable migration guidance." +resolution = "The gateway now accepts one non-empty OPENSHELL_DRIVERS value, preserves canonical selector precedence, rejects multiple values and conflicting canonical and legacy values, and emits one deprecation warning after tracing initialization without logging the selector." +validation = "Paired frozen-baseline and candidate processes reached readiness through OPENSHELL_DRIVERS=podman; focused tests cover empty, plural, malformed, non-UTF-8, equal canonical, conflicting canonical, and remote-socket cases." +owner_step = 6 + +[[gaps]] +id = "debian-snap-v1-upgrade" +severity = "none" +parity_relation = "upgrade_regression" +disposition = "resolved" +origin_main_behavior = "Debian auto-discovers a persistent schema-v1 XDG gateway.toml and Snap passes a persistent schema-v1 SNAP_COMMON gateway.toml." +candidate_behavior = "Debian and Snap run the gateway's source-free, read-only preflight against the same effectively selected config before certificate generation or daemon startup; rejected legacy files, schema/layout errors, and documented cross-field startup errors remain unchanged with actionable migration guidance. Selected driver-specific fields remain lazily validated at driver construction." +impact = "The package startup regression is resolved without inferring provenance or overwriting operator-owned configuration; release remains gated on real Debian upgrade and Snap refresh execution." +resolution = "Use the shared strict schema-v2 layout parser plus documented read-only effective startup checks, fail closed for explicit, unsafe, or legacy paths, preserve optional absence, and require manual migration because historical Debian and Snap generators have no safe default-provenance marker." +validation = "Deterministic Rust, shell, and Python tests cover selection precedence, stable diagnostics, content non-disclosure, semantic checks, file preservation, and package ordering. Real prior Debian upgrade and Snap refresh lanes remain platform-blocked and must verify preserved v1 files plus successful restart after manual conversion." +owner_step = 12 + +[[gaps]] +id = "tls-require-client-auth-ignored" +severity = "none" +parity_relation = "preexisting_inaccessible_option" +disposition = "resolved" +origin_main_behavior = "The TOML schema accepts and documents require_client_auth, but runtime derives the value from client CA presence and OIDC instead of using the configured boolean." +candidate_behavior = "Schema v2 rejects require_client_auth and documents the derived client CA and OIDC policy instead of silently accepting an ineffective security option." +impact = "The misleading security-sensitive configuration claim is removed without weakening CA-only gateways or breaking bearer-only OIDC clients." +resolution = "Security review selected removal over runtime wiring: the file-only TLS schema excludes require_client_auth while the internal runtime field continues enforcing the established client CA and OIDC policy." +validation = "File parsing rejects require_client_auth; preparation tests cover CA-only and CA-plus-OIDC derivation; TLS integration tests cover required, optional, valid-CA, wrong-CA, and no-certificate handshakes." +owner_step = 6 + +[[gaps]] +id = "unselected-driver-validation-claim" +severity = "none" +parity_relation = "documentation_gap" +disposition = "resolved" +origin_main_behavior = "Only the selected driver table receives driver-specific deserialization; unknown fields inside an unselected table are ignored." +candidate_behavior = "Schema v2 validates every driver entry is a table and lazily validates driver-specific fields only when that driver is selected, as the reference now states." +impact = "The documentation now distinguishes table-shape validation from selected-driver deserialization, so it no longer overstates startup guarantees." +resolution = "The gateway configuration reference was narrowed to state that driver-specific fields are validated when their driver is selected." +validation = "Configuration tests establish that every driver entry must be a TOML table and that selected driver tables reject unknown fields; the published reference describes the lazy boundary." +owner_step = 6 + +[[gaps]] +id = "multi-driver-runtime-loss" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Schema v1 accepts a list-shaped selector but runtime rejects configurations containing more than one driver." +candidate_behavior = "Schema v2 represents the existing invariant as one optional scalar driver." +impact = "No working multi-driver runtime capability was removed." +resolution = "Keep the singular selector in the intentional-change ledger and validate explicit selection plus auto-detection." +validation = "Confirm origin/main rejects two configured drivers and the candidate supports one scalar or omission." +owner_step = 5 + +[[gaps]] +id = "generated-e2e-selector-shape" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Committed E2E fixtures select one driver through the schema-v1 list." +candidate_behavior = "Committed Docker and Podman fixtures and e2e/run.sh consistently use and require the schema-v2 scalar selector." +impact = "No committed fixture mismatch was found." +resolution = "Use the existing fixtures in live Docker and Podman parity lanes." +validation = "Run both driver E2E wrappers and confirm the selected runtime creates a sandbox." +owner_step = 5 + +[[gaps]] +id = "rpm-exact-default-migration" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "The RPM service seeds its package-owned schema-v1 default." +candidate_behavior = "The RPM pre-start migrator replaces only a byte-identical package v1 default and preserves edited or unsafe paths." +impact = "The implemented exact-default path does not block parity, but environment-file and edited-config behavior remain covered by separate gaps." +resolution = "Retain deterministic migration tests and add a real package upgrade lane." +validation = "Upgrade an installed prior RPM with exact and edited defaults and verify restart, bytes, ownership, and mode." +owner_step = 12 + +[[gaps]] +id = "gateway-owned-guest-tls" +severity = "none" +parity_relation = "non_finding" +disposition = "no_action" +origin_main_behavior = "Guest callback TLS may be inherited from gateway scope or overridden in the selected local driver table." +candidate_behavior = "One complete gateway-owned bundle is validated and injected into the selected local driver; misplaced driver fields fail explicitly." +impact = "The ownership change fails closed rather than silently disabling callback TLS." +resolution = "Keep the ownership change in the intentional-change ledger and validate callback connectivity on Docker, Podman, and VM." +validation = "Exercise complete, partial, misplaced, and plaintext combinations, then establish a real sandbox callback on each local driver." +owner_step = 10 diff --git a/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt b/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt new file mode 100644 index 0000000000..823c5dd1ba --- /dev/null +++ b/e2e/configs/gateway/schema-v2-step11-baseline-attestation.txt @@ -0,0 +1,101 @@ +schema_v2_step11_deterministic_attestation_version=1 +variant=baseline +source_commit=74960ebfaeec4673885089ed995fad902459749f +source_tree_clean=true +cargo_target_scope=checkout-local +rustc_wrapper=disabled +cargo 1.95.0 (f2d3ce0bd 2026-03-21) +rustc 1.95.0 (59807616e 2026-04-14) + +=== suite:server-lib === +command=cargo test -q -p openshell-server --lib +--- output --- + +running 1463 tests +....................................................................................... 87/1463 +....................................................................................... 174/1463 +...............................................................................i....... 261/1463 +......................i................................................................ 348/1463 +..............................................................................i........ 435/1463 +....................................................................................... 522/1463 +....................................................................................... 609/1463 +....................................................................................... 696/1463 +....................................................................................... 783/1463 +......................................................................................i 870/1463 +....................................................................................... 957/1463 +....................................................................................... 1044/1463 +....................................................................................... 1131/1463 +................................................................................i...... 1218/1463 +.........................................ii............................................ 1305/1463 +...............................i....................................................... 1392/1463 +....................................................................... +test result: ok. 1455 passed; 0 failed; 8 ignored; 0 measured; 0 filtered out; finished in 9.13s + +--- exit_status:0 --- + +=== suite:gateway-interceptors === +command=cargo test -q -p openshell-gateway-interceptors +--- output --- + +running 49 tests +................................................. +test result: ok. 49 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.32s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:supervisor-middleware === +command=cargo test -q -p openshell-supervisor-middleware +--- output --- + +running 85 tests +..................................................................................... +test result: ok. 85 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.52s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:otel-test-support === +command=cargo test -q -p openshell-otel-test-support +--- output --- + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:multiplex-tls-integration === +command=cargo test -q -p openshell-server --test multiplex_tls_integration +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.14s + +--- exit_status:0 --- + +=== suite:edge-tunnel-auth === +command=cargo test -q -p openshell-server --test edge_tunnel_auth +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + +--- exit_status:0 --- + +attestation_complete=true diff --git a/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt b/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt new file mode 100644 index 0000000000..5b2cab4905 --- /dev/null +++ b/e2e/configs/gateway/schema-v2-step11-candidate-attestation.txt @@ -0,0 +1,102 @@ +schema_v2_step11_deterministic_attestation_version=1 +variant=candidate +source_commit=363d8540830b2ea294d43198daa2b7a283a2face +source_tree_clean=true +cargo_target_scope=checkout-local +rustc_wrapper=disabled +cargo 1.95.0 (f2d3ce0bd 2026-03-21) +rustc 1.95.0 (59807616e 2026-04-14) + +=== suite:server-lib === +command=cargo test -q -p openshell-server --lib +--- output --- + +running 1489 tests +....................................................................................... 87/1489 +....................................................................................... 174/1489 +....................................................................................... 261/1489 +...........i.............................i............................................. 348/1489 +....................................................................................... 435/1489 +................i...................................................................... 522/1489 +....................................................................................... 609/1489 +....................................................................................... 696/1489 +....................................................................................... 783/1489 +....................................................................................... 870/1489 +........................i.............................................................. 957/1489 +....................................................................................... 1044/1489 +....................................................................................... 1131/1489 +....................................................................................... 1218/1489 +..................i...............................................ii................... 1305/1489 +........................................................i.............................. 1392/1489 +....................................................................................... 1479/1489 +.......... +test result: ok. 1481 passed; 0 failed; 8 ignored; 0 measured; 0 filtered out; finished in 9.12s + +--- exit_status:0 --- + +=== suite:gateway-interceptors === +command=cargo test -q -p openshell-gateway-interceptors +--- output --- + +running 49 tests +................................................. +test result: ok. 49 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.27s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:supervisor-middleware === +command=cargo test -q -p openshell-supervisor-middleware +--- output --- + +running 85 tests +..................................................................................... +test result: ok. 85 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:otel-test-support === +command=cargo test -q -p openshell-otel-test-support +--- output --- + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +--- exit_status:0 --- + +=== suite:multiplex-tls-integration === +command=cargo test -q -p openshell-server --test multiplex_tls_integration +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + +--- exit_status:0 --- + +=== suite:edge-tunnel-auth === +command=cargo test -q -p openshell-server --test edge_tunnel_auth +--- output --- + +running 5 tests +..... +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s + +--- exit_status:0 --- + +attestation_complete=true diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 5d650bae89..4bda320440 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -11,16 +11,16 @@ ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes -ENV OPENSHELL_DRIVERS=kubernetes \ +ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \ OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ OPENSHELL_GATEWAY_ID=openshell \ OPENSHELL_SANDBOX_NAMESPACE=openshell \ OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ - OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \ OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ OPENSHELL_K8S_TOPOLOGY=combined diff --git a/e2e/parity/gateway-options.sh b/e2e/parity/gateway-options.sh new file mode 100755 index 0000000000..1a0a98a0d4 --- /dev/null +++ b/e2e/parity/gateway-options.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Paired process-level checks for gateway-wide options that do not require a +# sandbox. The caller supplies immutable baseline and candidate gateway builds. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BASELINE_GATEWAY="${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" +CANDIDATE_GATEWAY="${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" +RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-${ROOT}/target/parity/gateway-options}" +BASELINE_SHA="74960ebfaeec4673885089ed995fad902459749f" +CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-gateway-options.XXXXXX")" +PIDS=() + +cleanup() { + local status=$? pid + for pid in "${PIDS[@]}"; do + kill -INT "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + done + rm -rf "${WORKDIR}" + exit "${status}" +} +trap cleanup EXIT + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +for binary in "${BASELINE_GATEWAY}" "${CANDIDATE_GATEWAY}"; do + [ -x "${binary}" ] || fail "gateway binary is not executable: ${binary:-}" +done +command -v curl >/dev/null 2>&1 || fail "curl is required" +command -v podman >/dev/null 2>&1 || fail "podman is required" +podman info >/dev/null 2>&1 || fail "podman service is not reachable" + +PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock}" +[ -S "${PODMAN_SOCKET}" ] || fail "Podman API socket is unavailable: ${PODMAN_SOCKET}" +mkdir -p "${RESULTS_DIR}" + +pick_port() { + python3 - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +} + +write_config() { + local output=$1 schema=$2 variant=$3 file_port=$4 health_port=$5 metrics_port=$6 + cat >"${output}" <>"${output}" + else + printf 'compute_driver = "podman"\n' >>"${output}" + fi + cat >>"${output}" <<'EOF' + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.drivers.podman] +EOF +} + +wait_for_url() { + local pid=$1 url=$2 log=$3 elapsed=0 + while [ "${elapsed}" -lt 100 ]; do + if curl --noproxy '*' --max-time 1 -fsS "${url}" >/dev/null 2>&1; then + return 0 + fi + if ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "=== gateway log ===" >&2 + cat "${log}" >&2 || true + return 1 + fi + sleep 0.1 + elapsed=$((elapsed + 1)) + done + echo "timed out waiting for ${url}" >&2 + cat "${log}" >&2 || true + return 1 +} + +stop_gateway() { + local pid=$1 + kill -INT "${pid}" >/dev/null 2>&1 || true + wait "${pid}" >/dev/null 2>&1 || true + PIDS=() +} + +run_variant() { + local variant=$1 schema=$2 sha=$3 gateway=$4 + local dir="${WORKDIR}/${variant}" + local config="${dir}/gateway.toml" db="${dir}/gateway.db" log="${dir}/gateway.log" + local file_port env_port primary_port health_port metrics_port second_primary + mkdir -p "${dir}/state" + file_port="$(pick_port)" + env_port="$(pick_port)" + primary_port="$(pick_port)" + health_port="$(pick_port)" + metrics_port="$(pick_port)" + second_primary="$(pick_port)" + write_config "${config}" "${schema}" "${variant}" "${file_port}" "${health_port}" "${metrics_port}" + + echo "==> ${variant}: precedence, listeners, and initial SQLite open" + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + OPENSHELL_SERVER_PORT="${env_port}" \ + "${gateway}" \ + --config "${config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --name "${variant}-cli" \ + --port "${primary_port}" \ + --log-level debug >"${log}" 2>&1 & + local pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${health_port}/healthz" "${log}" \ + || fail "${variant} health listener did not start" + curl --noproxy '*' --max-time 2 -fsS "http://127.0.0.1:${metrics_port}/metrics" >/dev/null \ + || fail "${variant} metrics listener is unavailable" + # A plain HTTP request to the gRPC listener returns 404; successful TCP/HTTP + # exchange is sufficient to prove that the selected primary port is bound. + curl --noproxy '*' --max-time 2 -sS "http://127.0.0.1:${primary_port}/" >/dev/null \ + || fail "${variant} primary listener is unavailable" + if curl --noproxy '*' --max-time 1 -sS "http://127.0.0.1:${file_port}/" >/dev/null 2>&1; then + fail "${variant} file port unexpectedly beat the CLI port" + fi + if curl --noproxy '*' --max-time 1 -sS "http://127.0.0.1:${env_port}/" >/dev/null 2>&1; then + fail "${variant} environment port unexpectedly beat the CLI port" + fi + grep -F "127.0.0.1:${primary_port}" "${log}" >/dev/null \ + || fail "${variant} startup log did not identify the effective primary bind" + [ -s "${db}" ] || fail "${variant} SQLite database was not created" + stop_gateway "${pid}" + + echo "==> ${variant}: SQLite reopen and health-port zero override" + : >"${log}" + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + "${gateway}" \ + --config "${config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --port "${second_primary}" \ + --health-port 0 >"${log}" 2>&1 & + pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${metrics_port}/metrics" "${log}" \ + || fail "${variant} did not reopen its SQLite database" + if curl --noproxy '*' --max-time 1 -fsS "http://127.0.0.1:${health_port}/healthz" >/dev/null 2>&1; then + fail "${variant} health listener remained active after --health-port 0" + fi + stop_gateway "${pid}" + + echo "==> ${variant}: legacy singleton environment selector" + local legacy_config="${dir}/legacy-selector.toml" + grep -v '^compute_driver' "${config}" >"${legacy_config}" + : >"${log}" + env -u OPENSHELL_COMPUTE_DRIVER \ + XDG_CONFIG_HOME="${dir}/config" \ + XDG_STATE_HOME="${dir}/state" \ + OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + OPENSHELL_DRIVERS=podman \ + "${gateway}" \ + --config "${legacy_config}" \ + --db-url "sqlite:${db}?mode=rwc" \ + --port "${second_primary}" >"${log}" 2>&1 & + pid=$! + PIDS=("${pid}") + wait_for_url "${pid}" "http://127.0.0.1:${health_port}/healthz" "${log}" \ + || fail "${variant} did not accept the legacy singleton selector" + if [ "${variant}" = candidate ]; then + grep -F 'OPENSHELL_DRIVERS is deprecated' "${log}" >/dev/null \ + || fail "candidate did not emit the legacy selector deprecation warning" + fi + stop_gateway "${pid}" + + echo "==> ${variant}: database URL in TOML is rejected without value disclosure" + local invalid="${dir}/invalid.toml" secret="parity-secret-${variant}" + awk -v secret="${secret}" ' + /^name =/ { print "database_url = \"postgres://user:" secret "@127.0.0.1/db\"" } + { print } + ' "${config}" >"${invalid}" + if OPENSHELL_PODMAN_SOCKET="${PODMAN_SOCKET}" \ + "${gateway}" --config "${invalid}" >"${dir}/invalid.log" 2>&1; then + fail "${variant} accepted database_url in gateway TOML" + fi + grep -F 'database_url' "${dir}/invalid.log" >/dev/null \ + || fail "${variant} database rejection did not identify the field" + if grep -F "${secret}" "${dir}/invalid.log" >/dev/null; then + fail "${variant} database rejection disclosed the configured secret" + fi + + cat >"${RESULTS_DIR}/gateway-options-${variant}.json" <"${RESULTS_DIR}/gateway-options-comparison.json" <<'EOF' +{"profile":"gateway-options","baseline_success":true,"candidate_success":true,"parity":true} +EOF + +echo "Gateway option parity passed." diff --git a/e2e/parity/kubernetes-options-test.sh b/e2e/parity/kubernetes-options-test.sh new file mode 100644 index 0000000000..4f14687e7a --- /dev/null +++ b/e2e/parity/kubernetes-options-test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCRIPT="${ROOT}/e2e/parity/kubernetes-options.sh" +TMP="$(mktemp -d)" +trap 'rm -rf "${TMP}"' EXIT + +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config baseline >"${TMP}/baseline.toml" +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 bash "${SCRIPT}" --print-config candidate >"${TMP}/candidate.toml" +python3 -I - "${TMP}/baseline.toml" "${TMP}/candidate.toml" <<'PY' +import sys, tomllib +def check(condition, message): + if not condition: + raise RuntimeError(message) +baseline=tomllib.load(open(sys.argv[1],'rb'))['openshell'] +candidate=tomllib.load(open(sys.argv[2],'rb'))['openshell'] +check(baseline['version']==1 and candidate['version']==2,'schema versions differ') +check(baseline['gateway']['compute_drivers']==['kubernetes'],'baseline selector differs') +check(candidate['gateway']['compute_driver']=='kubernetes','candidate selector differs') +shared={'default_image','supervisor_image','client_tls_secret_name','service_account_name','host_gateway_ip','enable_user_namespaces','sa_token_ttl_secs'} +check(shared <= baseline['gateway'].keys(),'baseline inherited fields missing') +check(shared.isdisjoint(candidate['gateway'].keys()),'candidate leaked driver fields into gateway table') +check(shared <= candidate['drivers']['kubernetes'].keys(),'candidate driver fields missing') +b=dict(baseline['drivers']['kubernetes']); b.update({key:baseline['gateway'][key] for key in shared}) +c=dict(candidate['drivers']['kubernetes']) +for projection in (b,c): + projection['gateway_id']='' + projection['grpc_endpoint']='http://host.openshell.internal:' + for field in ('image_pull_policy','supervisor_image_pull_policy'): + projection[field]={'IfNotPresent':'if_not_present'}.get(projection[field],projection[field]) +check(b==c,'schema-independent Kubernetes option projections differ') +PY + +: >"${TMP}/kubeconfig" +set +e +OPENSHELL_PARITY_BASELINE_ROOT="${TMP}/not-a-worktree" \ +OPENSHELL_PARITY_BASELINE_GATEWAY=/bin/true \ +OPENSHELL_PARITY_CANDIDATE_GATEWAY=/bin/true \ +OPENSHELL_PARITY_CLI=/bin/true \ +OPENSHELL_PARITY_KUBECONFIG="${TMP}/kubeconfig" \ +OPENSHELL_PARITY_KUBE_CONTEXT=default/external-production-cluster \ +OPENSHELL_PARITY_HOST_GATEWAY_IP=169.254.1.2 \ +bash "${SCRIPT}" >"${TMP}/unsafe.out" 2>&1 +status=$? +set -e +[ "${status}" -ne 0 ] +grep -F 'refusing non-parity context' "${TMP}/unsafe.out" >/dev/null + +echo "Kubernetes option parity deterministic tests passed." diff --git a/e2e/parity/kubernetes-options.sh b/e2e/parity/kubernetes-options.sh new file mode 100644 index 0000000000..658e751c3e --- /dev/null +++ b/e2e/parity/kubernetes-options.sh @@ -0,0 +1,432 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compare the frozen schema-v1 and current schema-v2 Kubernetes driver against +# one explicitly supplied, disposable kind cluster. Gateway processes run on +# the host so the same cluster and candidate-owned oracle exercise both schemas. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BASELINE_SHA="${OPENSHELL_PARITY_BASELINE_SHA:-74960ebfaeec4673885089ed995fad902459749f}" +CANDIDATE_SHA="${OPENSHELL_PARITY_CANDIDATE_SHA:-$(git -C "${ROOT}" rev-parse HEAD)}" +BASELINE_ROOT="${OPENSHELL_PARITY_BASELINE_ROOT:-}" +BASELINE_GATEWAY="${OPENSHELL_PARITY_BASELINE_GATEWAY:-}" +CANDIDATE_GATEWAY="${OPENSHELL_PARITY_CANDIDATE_GATEWAY:-}" +CLI="${OPENSHELL_PARITY_CLI:-}" +ARTIFACT_MANIFEST="${OPENSHELL_PARITY_ARTIFACT_MANIFEST:-}" +KUBECONFIG_PATH="${OPENSHELL_PARITY_KUBECONFIG:-}" +KUBE_CONTEXT="${OPENSHELL_PARITY_KUBE_CONTEXT:-}" +HOST_GATEWAY_IP="${OPENSHELL_PARITY_HOST_GATEWAY_IP:-}" +RUN_ID="${OPENSHELL_PARITY_RUN_ID:-$(date +%s)-$$}" +OUT="${OPENSHELL_PARITY_OUTPUT_DIR:-${ROOT}/target/parity/step8-kubernetes-${CANDIDATE_SHA:0:8}}" +SANDBOX_IMAGE="${OPENSHELL_PARITY_KUBERNETES_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +SUPERVISOR_IMAGE="${OPENSHELL_PARITY_KUBERNETES_SUPERVISOR_IMAGE:-ghcr.io/nvidia/openshell/supervisor:latest}" +RUNTIME_CLASS="openshell-parity-runc-${RUN_ID}" + +fail() { + echo "ERROR: Kubernetes option parity: $*" >&2 + exit 1 +} + +kctl() { + kubectl --kubeconfig "${KUBECONFIG_PATH}" --context "${KUBE_CONTEXT}" "$@" +} + +pick_port() { + python3 -I - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("0.0.0.0", 0)) + print(sock.getsockname()[1]) +PY +} + +write_config() { + local variant=$1 + local path=$2 + local namespace=$3 + local port=$4 + local run_dir=$5 + local gateway_id="step8-${variant}-${RUN_ID}" + local pull_policy + + if [ "${variant}" = baseline ]; then + pull_policy=IfNotPresent + cat >"${path}" <"${path}" <}" ;; esac +[[ "${BASELINE_SHA}" =~ ^[0-9a-f]{40}$ ]] || fail "baseline SHA must be a full lowercase SHA-1" +[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]] || fail "candidate SHA must be a full lowercase SHA-1" +[[ "${RUN_ID}" =~ ^[a-z0-9]([a-z0-9-]{0,30}[a-z0-9])?$ ]] || fail "run ID must be a lowercase DNS label of at most 32 characters" +[[ "${SANDBOX_IMAGE}" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@+-]{0,254}$ ]] || fail "sandbox image contains unsafe characters" +[[ "${SUPERVISOR_IMAGE}" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@+-]{0,254}$ ]] || fail "supervisor image contains unsafe characters" +python3 -I - "${HOST_GATEWAY_IP}" <<'PY' +import ipaddress, sys +value=ipaddress.ip_address(sys.argv[1]) +if value.version != 4: + raise SystemExit("host gateway IP must be IPv4") +PY +[ "$(git -C "${BASELINE_ROOT}" rev-parse HEAD)" = "${BASELINE_SHA}" ] || fail "baseline worktree is not ${BASELINE_SHA}" +[ "$(git -C "${ROOT}" rev-parse HEAD)" = "${CANDIDATE_SHA}" ] || fail "candidate worktree is not ${CANDIDATE_SHA}" +[ "$(kubectl --kubeconfig "${KUBECONFIG_PATH}" config current-context)" = "${KUBE_CONTEXT}" ] || fail "private kubeconfig current context differs from requested parity context" +[ "$(kctl -n kube-system get configmap openshell-parity-guard -o jsonpath='{.data.context}')" = "${KUBE_CONTEXT}" ] || fail "cluster lacks the matching provisioning-time parity guard" +[ "$(kctl -n kube-system get configmap openshell-parity-guard -o jsonpath='{.data.purpose}')" = schema-v2-capability-parity ] || fail "cluster parity guard has the wrong purpose" +kctl get nodes -o name | grep -q '^node/openshell-parity-' || fail "requested context is not the dedicated OpenShell parity cluster" +[ "$(kctl get crd sandboxes.agents.x-k8s.io -o jsonpath='{.status.conditions[?(@.type=="Established")].status}')" = True ] || fail "Agent Sandbox CRD is not established" +[ -f "${ARTIFACT_MANIFEST}" ] || fail "OPENSHELL_PARITY_ARTIFACT_MANIFEST is required" +python3 -I - "${ARTIFACT_MANIFEST}" "${BASELINE_SHA}" "${CANDIDATE_SHA}" "${BASELINE_GATEWAY}" "${CANDIDATE_GATEWAY}" "${CLI}" <<'PY' +import hashlib, pathlib, sys, tomllib +manifest=tomllib.load(open(sys.argv[1],'rb')) +expected={'baseline_commit':sys.argv[2],'candidate_commit':sys.argv[3]} +for key, value in expected.items(): + if manifest.get(key) != value: + raise SystemExit(f'artifact manifest {key} does not match') +for key, path in zip(('baseline_gateway_sha256','candidate_gateway_sha256','candidate_cli_sha256'),sys.argv[4:]): + digest=hashlib.sha256(pathlib.Path(path).read_bytes()).hexdigest() + if manifest.get(key) != digest: + raise SystemExit(f'artifact manifest {key} does not match supplied binary') +PY + +PARITY_ROOT="$(realpath -m "${ROOT}/target/parity")" +OUT="$(realpath -m "${OUT}")" +case "${OUT}" in "${PARITY_ROOT}"/step8-kubernetes-*) ;; *) fail "output must be a step8-kubernetes-* directory below ${PARITY_ROOT}" ;; esac +[ ! -L "${OUT}" ] || fail "output directory must not be a symlink" +rm -rf --one-file-system "${OUT}" +umask 077 +mkdir -p "${OUT}/raw" +cp "${ARTIFACT_MANIFEST}" "${OUT}/artifact-manifest.toml" +printf '%s\n' "${BASELINE_SHA}" >"${OUT}/baseline.sha" +printf '%s\n' "${CANDIDATE_SHA}" >"${OUT}/candidate.sha" +printf '%s\n' "${KUBE_CONTEXT}" >"${OUT}/context" + +runtime_class_created=false +cleanup_cluster_fixture() { + local status=$? + local cleanup_status=0 + set +e + if ${runtime_class_created}; then + kctl delete runtimeclass "${RUNTIME_CLASS}" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 + cleanup_status=$? + fi + set -e + if [ "${status}" -eq 0 ] && [ "${cleanup_status}" -ne 0 ]; then + echo "ERROR: failed to confirm RuntimeClass cleanup" >&2 + exit 1 + fi + exit "${status}" +} +trap cleanup_cluster_fixture EXIT +cat </dev/null +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: ${RUNTIME_CLASS} +handler: runc +EOF +runtime_class_created=true + +run_variant() ( + set -euo pipefail + local variant=$1 + local gateway=$2 + local namespace="openshell-parity-${variant}-${RUN_ID}" + local sandbox="k8s-${variant:0:1}-${RUN_ID: -6}" + local resource="default--${sandbox}" + local run_dir="${OUT}/raw/${variant}" + local config="${run_dir}/gateway.toml" + local port + local gateway_pid= + local registered_endpoint + local namespace_created=false + mkdir -p "${run_dir}/jwt" "${run_dir}/xdg-config/openshell/gateways/parity" "${run_dir}/xdg-state" "${run_dir}/xdg-data" + + cleanup_variant() { + local status=$? + local cleanup_status=0 + set +e + if [ -n "${gateway_pid}" ]; then + kill "${gateway_pid}" >/dev/null 2>&1 || true + wait "${gateway_pid}" >/dev/null 2>&1 || true + fi + rm -f "${run_dir}/jwt/signing.pem" "${run_dir}/client.key" + if ${namespace_created}; then + kctl delete namespace "${namespace}" --ignore-not-found --wait=true --timeout=120s >"${run_dir}/namespace-delete.log" 2>&1 + cleanup_status=$? + fi + set -e + if [ "${status}" -eq 0 ] && [ "${cleanup_status}" -ne 0 ]; then + echo "ERROR: ${variant} namespace cleanup was not confirmed" >&2 + exit 1 + fi + exit "${status}" + } + trap cleanup_variant EXIT + + openssl genpkey -algorithm ED25519 -out "${run_dir}/jwt/signing.pem" >/dev/null 2>&1 + openssl pkey -in "${run_dir}/jwt/signing.pem" -pubout -out "${run_dir}/jwt/public.pem" >/dev/null 2>&1 + printf 'step8-%s\n' "${variant}" >"${run_dir}/jwt/kid" + openssl req -x509 -newkey rsa:2048 -nodes -subj "/CN=step8-parity-client" \ + -keyout "${run_dir}/client.key" -out "${run_dir}/client.crt" -days 1 >/dev/null 2>&1 + + kctl create namespace "${namespace}" >"${run_dir}/namespace-create.log" + namespace_created=true + kctl -n "${namespace}" create serviceaccount parity-sandbox >"${run_dir}/service-account.log" + kctl -n "${namespace}" create secret generic parity-pull-secret \ + --type=kubernetes.io/dockerconfigjson --from-literal=.dockerconfigjson='{"auths":{}}' >"${run_dir}/pull-secret.log" + kctl -n "${namespace}" create secret generic parity-client-tls \ + --from-file=ca.crt="${run_dir}/client.crt" \ + --from-file=tls.crt="${run_dir}/client.crt" \ + --from-file=tls.key="${run_dir}/client.key" >"${run_dir}/client-tls-secret.log" + + port="$(pick_port)" + write_config "${variant}" "${config}" "${namespace}" "${port}" "${run_dir}" + KUBECONFIG="${KUBECONFIG_PATH}" \ + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + OPENSHELL_DB_URL="sqlite:${run_dir}/gateway.db" \ + "${gateway}" --config "${config}" >"${run_dir}/gateway.log" 2>&1 & + gateway_pid=$! + listener_ready=false + for _ in $(seq 1 60); do + if ! kill -0 "${gateway_pid}" >/dev/null 2>&1; then + fail "${variant} gateway exited before binding; see ${run_dir}/gateway.log" + fi + if python3 -I - "${port}" <<'PY' +import socket, sys +try: + with socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=.2): + pass +except OSError: + raise SystemExit(1) +PY + then + listener_ready=true + break + fi + sleep 0.5 + done + ${listener_ready} || fail "${variant} gateway did not bind within 30 seconds" + + registered_endpoint="http://127.0.0.1:${port}" + cat >"${run_dir}/xdg-config/openshell/gateways/parity/metadata.json" <"${run_dir}/xdg-config/openshell/active_gateway" + + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 360 "${CLI}" sandbox create --name "${sandbox}" --cpu 250m --memory 128Mi --detach \ + >"${run_dir}/create.log" 2>&1 + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 60 "${CLI}" sandbox exec --name "${sandbox}" --no-tty -- \ + sh -c 'printf step8-kubernetes-exec' >"${run_dir}/exec.log" 2>&1 + grep -q 'step8-kubernetes-exec' "${run_dir}/exec.log" || fail "${variant} callback exec marker missing" + + kctl -n "${namespace}" get sandbox "${resource}" -o json >"${run_dir}/sandbox.json" + kctl -n "${namespace}" get pod "${resource}" -o json >"${run_dir}/pod.json" + kctl -n "${namespace}" get pvc "workspace-${resource}" -o json >"${run_dir}/pvc.json" + + python3 -I - "${run_dir}" "${SANDBOX_IMAGE}" "${SUPERVISOR_IMAGE}" "${HOST_GATEWAY_IP}" "${RUNTIME_CLASS}" <<'PY' +import json, pathlib, sys +def check(condition, message): + if not condition: + raise RuntimeError(message) +run=pathlib.Path(sys.argv[1]); sandbox_image, supervisor_image, host_ip, runtime_class=sys.argv[2:] +pod=json.loads((run/'pod.json').read_text()); pvc=json.loads((run/'pvc.json').read_text()); sb=json.loads((run/'sandbox.json').read_text()) +spec=pod['spec']; agent=next(c for c in spec['containers'] if c['name']=='agent'); env={x['name']:x.get('value','') for x in agent.get('env',[])} +inits={c['name']:c for c in spec.get('initContainers',[])}; install=inits['openshell-supervisor-install'] +vols={v['name']:v for v in spec.get('volumes',[])}; mounts={m['name']:m for m in agent.get('volumeMounts',[])} +hosts={(h, a['ip']) for a in spec.get('hostAliases',[]) for h in a.get('hostnames',[])} +check(pod['status']['phase']=='Running','Pod is not Running') +conditions={c['type']:c['status'] for c in sb.get('status',{}).get('conditions',[])} +check(conditions.get('Ready')=='True','Sandbox Ready condition is not true') +check(agent['image']==sandbox_image and agent['imagePullPolicy']=='IfNotPresent','sandbox image or pull policy differs') +check(install['image']==supervisor_image and install['imagePullPolicy']=='IfNotPresent','supervisor image or pull policy differs') +check([x['name'] for x in spec.get('imagePullSecrets',[])]==['parity-pull-secret'],'image pull Secret differs') +check(spec['serviceAccountName']=='parity-sandbox','ServiceAccount differs') +check(env['OPENSHELL_ENDPOINT'].startswith('http://host.openshell.internal:'),'callback endpoint differs') +check(env['OPENSHELL_SSH_SOCKET_PATH']=='/run/openshell/parity-kubernetes-ssh.sock','SSH socket differs') +check(env['OPENSHELL_SANDBOX_UID']=='1000' and env['OPENSHELL_SANDBOX_GID']=='1000','sandbox identity differs') +check(('host.openshell.internal',host_ip) in hosts and ('host.docker.internal',host_ip) in hosts,'host aliases differ') +check(spec['runtimeClassName']==runtime_class and spec.get('hostUsers',True) is not False,'RuntimeClass or user namespace posture differs') +check(agent['securityContext']['appArmorProfile']['type']=='Unconfined','AppArmor profile differs') +check(agent['resources']['requests']=={'cpu':'250m','memory':'128Mi'},'resource requests differ') +check(agent['resources']['limits']=={'cpu':'250m','memory':'128Mi'},'resource limits differ') +check(vols['openshell-sa-token']['projected']['sources'][0]['serviceAccountToken']['expirationSeconds']==600,'ServiceAccount token TTL differs') +check(vols['openshell-client-tls']['secret']['secretName']=='parity-client-tls','client TLS Secret differs') +check(mounts['openshell-client-tls']['readOnly'] is True,'client TLS mount is not read-only') +check(pvc['status']['phase']=='Bound' and pvc['spec']['storageClassName']=='standard','PVC phase or StorageClass differs') +check(pvc['spec']['resources']['requests']['storage']=='64Mi','PVC storage request differs') +labels=sb['metadata']['labels'] +for key in ('openshell.ai/sandbox-id','openshell.ai/sandbox-name','openshell.ai/sandbox-workspace','openshell.ai/gateway-id','openshell.ai/managed-by'): + check(labels.get(key),f'managed label {key} missing') +observed_sideload='init-container' if 'openshell-supervisor-install' in inits else 'unknown' +observed_topology='combined' if [c['name'] for c in spec['containers']]==['agent'] else 'other' +observed_workspace_mode='shared' if pvc['metadata']['namespace']==pod['metadata']['namespace'] and pvc['metadata']['name'].startswith('workspace-default--') else 'other' +check(observed_sideload=='init-container','supervisor sideload method differs') +check(observed_topology=='combined','supervisor topology differs') +check(observed_workspace_mode=='shared','workspace placement differs') +normalized={ + 'scenario':'kubernetes-core-options','pod_phase':'Running','sandbox_ready':True, + 'sandbox_image':agent['image'],'sandbox_image_pull_policy':agent['imagePullPolicy'], + 'image_pull_secrets':['parity-pull-secret'],'service_account':'parity-sandbox', + 'supervisor_image':install['image'],'supervisor_image_pull_policy':install['imagePullPolicy'], + 'supervisor_sideload_method':observed_sideload,'topology':observed_topology, + 'callback_endpoint_host':'host.openshell.internal','callback_exec':True, + 'ssh_socket_path':env['OPENSHELL_SSH_SOCKET_PATH'],'client_tls_secret':'parity-client-tls', + 'host_gateway_ip':host_ip,'sa_token_ttl_secs':600,'runtime_class_handler':'runc', + 'enable_user_namespaces':False,'app_armor_profile':'Unconfined','sandbox_uid':1000,'sandbox_gid':1000, + 'workspace_mode':observed_workspace_mode,'workspace_storage':'64Mi','workspace_storage_class':'standard','pvc_phase':'Bound', + 'cpu':'250m','memory':'128Mi','managed_labels':True, +} +(run.parent.parent/f'{run.name}.normalized.json').write_text(json.dumps(normalized,sort_keys=True,separators=(',',':'))+'\n') +PY + + XDG_CONFIG_HOME="${run_dir}/xdg-config" XDG_STATE_HOME="${run_dir}/xdg-state" XDG_DATA_HOME="${run_dir}/xdg-data" \ + timeout 60 "${CLI}" sandbox delete "${sandbox}" >"${run_dir}/delete.log" 2>&1 + for _ in $(seq 1 60); do + if ! kctl -n "${namespace}" get sandbox "${resource}" >/dev/null 2>&1 \ + && ! kctl -n "${namespace}" get pod "${resource}" >/dev/null 2>&1 \ + && ! kctl -n "${namespace}" get pvc "workspace-${resource}" >/dev/null 2>&1; then + break + fi + sleep 1 + done + ! kctl -n "${namespace}" get sandbox "${resource}" >/dev/null 2>&1 || fail "${variant} Sandbox remained after delete" + ! kctl -n "${namespace}" get pod "${resource}" >/dev/null 2>&1 || fail "${variant} Pod remained after delete" + ! kctl -n "${namespace}" get pvc "workspace-${resource}" >/dev/null 2>&1 || fail "${variant} PVC remained after delete" +) + +set +e +run_variant baseline "${BASELINE_GATEWAY}" +baseline_status=$? +run_variant candidate "${CANDIDATE_GATEWAY}" +candidate_status=$? +set -e + +baseline_success=false; candidate_success=false +[ "${baseline_status}" -eq 0 ] && baseline_success=true +[ "${candidate_status}" -eq 0 ] && candidate_success=true +parity=false; classification=regression; accepted=false +if ${baseline_success} && ${candidate_success} && [ -f "${OUT}/baseline.normalized.json" ] && [ -f "${OUT}/candidate.normalized.json" ]; then + if cmp -s "${OUT}/baseline.normalized.json" "${OUT}/candidate.normalized.json"; then + parity=true; classification=pass; accepted=true + fi +fi +cat >"${OUT}/comparison.json" <&2; exit 2 ;; +esac + +fail() { echo "ERROR: podman-options oracle: $*" >&2; exit 1; } +json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } +podman_cmd() { + if [ "${OPENSHELL_E2E_CONTAINER_ENGINE_UNSET_XDG_CONFIG_HOME:-0}" = 1 ]; then + env -u XDG_CONFIG_HOME podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + elif [ -n "${OPENSHELL_E2E_CONTAINER_ENGINE_XDG_CONFIG_HOME:-}" ]; then + XDG_CONFIG_HOME="${OPENSHELL_E2E_CONTAINER_ENGINE_XDG_CONFIG_HOME}" podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + else + podman --url "unix://${OPENSHELL_PODMAN_SOCKET}" "$@" + fi +} +cleanup() { + status=$? + if [ "${CREATED}" = 1 ]; then "${CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true; fi + rm -rf "${WORKDIR}" + exit "${status}" +} +trap cleanup EXIT + +mkdir -p "${WORKDIR}/bind-source" +printf '%s\n' parity-bind-mount >"${WORKDIR}/bind-source/probe" +bind_source="$(json_escape "${WORKDIR}/bind-source")" +DRIVER_CONFIG="{\"podman\":{\"mounts\":[{\"type\":\"bind\",\"source\":\"${bind_source}\",\"target\":\"/tmp/parity-bind\",\"read_only\":true,\"selinux_label\":\"private\"},{\"type\":\"tmpfs\",\"target\":\"/tmp/parity-cache\",\"options\":[\"nosuid\",\"nodev\"],\"size_bytes\":1048576,\"mode\":448}]}}" +"${CLI}" sandbox create --name "${NAME}" --cpu 750m --memory 384Mi \ + --driver-config-json "${DRIVER_CONFIG}" --detach +CREATED=1 +podman_cmd ps -aq --filter label=openshell.managed=true --filter "label=openshell.ai/sandbox-name=${NAME}" > "${WORKDIR}/ids" +env wc -l "${WORKDIR}/ids" | env grep -E "^[[:space:]]*1[[:space:]]" >/dev/null || fail "expected exactly one managed container" + +# Image IDs, names, and inspect attributes are checked but never emitted. +while IFS= read -r id; do + podman_cmd image inspect --format "{{.Id}}" "${IMAGE}" | env sed "s/^sha256://" > "${WORKDIR}/expected-image" + podman_cmd inspect --format "{{.Image}}" "${id}" | env sed "s/^sha256://" > "${WORKDIR}/actual-image" + env cmp -s "${WORKDIR}/expected-image" "${WORKDIR}/actual-image" || fail "selected sandbox image ID differs" + podman_cmd inspect --format "{{index .Config.Labels \"openshell.managed\"}}" "${id}" | env grep -Fx true >/dev/null || fail "managed label missing" + podman_cmd inspect --format "{{index .Config.Labels \"openshell.ai/sandbox-name\"}}" "${id}" | env grep -Fx "${NAME}" >/dev/null || fail "sandbox name label missing" + for label in openshell.ai/sandbox-id openshell.ai/sandbox-workspace; do + podman_cmd inspect --format "{{index .Config.Labels \"${label}\"}}" "${id}" | env grep -Ev "^(|)$" >/dev/null || fail "${label} missing" + done + actual_pids_limit="$(podman_cmd inspect --format "{{.HostConfig.PidsLimit}}" "${id}")" + [ "${actual_pids_limit}" = "${EXPECTED_PIDS_LIMIT}" ] || fail "pids limit is ${actual_pids_limit}, expected ${EXPECTED_PIDS_LIMIT}" + podman_cmd inspect --format "{{.HostConfig.CpuQuota}}" "${id}" | env grep -Fx 75000 >/dev/null || fail "CPU quota is not 750m" + podman_cmd inspect --format "{{.HostConfig.CpuPeriod}}" "${id}" | env grep -Fx 100000 >/dev/null || fail "CPU period is not 100000" + podman_cmd inspect --format "{{.HostConfig.Memory}}" "${id}" | env grep -Fx 402653184 >/dev/null || fail "memory limit is not 384Mi" + podman_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/tmp/parity-bind"}}{{.RW}}{{end}}{{end}}' "${id}" \ + | env grep -Fx false >/dev/null || fail "bind mount is not read-only" + podman_cmd inspect --format "{{.Config.Entrypoint}}" "${id}" | env grep -F /opt/openshell/bin/openshell-sandbox >/dev/null || fail "supervisor entrypoint missing" + podman_cmd inspect --format "{{index .Config.Cmd 0}} {{index .Config.Cmd 1}}" "${id}" | env grep -Fx -- "--workdir /sandbox" >/dev/null || fail "supervisor workdir differs" + podman_cmd inspect --format "{{range .Config.Env}}{{println .}}{{end}}" "${id}" | env grep -Fx OPENSHELL_SSH_SOCKET_PATH=/run/openshell/parity-ssh.sock >/dev/null || fail "SSH environment differs" + podman_cmd inspect --format "{{range .Config.Env}}{{println .}}{{end}}" "${id}" | env grep -E "^OPENSHELL_ENDPOINT=https://host\.containers\.internal:" >/dev/null || fail "callback endpoint differs" +done < "${WORKDIR}/ids" +# Both schema spellings must map to Podman's pull-if-missing request. Inspect +# the driver emission so regressions to always or never do not pass merely +# because the wrapper preloaded the image. +sed $'s/\033\[[0-9;]*m//g' "${GATEWAY_LOG}" \ + | env grep -F 'Ensuring sandbox image' \ + | env grep -F 'policy=missing' >/dev/null \ + || fail "image pull policy did not map to Podman missing" + +# Podman 5.8 reports Healthcheck.Interval in nanoseconds; wait for the +# eventual state instead of accepting a merely running container. +healthy=0 +for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90; do + while IFS= read -r id; do + podman_cmd inspect --format "{{.Config.Healthcheck.Interval}}" "${id}" | env grep -E "^(7000000000|7s)$" >/dev/null || fail "health interval is not 7 seconds" + if podman_cmd inspect --format "{{.State.Health.Status}}" "${id}" | env grep -Fx healthy >/dev/null; then healthy=1; fi + done < "${WORKDIR}/ids" + [ "${healthy}" = 1 ] && break + sleep 1 +done +[ "${healthy}" = 1 ] || fail "container did not become healthy" +container_id="$(cat "${WORKDIR}/ids")" +podman_cmd exec "${container_id}" sh -c 'test "$(cat /tmp/parity-bind/probe)" = parity-bind-mount' \ + || fail "read-only bind mount is unavailable" +podman_cmd exec "${container_id}" test -d /tmp/parity-cache \ + || fail "tmpfs mount is unavailable" +podman_cmd exec "${container_id}" test -s /etc/openshell/auth/sandbox.jwt \ + || fail "sandbox token mount is unavailable" +for tls_file in ca.crt tls.crt tls.key; do + podman_cmd exec "${container_id}" test -s "/etc/openshell/tls/client/${tls_file}" \ + || fail "guest TLS mount ${tls_file} is unavailable" +done +"${CLI}" sandbox exec --name "${NAME}" --no-tty --no-login-shell -- true + +# This is the normalized result: no container IDs, timestamps, IPs, or ports. +escaped_image="$(json_escape "${IMAGE}")" +printf "%s\n" "{\"scenario\":\"podman-options\",\"sandbox_image\":\"${escaped_image}\",\"image_pull_policy\":\"if_not_present\",\"managed_labels\":true,\"supervisor_entrypoint\":\"/opt/openshell/bin/openshell-sandbox\",\"supervisor_workdir\":\"/sandbox\",\"callback_endpoint_scheme\":\"https\",\"callback_endpoint_host\":\"host.containers.internal\",\"ssh_socket_path\":\"/run/openshell/parity-ssh.sock\",\"cpu_millis\":750,\"memory_bytes\":402653184,\"pids_limit\":${actual_pids_limit},\"bind_mount\":\"read_only\",\"tmpfs_mount\":true,\"sandbox_token_mount\":true,\"guest_tls_mounts\":true,\"health_check_interval_secs\":7,\"health\":\"healthy\",\"callback_exec\":true}" > "${RESULT}" diff --git a/e2e/parity/run.sh b/e2e/parity/run.sh new file mode 100755 index 0000000000..b83784d7f7 --- /dev/null +++ b/e2e/parity/run.sh @@ -0,0 +1,620 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compare the frozen schema-v1 gateway contract with the checkout's schema-v2 +# contract. This intentionally begins with one small semantic scenario; later +# parity waves add scenarios without changing the isolated variant runner. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MANIFEST="${OPENSHELL_PARITY_CAPABILITY_MANIFEST:-${ROOT}/e2e/configs/gateway/schema-v2-capability-parity.toml}" +DRIVER="" +SCENARIO="smoke" +COMMAND_CLASS="conformance_smoke" +BASELINE_WORKTREE="${OPENSHELL_PARITY_BASELINE_WORKTREE:-}" +CANDIDATE_WORKTREE="${OPENSHELL_PARITY_CANDIDATE_WORKTREE:-${ROOT}}" +RESULTS_DIR="${OPENSHELL_PARITY_RESULTS_DIR:-}" +WRAPPER="${OPENSHELL_PARITY_PODMAN_WRAPPER:-${ROOT}/e2e/with-podman-gateway.sh}" +PODMAN_OPTIONS_ORACLE="${OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE:-${ROOT}/e2e/parity/podman-options.sh}" +PODMAN_BIN="${OPENSHELL_PARITY_PODMAN_BIN:-podman}" +DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}" +PARITY_SANDBOX_RUNTIME_IMAGE="" +PARITY_SANDBOX_COMMUNITY_REGISTRY="" +PARITY_SUPERVISOR_BASE_IMAGE="" +PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE="" +TEMP_WORKTREE="" +RUN_DIR="" + +usage() { + cat >&2 <&2; exit 2; } + DRIVER=$2 + shift 2 + ;; + --scenario) + [ "$#" -ge 2 ] || { echo "ERROR: --scenario requires a value." >&2; exit 2; } + SCENARIO=$2 + shift 2 + ;; + --baseline-worktree) + [ "$#" -ge 2 ] || { echo "ERROR: --baseline-worktree requires a path." >&2; exit 2; } + BASELINE_WORKTREE=$2 + shift 2 + ;; + --results-dir) + [ "$#" -ge 2 ] || { echo "ERROR: --results-dir requires a path." >&2; exit 2; } + RESULTS_DIR=$2 + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; usage; exit 2 ;; + esac +done + +case "${SCENARIO}" in + smoke) COMMAND_CLASS="conformance_smoke" ;; + external-driver) COMMAND_CLASS="external_driver_conformance_smoke" ;; + podman-options) COMMAND_CLASS="podman_options" ;; + *) echo "ERROR: unsupported parity scenario: ${SCENARIO}." >&2; exit 2 ;; +esac + +if [ "${DRIVER}" != "podman" ]; then + echo "ERROR: only --driver podman is supported by the schema parity harness (got ${DRIVER:-})." >&2 + echo " Docker, Kubernetes, and VM backends are reserved for later parity waves." >&2 + exit 2 +fi + +if [ ! -f "${MANIFEST}" ]; then + echo "ERROR: parity capability manifest not found: ${MANIFEST}" >&2 + exit 2 +fi +BASELINE_SHA="$(awk -F '"' '/^[[:space:]]*baseline_commit[[:space:]]*=/ { print $2; exit }' "${MANIFEST}")" +if ! [[ "${BASELINE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: manifest baseline_commit must be a full 40-character SHA: ${MANIFEST}" >&2 + exit 2 +fi +BASELINE_SHA="${BASELINE_SHA,,}" +EXPECTED_CANDIDATE_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +if [ ! -d "${CANDIDATE_WORKTREE}" ]; then + echo "ERROR: candidate worktree does not exist: ${CANDIDATE_WORKTREE}" >&2 + exit 2 +fi +CANDIDATE_SHA="$(git -C "${CANDIDATE_WORKTREE}" rev-parse HEAD 2>/dev/null || true)" +if [ "${CANDIDATE_SHA}" != "${EXPECTED_CANDIDATE_SHA}" ]; then + echo "ERROR: candidate worktree must be at current commit ${EXPECTED_CANDIDATE_SHA}: ${CANDIDATE_WORKTREE}" >&2 + exit 2 +fi + +cleanup() { + local status=$? + if [ -n "${TEMP_WORKTREE}" ]; then + git -C "${ROOT}" worktree remove --force "${TEMP_WORKTREE}" >/dev/null 2>&1 || true + fi + if [ -n "${RUN_DIR}" ]; then + # Rootless Podman overlay files can be owned by subordinate UIDs. Remove an + # isolated container store from Podman's user namespace before falling back + # to ordinary cleanup for runs that never reached the container runtime. + if { [ -d "${RUN_DIR}/baseline/data/containers/storage" ] \ + || [ -d "${RUN_DIR}/candidate/data/containers/storage" ] \ + || [ -d "${RUN_DIR}/sandbox-resolver/data/containers/storage" ]; } \ + && command -v "${PODMAN_BIN}" >/dev/null 2>&1; then + "${PODMAN_BIN}" unshare rm -rf -- "${RUN_DIR}" >/dev/null 2>&1 || true + fi + rm -rf "${RUN_DIR}" >/dev/null 2>&1 || true + fi + exit "${status}" +} +trap cleanup EXIT + +if [ -n "${BASELINE_WORKTREE}" ]; then + if [ ! -d "${BASELINE_WORKTREE}" ]; then + echo "ERROR: baseline worktree does not exist: ${BASELINE_WORKTREE}" >&2 + exit 2 + fi + resolved_baseline_sha="$(git -C "${BASELINE_WORKTREE}" rev-parse HEAD 2>/dev/null || true)" + if [ "${resolved_baseline_sha}" != "${BASELINE_SHA}" ]; then + echo "ERROR: baseline worktree is not frozen manifest commit ${BASELINE_SHA}: ${BASELINE_WORKTREE}" >&2 + exit 2 + fi +else + if ! git -C "${ROOT}" cat-file -e "${BASELINE_SHA}^{commit}" 2>/dev/null; then + echo "ERROR: frozen baseline ${BASELINE_SHA} is unavailable locally; fetch it before running parity." >&2 + exit 2 + fi + TEMP_WORKTREE="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-baseline.XXXXXX")" + # Remove mktemp's directory so git worktree can create and register it. + rmdir "${TEMP_WORKTREE}" + git -C "${ROOT}" worktree add --detach "${TEMP_WORKTREE}" "${BASELINE_SHA}" >/dev/null + BASELINE_WORKTREE="${TEMP_WORKTREE}" +fi + +require_clean_source() { + local variant=$1 source_root=$2 dirty + dirty="$(git -C "${source_root}" status --porcelain=v1 --untracked-files=all)" + if [ -n "${dirty}" ]; then + echo "ERROR: ${variant} source worktree must be clean before building parity artifacts: ${source_root}" >&2 + printf '%s\n' "${dirty}" >&2 + exit 2 + fi +} + +RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-run.XXXXXX")" +RESULTS_DIR="${RESULTS_DIR:-${ROOT}/target/parity/results}" +mkdir -p "${RESULTS_DIR}" +RESULTS_DIR="$(cd "${RESULTS_DIR}" && pwd)" + +require_executable() { + local label=$1 + local binary=$2 + if [ ! -x "${binary}" ]; then + echo "ERROR: ${label} binary is not executable: ${binary}" >&2 + exit 2 + fi +} + +build_variant() { + local variant=$1 source_root=$2 target_dir=$3 gateway_override=$4 cli_override=$5 conformance_override=$6 + local gateway_var=$7 cli_var=$8 conformance_var=$9 + local gateway cli conformance jobs=() gateway_features=() + + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then jobs=(-j "${CARGO_BUILD_JOBS}"); fi + if [ "${SCENARIO}" = external-driver ]; then + gateway_features=(--no-default-features --features telemetry) + fi + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + gateway="${gateway_override:-${target_dir}/debug/openshell-gateway}" + cli="${cli_override:-${target_dir}/debug/openshell}" + conformance="${conformance_override:-${target_dir}/debug/openshell-conformance}" + + if [ -z "${gateway_override}" ]; then + require_clean_source "${variant}" "${source_root}" + echo "Building ${variant} gateway in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-gateway --bin openshell-gateway "${gateway_features[@]}") + fi + if [ -z "${cli_override}" ]; then + require_clean_source "${variant}" "${source_root}" + echo "Building ${variant} CLI in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-cli) + fi + if [ -z "${conformance_override}" ]; then + require_clean_source "${variant}" "${source_root}" + echo "Building ${variant} conformance CLI in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" -p openshell-conformance-cli) + fi + require_executable "${variant} gateway" "${gateway}" + require_executable "${variant} CLI" "${cli}" + require_executable "${variant} conformance" "${conformance}" + printf -v "${gateway_var}" '%s' "${gateway}" + printf -v "${cli_var}" '%s' "${cli}" + printf -v "${conformance_var}" '%s' "${conformance}" +} + +BASELINE_GATEWAY="" BASELINE_CLI="" BASELINE_CONFORMANCE="" +CANDIDATE_GATEWAY="" CANDIDATE_CLI="" CANDIDATE_CONFORMANCE="" +build_variant baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" BASELINE_GATEWAY BASELINE_CLI BASELINE_CONFORMANCE +build_variant candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" CANDIDATE_GATEWAY CANDIDATE_CLI CANDIDATE_CONFORMANCE + +BASELINE_GATEWAY_ORIGIN=built_by_harness +BASELINE_CLI_ORIGIN=built_by_harness +BASELINE_CONFORMANCE_ORIGIN=built_by_harness +CANDIDATE_GATEWAY_ORIGIN=built_by_harness +CANDIDATE_CLI_ORIGIN=built_by_harness +CANDIDATE_CONFORMANCE_ORIGIN=built_by_harness +[ -z "${OPENSHELL_PARITY_BASELINE_GATEWAY_BIN:-}" ] || BASELINE_GATEWAY_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_BASELINE_CLI_BIN:-}" ] || BASELINE_CLI_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN:-}" ] || BASELINE_CONFORMANCE_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN:-}" ] || CANDIDATE_GATEWAY_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_CLI_BIN:-}" ] || CANDIDATE_CLI_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN:-}" ] || CANDIDATE_CONFORMANCE_ORIGIN=supplied_override + +build_external_driver() { + local variant=$1 source_root=$2 target_dir=$3 override=$4 output_var=$5 + local binary + if [ "${SCENARIO}" != external-driver ]; then + printf -v "${output_var}" '%s' "" + return + fi + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + binary="${override:-${target_dir}/debug/openshell-driver-podman}" + if [ -z "${override}" ]; then + require_clean_source "${variant}" "${source_root}" + echo "Building ${variant} external Podman driver in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build -p openshell-driver-podman --bin openshell-driver-podman) + fi + require_executable "${variant} external Podman driver" "${binary}" + printf -v "${output_var}" '%s' "${binary}" +} + +BASELINE_EXTERNAL_DRIVER="" CANDIDATE_EXTERNAL_DRIVER="" +BASELINE_EXTERNAL_DRIVER_ORIGIN=not_applicable +CANDIDATE_EXTERNAL_DRIVER_ORIGIN=not_applicable +if [ "${SCENARIO}" = external-driver ]; then + BASELINE_EXTERNAL_DRIVER_ORIGIN=built_by_harness + CANDIDATE_EXTERNAL_DRIVER_ORIGIN=built_by_harness + [ -z "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" ] || BASELINE_EXTERNAL_DRIVER_ORIGIN=supplied_override + [ -z "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" ] || CANDIDATE_EXTERNAL_DRIVER_ORIGIN=supplied_override +fi +build_external_driver baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN:-}" BASELINE_EXTERNAL_DRIVER +build_external_driver candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN:-}" CANDIDATE_EXTERNAL_DRIVER +if [ "${SCENARIO}" = external-driver ]; then + baseline_external_realpath="$(realpath "${BASELINE_EXTERNAL_DRIVER}")" + candidate_external_realpath="$(realpath "${CANDIDATE_EXTERNAL_DRIVER}")" + if [ "${baseline_external_realpath}" = "${candidate_external_realpath}" ] \ + || [ "${BASELINE_EXTERNAL_DRIVER}" -ef "${CANDIDATE_EXTERNAL_DRIVER}" ]; then + echo "ERROR: external-driver parity requires distinct baseline and candidate driver artifacts." >&2 + exit 2 + fi +fi + +supervisor_target_triple() { + case "$(uname -sm)" in + "Linux x86_64") printf '%s\n' x86_64-unknown-linux-musl ;; + "Linux aarch64"|"Linux arm64") printf '%s\n' aarch64-unknown-linux-musl ;; + *) echo "ERROR: Podman parity supervisor builds require Linux x86_64 or arm64." >&2; return 2 ;; + esac +} + +build_supervisor() { + local variant=$1 source_root=$2 target_dir=$3 override=$4 output_var=$5 + local binary target jobs=() + target="$(supervisor_target_triple)" + target_dir="${target_dir:-${ROOT}/target/parity/${variant}}" + case "${target_dir}" in /*) ;; *) target_dir="${ROOT}/${target_dir}" ;; esac + binary="${override:-${target_dir}/${target}/release/openshell-sandbox}" + if [ -z "${override}" ]; then + require_clean_source "${variant}" "${source_root}" + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then jobs=(-j "${CARGO_BUILD_JOBS}"); fi + echo "Building ${variant} supervisor in ${target_dir}..." + (cd "${source_root}" && CARGO_TARGET_DIR="${target_dir}" cargo build "${jobs[@]}" --release --target "${target}" -p openshell-sandbox --bin openshell-sandbox) + "${source_root}/tasks/scripts/verify-static-binary.sh" "${binary}" + fi + require_executable "${variant} supervisor" "${binary}" + printf -v "${output_var}" '%s' "${binary}" +} + +BASELINE_SUPERVISOR="" CANDIDATE_SUPERVISOR="" +BASELINE_SUPERVISOR_ORIGIN=built_by_harness +CANDIDATE_SUPERVISOR_ORIGIN=built_by_harness +[ -z "${OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN:-}" ] || BASELINE_SUPERVISOR_ORIGIN=supplied_override +[ -z "${OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN:-}" ] || CANDIDATE_SUPERVISOR_ORIGIN=supplied_override +build_supervisor baseline "${BASELINE_WORKTREE}" "${OPENSHELL_PARITY_BASELINE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN:-}" BASELINE_SUPERVISOR +build_supervisor candidate "${CANDIDATE_WORKTREE}" "${OPENSHELL_PARITY_CANDIDATE_CARGO_TARGET_DIR:-}" "${OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN:-}" CANDIDATE_SUPERVISOR + +stage_artifact() { + local variant=$1 role=$2 source=$3 mode=$4 path_var=$5 digest_var=$6 + local destination="${RESULTS_DIR}/artifacts/${variant}/${role}" + mkdir -p "$(dirname "${destination}")" + install -m "${mode}" "${source}" "${destination}" + printf -v "${path_var}" '%s' "${destination}" + printf -v "${digest_var}" '%s' "$(sha256sum "${destination}" | cut -d' ' -f1)" +} + +stage_executable() { + stage_artifact "$1" "$2" "$3" 0555 "$4" "$5" +} + +BASELINE_GATEWAY_DIGEST="" BASELINE_CLI_DIGEST="" BASELINE_CONFORMANCE_DIGEST="" BASELINE_EXTERNAL_DRIVER_DIGEST="" BASELINE_SUPERVISOR_DIGEST="" BASELINE_SUPERVISOR_DOCKERFILE="" BASELINE_SUPERVISOR_DOCKERFILE_DIGEST="" BASELINE_CLI_TRACE_WRAPPER="" BASELINE_CLI_TRACE_WRAPPER_DIGEST="" +CANDIDATE_GATEWAY_DIGEST="" CANDIDATE_CLI_DIGEST="" CANDIDATE_CONFORMANCE_DIGEST="" CANDIDATE_EXTERNAL_DRIVER_DIGEST="" CANDIDATE_SUPERVISOR_DIGEST="" CANDIDATE_SUPERVISOR_DOCKERFILE="" CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST="" CANDIDATE_CLI_TRACE_WRAPPER="" CANDIDATE_CLI_TRACE_WRAPPER_DIGEST="" +stage_executable baseline gateway "${BASELINE_GATEWAY}" BASELINE_GATEWAY BASELINE_GATEWAY_DIGEST +stage_executable baseline cli "${BASELINE_CLI}" BASELINE_CLI BASELINE_CLI_DIGEST +stage_executable baseline conformance "${BASELINE_CONFORMANCE}" BASELINE_CONFORMANCE BASELINE_CONFORMANCE_DIGEST +stage_executable baseline supervisor "${BASELINE_SUPERVISOR}" BASELINE_SUPERVISOR BASELINE_SUPERVISOR_DIGEST +stage_artifact baseline supervisor.Dockerfile "${BASELINE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 BASELINE_SUPERVISOR_DOCKERFILE BASELINE_SUPERVISOR_DOCKERFILE_DIGEST +stage_artifact baseline cli-trace-wrapper "${ROOT}/e2e/parity/trace-cli.sh" 0555 BASELINE_CLI_TRACE_WRAPPER BASELINE_CLI_TRACE_WRAPPER_DIGEST +stage_executable candidate gateway "${CANDIDATE_GATEWAY}" CANDIDATE_GATEWAY CANDIDATE_GATEWAY_DIGEST +stage_executable candidate cli "${CANDIDATE_CLI}" CANDIDATE_CLI CANDIDATE_CLI_DIGEST +stage_executable candidate conformance "${CANDIDATE_CONFORMANCE}" CANDIDATE_CONFORMANCE CANDIDATE_CONFORMANCE_DIGEST +stage_executable candidate supervisor "${CANDIDATE_SUPERVISOR}" CANDIDATE_SUPERVISOR CANDIDATE_SUPERVISOR_DIGEST +stage_artifact candidate supervisor.Dockerfile "${CANDIDATE_WORKTREE}/deploy/docker/Dockerfile.supervisor" 0444 CANDIDATE_SUPERVISOR_DOCKERFILE CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST +stage_artifact candidate cli-trace-wrapper "${ROOT}/e2e/parity/trace-cli.sh" 0555 CANDIDATE_CLI_TRACE_WRAPPER CANDIDATE_CLI_TRACE_WRAPPER_DIGEST +if [ "${SCENARIO}" = external-driver ]; then + stage_executable baseline external-driver "${BASELINE_EXTERNAL_DRIVER}" BASELINE_EXTERNAL_DRIVER BASELINE_EXTERNAL_DRIVER_DIGEST + stage_executable candidate external-driver "${CANDIDATE_EXTERNAL_DRIVER}" CANDIDATE_EXTERNAL_DRIVER CANDIDATE_EXTERNAL_DRIVER_DIGEST + if [ "${BASELINE_EXTERNAL_DRIVER_DIGEST}" = "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" ]; then + echo "ERROR: external-driver parity requires different baseline and candidate driver content." >&2 + exit 2 + fi +fi + +require_executable "Podman parity wrapper" "${WRAPPER}" +if [ "${SCENARIO}" = "podman-options" ] && [ ! -f "${PODMAN_OPTIONS_ORACLE}" ]; then + echo "ERROR: Podman options oracle does not exist: ${PODMAN_OPTIONS_ORACLE}" >&2 + exit 2 +fi + +podman_in_resolver_store() { + local resolver_home="${RUN_DIR}/sandbox-resolver" + mkdir -p "${resolver_home}/config" "${resolver_home}/state" \ + "${resolver_home}/cache" "${resolver_home}/data" + env -u CONTAINER_HOST -u CONTAINER_CONNECTION -u CONTAINERS_STORAGE_CONF \ + -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ + -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ + XDG_CONFIG_HOME="${resolver_home}/config" \ + XDG_STATE_HOME="${resolver_home}/state" \ + XDG_CACHE_HOME="${resolver_home}/cache" \ + XDG_DATA_HOME="${resolver_home}/data" \ + "${PODMAN_BIN}" "$@" +} + +resolve_parity_sandbox_image() { + local image_id image_digest repository + if [ -n "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" ] \ + && ! [[ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE must be digest-pinned for parity runs." >&2 + exit 2 + fi + case "${SANDBOX_IMAGE_REQUEST}" in + */*) ;; + *) + echo "ERROR: parity sandbox image must use a fully qualified repository: ${SANDBOX_IMAGE_REQUEST}" >&2 + exit 2 + ;; + esac + echo "Resolving parity sandbox image once: ${SANDBOX_IMAGE_REQUEST}" + podman_in_resolver_store pull "${SANDBOX_IMAGE_REQUEST}" >/dev/null + image_id="$(podman_in_resolver_store image inspect --format '{{.Id}}' "${SANDBOX_IMAGE_REQUEST}")" + image_id="${image_id#sha256:}" + image_digest="$(podman_in_resolver_store image inspect --format '{{.Digest}}' "${SANDBOX_IMAGE_REQUEST}")" + repository="${SANDBOX_IMAGE_REQUEST%%@*}" + case "${repository##*/}" in + *:*) repository="${repository%:*}" ;; + esac + PARITY_SANDBOX_RUNTIME_IMAGE="${repository}@${image_digest}" + if ! [[ "${image_id}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${PARITY_SANDBOX_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve one immutable parity sandbox image from ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 + fi + PARITY_SANDBOX_COMMUNITY_REGISTRY="${repository%/base}" + if [ "${PARITY_SANDBOX_COMMUNITY_REGISTRY}" = "${repository}" ]; then + echo "ERROR: parity sandbox repository must end in /base for the conformance alias: ${repository}." >&2 + exit 2 + fi + echo "Using one immutable parity sandbox image for both variants: ${PARITY_SANDBOX_RUNTIME_IMAGE} (ID ${image_id})" +} + +resolve_parity_supervisor_base_image() { + local baseline_base candidate_base + baseline_base="$(awk '$1 == "FROM" { print $2; exit }' "${BASELINE_SUPERVISOR_DOCKERFILE}")" + candidate_base="$(awk '$1 == "FROM" { print $2; exit }' "${CANDIDATE_SUPERVISOR_DOCKERFILE}")" + if [ -z "${baseline_base}" ] || [ "${baseline_base}" != "${candidate_base}" ]; then + echo "ERROR: parity supervisor Dockerfiles must select the same base image." >&2 + exit 2 + fi + PARITY_SUPERVISOR_BASE_IMAGE="${baseline_base}" + echo "Resolving parity supervisor base image once: ${PARITY_SUPERVISOR_BASE_IMAGE}" + podman_in_resolver_store pull "${PARITY_SUPERVISOR_BASE_IMAGE}" >/dev/null + PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE="$(podman_in_resolver_store image inspect --format '{{index .RepoDigests 0}}' "${PARITY_SUPERVISOR_BASE_IMAGE}")" + if ! [[ "${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve one immutable supervisor base image from ${PARITY_SUPERVISOR_BASE_IMAGE}." >&2 + exit 2 + fi + echo "Using one immutable supervisor base image for both variants: ${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" +} + +resolve_parity_sandbox_image +resolve_parity_supervisor_base_image + +write_result() { + local variant=$1 source_sha=$2 schema=$3 status=$4 + local gateway_digest=$5 cli_digest=$6 conformance_digest=$7 external_driver_digest_value=$8 supervisor_digest=$9 supervisor_dockerfile_digest=${10} cli_trace_wrapper_digest=${11} + local normalized_result="" external_driver_digest="" gateway_profile="in-tree" + local gateway_features=default + local gateway_origin cli_origin conformance_origin external_driver_origin supervisor_origin + if [ "${variant}" = baseline ]; then + gateway_origin=${BASELINE_GATEWAY_ORIGIN} + cli_origin=${BASELINE_CLI_ORIGIN} + conformance_origin=${BASELINE_CONFORMANCE_ORIGIN} + external_driver_origin=${BASELINE_EXTERNAL_DRIVER_ORIGIN} + supervisor_origin=${BASELINE_SUPERVISOR_ORIGIN} + else + gateway_origin=${CANDIDATE_GATEWAY_ORIGIN} + cli_origin=${CANDIDATE_CLI_ORIGIN} + conformance_origin=${CANDIDATE_CONFORMANCE_ORIGIN} + external_driver_origin=${CANDIDATE_EXTERNAL_DRIVER_ORIGIN} + supervisor_origin=${CANDIDATE_SUPERVISOR_ORIGIN} + fi + if [ -n "${external_driver_digest_value}" ]; then + external_driver_digest=",\"external_driver_sha256\":\"${external_driver_digest_value}\"" + gateway_profile="driver-free" + gateway_features="--no-default-features --features telemetry" + fi + if [ "${SCENARIO}" = "podman-options" ]; then normalized_result=",\"normalized_result\":\"${variant}.normalized.json\""; fi + cat >"${RESULTS_DIR}/${variant}.json" <"${RUN_DIR}/baseline.semantic" + sed -E 's/"pids_limit":[0-9]+/"pids_limit":IGNORED/' "${RESULTS_DIR}/candidate.normalized.json" >"${RUN_DIR}/candidate.semantic" + if grep -F '"pids_limit":2048' "${RESULTS_DIR}/baseline.normalized.json" >/dev/null \ + && grep -F '"pids_limit":31' "${RESULTS_DIR}/candidate.normalized.json" >/dev/null \ + && cmp -s "${RUN_DIR}/baseline.semantic" "${RUN_DIR}/candidate.semantic"; then + COMPARISON_ACCEPTED=true + COMPARISON_CLASSIFICATION="intentional_change" + intentional_change_id='"podman-pid-limit-restored"' + fi + fi + fi + cat >"${RESULTS_DIR}/comparison.json" <&2 + return 1 + fi +} + +run_variant() { + local variant=$1 source_sha=$2 schema=$3 gateway=$4 cli=$5 conformance=$6 external_driver=$7 supervisor=$8 supervisor_dockerfile=$9 + local gateway_digest=${10} cli_digest=${11} conformance_digest=${12} external_driver_digest=${13} supervisor_digest=${14} supervisor_dockerfile_digest=${15} cli_trace_wrapper=${16} cli_trace_wrapper_digest=${17} + local result_status variant_home="${RUN_DIR}/${variant}" + local supervisor_image="localhost/openshell/supervisor:parity-${variant}-${source_sha:0:12}" + mkdir -p "${variant_home}/config" "${variant_home}/state" "${variant_home}/cache" "${variant_home}/data" + echo "==> schema parity ${variant} (schema v${schema}, ${DRIVER}, ${SCENARIO})" + local option_profile="" + local -a command + if [ "${SCENARIO}" = "podman-options" ]; then + option_profile="podman-options" + command=(bash "${PODMAN_OPTIONS_ORACLE}") + else + command=("${conformance}" run --openshell-bin "${cli_trace_wrapper}" --output json) + fi + verify_artifact_digest "${variant} gateway before execution" "${gateway}" "${gateway_digest}" || return 1 + verify_artifact_digest "${variant} CLI before execution" "${cli}" "${cli_digest}" || return 1 + verify_artifact_digest "${variant} conformance CLI before execution" "${conformance}" "${conformance_digest}" || return 1 + verify_artifact_digest "${variant} supervisor before execution" "${supervisor}" "${supervisor_digest}" || return 1 + verify_artifact_digest "${variant} supervisor Dockerfile before execution" "${supervisor_dockerfile}" "${supervisor_dockerfile_digest}" || return 1 + verify_artifact_digest "${variant} CLI trace wrapper before execution" "${cli_trace_wrapper}" "${cli_trace_wrapper_digest}" || return 1 + if [ -n "${external_driver}" ]; then + verify_artifact_digest "${variant} external driver before execution" "${external_driver}" "${external_driver_digest}" || return 1 + fi + if env -u OPENSHELL_GATEWAY_ENDPOINT -u OPENSHELL_GATEWAY_CONFIG \ + -u OPENSHELL_COMPUTE_DRIVER -u OPENSHELL_COMPUTE_DRIVER_SOCKET -u OPENSHELL_DRIVERS \ + -u OPENSHELL_PODMAN_SOCKET \ + -u CONTAINER_HOST -u CONTAINER_CONNECTION -u CONTAINERS_STORAGE_CONF \ + -u CONTAINERS_CONF -u CONTAINERS_REGISTRIES_CONF -u CONTAINERS_REGISTRIES_CONF_DIR \ + -u CONTAINERS_POLICY -u PODMAN_CONNECTIONS_CONF -u DOCKER_HOST \ + -u OPENSHELL_SANDBOX_IMAGE -u OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE \ + -u OPENSHELL_GRPC_ENDPOINT -u OPENSHELL_PODMAN_HOST_GATEWAY_IP \ + -u OPENSHELL_PODMAN_USERNS -u OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET \ + -u OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET -u OPENSHELL_APP_ARMOR_PROFILE \ + -u OPENSHELL_SANDBOX_HTTPS_PROXY -u OPENSHELL_SANDBOX_NO_PROXY \ + -u OPENSHELL_SANDBOX_PROXY_AUTH_FILE -u OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE \ + -u OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME -u OPENSHELL_SANDBOX_PROXY_CA_BUNDLE \ + -u OPENSHELL_OTLP_ENDPOINT -u OPENSHELL_GATEWAY_NAME -u OPENSHELL_COMPUTE_DRIVER_BIND \ + -u OPENSHELL_COMMUNITY_REGISTRY \ + OPENSHELL_PARITY_VARIANT="${variant}" \ + OPENSHELL_E2E_CONFIG_SCHEMA_VERSION="${schema}" \ + OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER="$([ "${SCENARIO}" = external-driver ] && printf 1 || printf 0)" \ + OPENSHELL_EXTERNAL_DRIVER_BIN="${external_driver}" \ + OPENSHELL_E2E_SUPERVISOR_BIN="${supervisor}" \ + OPENSHELL_E2E_SUPERVISOR_DOCKERFILE="${supervisor_dockerfile}" \ + OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE=1 \ + OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE=1 \ + OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE="${PARITY_SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_COMMUNITY_REGISTRY="${PARITY_SANDBOX_COMMUNITY_REGISTRY}" \ + OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE="${PARITY_SUPERVISOR_BASE_IMAGE}" \ + OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE="${PARITY_SUPERVISOR_BASE_RUNTIME_IMAGE}" \ + OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256="${gateway_digest}" \ + OPENSHELL_E2E_EXPECTED_CLI_SHA256="${cli_digest}" \ + OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256="${conformance_digest}" \ + OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256="${external_driver_digest}" \ + OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256="${supervisor_digest}" \ + OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256="${supervisor_dockerfile_digest}" \ + OPENSHELL_E2E_EXPECTED_CLI_TRACE_WRAPPER_SHA256="${cli_trace_wrapper_digest}" \ + OPENSHELL_PARITY_REAL_CLI="${cli}" \ + OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE="${RESULTS_DIR}/${variant}.exec.stdout" \ + OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE="${RESULTS_DIR}/${variant}.driver.log" \ + OPENSHELL_SUPERVISOR_IMAGE="${supervisor_image}" \ + OPENSHELL_E2E_PODMAN_OPTION_PROFILE="${option_profile}" \ + OPENSHELL_PARITY_ORACLE_RESULT="${RESULTS_DIR}/${variant}.normalized.json" \ + OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE="${RESULTS_DIR}/${variant}.gateway.toml" \ + OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE="${RESULTS_DIR}/${variant}.launch.json" \ + OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE="${RESULTS_DIR}/artifacts/${variant}/supervisor.packages.txt" \ + OPENSHELL_GATEWAY_BIN="${gateway}" \ + OPENSHELL_BIN="${cli}" \ + OPENSHELL_CONFORMANCE_BIN="${conformance}" \ + MISE_TRUSTED_CONFIG_PATHS="${MISE_TRUSTED_CONFIG_PATHS:-${ROOT}}" \ + XDG_CONFIG_HOME="${variant_home}/config" \ + XDG_STATE_HOME="${variant_home}/state" \ + XDG_CACHE_HOME="${variant_home}/cache" \ + XDG_DATA_HOME="${variant_home}/data" \ + "${WRAPPER}" "${command[@]}" \ + 2>&1 | tee "${RESULTS_DIR}/${variant}.log"; then + result_status=true + else + result_status=false + fi + if [ ! -s "${RESULTS_DIR}/${variant}.launch.json" ]; then + echo "ERROR: ${variant} launcher did not emit a launch manifest." >&2 + result_status=false + fi + verify_artifact_digest "${variant} gateway" "${gateway}" "${gateway_digest}" || result_status=false + verify_artifact_digest "${variant} CLI" "${cli}" "${cli_digest}" || result_status=false + verify_artifact_digest "${variant} conformance CLI" "${conformance}" "${conformance_digest}" || result_status=false + verify_artifact_digest "${variant} supervisor" "${supervisor}" "${supervisor_digest}" || result_status=false + verify_artifact_digest "${variant} supervisor Dockerfile" "${supervisor_dockerfile}" "${supervisor_dockerfile_digest}" || result_status=false + verify_artifact_digest "${variant} CLI trace wrapper" "${cli_trace_wrapper}" "${cli_trace_wrapper_digest}" || result_status=false + if [ -n "${external_driver}" ]; then + verify_artifact_digest "${variant} external driver" "${external_driver}" "${external_driver_digest}" || result_status=false + fi + if [ "${SCENARIO}" != podman-options ] \ + && [ ! -s "${RESULTS_DIR}/${variant}.exec.stdout" ]; then + echo "ERROR: ${variant} CLI trace wrapper did not retain exec stdout." >&2 + result_status=false + fi + if [ "${SCENARIO}" = external-driver ] && [ ! -s "${RESULTS_DIR}/${variant}.driver.log" ]; then + echo "ERROR: ${variant} external driver log was not retained." >&2 + result_status=false + fi + write_result "${variant}" "${source_sha}" "${schema}" "${result_status}" "${gateway_digest}" "${cli_digest}" "${conformance_digest}" "${external_driver_digest}" "${supervisor_digest}" "${supervisor_dockerfile_digest}" "${cli_trace_wrapper_digest}" + [ "${result_status}" = true ] +} + +baseline_exit=0 +candidate_exit=0 +run_variant baseline "${BASELINE_SHA}" 1 "${BASELINE_GATEWAY}" "${BASELINE_CLI}" "${BASELINE_CONFORMANCE}" "${BASELINE_EXTERNAL_DRIVER}" "${BASELINE_SUPERVISOR}" "${BASELINE_SUPERVISOR_DOCKERFILE}" "${BASELINE_GATEWAY_DIGEST}" "${BASELINE_CLI_DIGEST}" "${BASELINE_CONFORMANCE_DIGEST}" "${BASELINE_EXTERNAL_DRIVER_DIGEST}" "${BASELINE_SUPERVISOR_DIGEST}" "${BASELINE_SUPERVISOR_DOCKERFILE_DIGEST}" "${BASELINE_CLI_TRACE_WRAPPER}" "${BASELINE_CLI_TRACE_WRAPPER_DIGEST}" || baseline_exit=$? +# Do not short-circuit: a candidate result is useful even when the frozen +# baseline failed, and two equal failures must never constitute parity. +run_variant candidate "${CANDIDATE_SHA}" 2 "${CANDIDATE_GATEWAY}" "${CANDIDATE_CLI}" "${CANDIDATE_CONFORMANCE}" "${CANDIDATE_EXTERNAL_DRIVER}" "${CANDIDATE_SUPERVISOR}" "${CANDIDATE_SUPERVISOR_DOCKERFILE}" "${CANDIDATE_GATEWAY_DIGEST}" "${CANDIDATE_CLI_DIGEST}" "${CANDIDATE_CONFORMANCE_DIGEST}" "${CANDIDATE_EXTERNAL_DRIVER_DIGEST}" "${CANDIDATE_SUPERVISOR_DIGEST}" "${CANDIDATE_SUPERVISOR_DOCKERFILE_DIGEST}" "${CANDIDATE_CLI_TRACE_WRAPPER}" "${CANDIDATE_CLI_TRACE_WRAPPER_DIGEST}" || candidate_exit=$? + +baseline_success=$([ "${baseline_exit}" -eq 0 ] && printf true || printf false) +candidate_success=$([ "${candidate_exit}" -eq 0 ] && printf true || printf false) +write_comparison "${baseline_success}" "${candidate_success}" + +if [ "${COMPARISON_ACCEPTED}" != true ]; then + echo "ERROR: schema parity comparison classified ${SCENARIO} as a regression." >&2 + exit 1 +fi +if [ "${COMPARISON_CLASSIFICATION}" = intentional_change ]; then + echo "Schema parity accepted an intentional change: podman-pid-limit-restored (${SCENARIO})." +else + echo "Schema parity passed: baseline schema v1 and candidate schema v2 succeeded (${SCENARIO})." +fi diff --git a/e2e/parity/test.sh b/e2e/parity/test.sh new file mode 100755 index 0000000000..5a4541930d --- /dev/null +++ b/e2e/parity/test.sh @@ -0,0 +1,380 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Deterministic contract tests for e2e/parity/run.sh. No container runtime is +# invoked; the Podman wrapper and all three artifacts are tiny local fakes. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-test.XXXXXX")" +trap 'rm -rf "${WORKDIR}"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } +assert_contains() { grep -F -- "$2" "$1" >/dev/null || fail "expected $1 to contain: $2"; } +assert_not_contains() { ! grep -F -- "$2" "$1" >/dev/null || fail "expected $1 not to contain: $2"; } +assert_status() { [ "$1" -eq "$2" ] || fail "expected status $2, got $1"; } + +# Schema generator behavior is separately deterministic and does not need a +# gateway, certificates, or Podman. +# shellcheck source=e2e/support/gateway-common.sh +source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck source=e2e/support/podman-gateway-config.sh +source "${ROOT}/e2e/support/podman-gateway-config.sh" +mkdir -p "${WORKDIR}/pki/client" "${WORKDIR}/jwt" +e2e_write_podman_gateway_config "${WORKDIR}/v1.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +e2e_write_podman_gateway_config "${WORKDIR}/v2.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 '' +assert_contains "${WORKDIR}/v1.toml" 'version = 1' +assert_contains "${WORKDIR}/v1.toml" 'compute_drivers = ["podman"]' +assert_contains "${WORKDIR}/v1.toml" 'image_pull_policy = "missing"' +assert_contains "${WORKDIR}/v1.toml" 'health_check_interval_secs = 0' +assert_contains "${WORKDIR}/v1.toml" 'guest_tls_ca = ' +assert_contains "${WORKDIR}/v2.toml" 'version = 2' +assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"' +assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"' +assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0' +# V2 guest TLS is emitted before its driver table; V1 is driver-local. +OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v1-options.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" +OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v2-options.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 "" +for config in "${WORKDIR}/v1-options.toml" "${WORKDIR}/v2-options.toml"; do + assert_contains "${config}" 'sandbox_pids_limit = 31' + assert_contains "${config}" 'health_check_interval_secs = 7' + assert_not_contains "${config}" 'app_armor_profile = ' +done +assert_contains "${WORKDIR}/v1-options.toml" 'sandbox_ssh_socket_path = "/run/openshell/parity-ssh.sock"' +assert_contains "${WORKDIR}/v2-options.toml" 'ssh_socket_path = "/run/openshell/parity-ssh.sock"' +if OPENSHELL_E2E_PODMAN_OPTION_PROFILE=unknown e2e_podman_option_profile >/dev/null 2>&1; then fail 'unknown option profile unexpectedly accepted'; fi + +v1_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v1.toml" | cut -d: -f1)" +v1_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v1.toml" | cut -d: -f1)" +v2_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v2.toml" | cut -d: -f1)" +v2_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v2.toml" | cut -d: -f1)" +[ "${v1_tls_line}" -gt "${v1_driver_line}" ] || fail 'v1 TLS must be driver-local' +[ "${v2_tls_line}" -lt "${v2_driver_line}" ] || fail 'v2 TLS must be gateway-owned' +if OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 e2e_podman_config_schema_version >/dev/null 2>&1; then + fail 'invalid schema version unexpectedly accepted' +fi +set +e +env -u OPENSHELL_GATEWAY_ENDPOINT \ + OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 \ + bash "${ROOT}/e2e/with-podman-gateway.sh" true >"${WORKDIR}/wrapper-schema.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/wrapper-schema.out" 'must be 1 or 2' + +cat >"${WORKDIR}/trace-cli-fixture" <<'EOF' +#!/usr/bin/env bash +printf 'openshell-conformance-tracefixture\n' +printf 'trace fixture stderr\n' >&2 +EOF +chmod +x "${WORKDIR}/trace-cli-fixture" +OPENSHELL_PARITY_REAL_CLI="${WORKDIR}/trace-cli-fixture" \ +OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE="${WORKDIR}/trace-cli.stdout" \ + bash "${ROOT}/e2e/parity/trace-cli.sh" sandbox exec -- echo marker \ + >"${WORKDIR}/trace-cli.forwarded.stdout" \ + 2>"${WORKDIR}/trace-cli.forwarded.stderr" +cmp -s "${WORKDIR}/trace-cli.stdout" "${WORKDIR}/trace-cli.forwarded.stdout" \ + || fail 'CLI trace wrapper did not preserve exact exec stdout' +assert_contains "${WORKDIR}/trace-cli.forwarded.stderr" 'trace fixture stderr' + +HEAD_SHA="$(git -C "${ROOT}" rev-parse HEAD)" +cat >"${WORKDIR}/manifest.toml" <"${WORKDIR}/bin/fake-wrapper" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +for variable in \ + OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER \ + OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET \ + CONTAINER_HOST CONTAINER_CONNECTION CONTAINERS_STORAGE_CONF CONTAINERS_CONF \ + CONTAINERS_REGISTRIES_CONF CONTAINERS_REGISTRIES_CONF_DIR CONTAINERS_POLICY \ + PODMAN_CONNECTIONS_CONF DOCKER_HOST OPENSHELL_SANDBOX_IMAGE \ + OPENSHELL_GRPC_ENDPOINT OPENSHELL_PODMAN_HOST_GATEWAY_IP OPENSHELL_PODMAN_USERNS \ + OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET \ + OPENSHELL_APP_ARMOR_PROFILE OPENSHELL_SANDBOX_HTTPS_PROXY OPENSHELL_SANDBOX_NO_PROXY \ + OPENSHELL_SANDBOX_PROXY_AUTH_FILE OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE \ + OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME OPENSHELL_SANDBOX_PROXY_CA_BUNDLE \ + OPENSHELL_OTLP_ENDPOINT OPENSHELL_GATEWAY_NAME OPENSHELL_COMPUTE_DRIVER_BIND; do + [ -z "${!variable:-}" ] || exit 23 +done +expected_sandbox="ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:$(printf '%064d' 0)" +[ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = 1 ] || exit 24 +[ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" = "${expected_sandbox}" ] || exit 25 +[ "${OPENSHELL_COMMUNITY_REGISTRY:-}" = "ghcr.io/nvidia/openshell-community/sandboxes" ] || exit 28 +expected_base="docker.io/library/alpine@sha256:$(printf '%064d' 0)" +[ "${OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE:-}" = alpine:3.22 ] || exit 26 +[ "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-}" = "${expected_base}" ] || exit 27 +printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS" +mkdir -p "$XDG_DATA_HOME/containers/storage" +case "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" in + 1) pull_policy=missing ;; + 2) pull_policy=if_not_present ;; +esac +transport=in_tree +external=false +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then + transport=remote_uds + external=true +fi +runtime_image="localhost/openshell/supervisor@sha256:$(printf '%064d' 0)" +printf '{"schema_version":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%064d","supervisor_image_digest":"sha256:%064d","supervisor_runtime_image":"%s"}\n' \ + "${OPENSHELL_E2E_CONFIG_SCHEMA_VERSION}" "${external}" "${transport}" "${pull_policy}" \ + "${OPENSHELL_SUPERVISOR_IMAGE}" 0 0 "${runtime_image}" \ + >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" +printf 'fixture-package-1.0-r0\n' >"${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE}" +printf 'fixture exec stdout\n' >"${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then + printf 'fixture external driver log\n' >"${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE}" +fi +if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then + replacement="${OPENSHELL_GATEWAY_BIN}.replacement" + printf '#!/usr/bin/env bash\nexit 0\n# mutated\n' >"${replacement}" + chmod 0555 "${replacement}" + mv "${replacement}" "${OPENSHELL_GATEWAY_BIN}" +fi +if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then + case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac + stable=true + if [ "${OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT:-0}" = 1 ] && [ "${OPENSHELL_PARITY_VARIANT}" = candidate ]; then stable=false; fi + if [ "${OPENSHELL_PARITY_TEST_SKIP_RESULT:-}" != "${OPENSHELL_PARITY_VARIANT}" ]; then + printf '%s\n' "{\"scenario\":\"podman-options\",\"stable\":${stable},\"pids_limit\":${pids}}" > "${OPENSHELL_PARITY_ORACLE_RESULT}" + fi +fi +exec "$@" +EOF +cat >"${WORKDIR}/bin/fake-podman" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"$OPENSHELL_PARITY_TEST_PODMAN_CALLS" +case "$1" in + pull) exit 0 ;; + image) + [ "$2" = inspect ] || exit 19 + case "$4" in + '{{.Id}}') printf 'sha256:%064d\n' 0 ;; + '{{.Digest}}') printf 'sha256:%064d\n' 0 ;; + '{{index .RepoDigests 0}}') printf 'docker.io/library/alpine@sha256:%064d\n' 0 ;; + *) exit 19 ;; + esac + ;; + unshare) + shift + exec "$@" + ;; + *) exit 19 ;; +esac +EOF +cat >"${WORKDIR}/bin/fake-conformance" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = "${OPENSHELL_PARITY_VARIANT:-}" ] || [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = both ]; then + exit 17 +fi +printf '{"untrusted":"raw output is intentionally not normalized"}\n' +EOF +for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver baseline-supervisor candidate-supervisor; do + printf '#!/usr/bin/env bash\n# %s\nexit 0\n' "${artifact}" >"${WORKDIR}/bin/${artifact}" +done +chmod +x "${WORKDIR}/bin/"* + +# shellcheck source=e2e/support/podman-gateway-config.sh +source "${ROOT}/e2e/support/podman-gateway-config.sh" +[ "$(e2e_podman_external_driver_pull_policy 1)" = missing ] || fail 'schema v1 external pull policy mismatch' +[ "$(e2e_podman_external_driver_pull_policy 2)" = if_not_present ] || fail 'schema v2 external pull policy mismatch' + +run_harness() { + OPENSHELL_PARITY_CAPABILITY_MANIFEST="${WORKDIR}/manifest.toml" \ + OPENSHELL_PARITY_BASELINE_WORKTREE="${ROOT}" \ + OPENSHELL_PARITY_PODMAN_WRAPPER="${WORKDIR}/bin/fake-wrapper" \ + OPENSHELL_PARITY_PODMAN_BIN="${WORKDIR}/bin/fake-podman" \ + OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_BASELINE_GATEWAY_BIN="${WORKDIR}/bin/baseline-gateway" \ + OPENSHELL_PARITY_BASELINE_CLI_BIN="${WORKDIR}/bin/baseline-cli" \ + OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN="${WORKDIR}/bin/candidate-gateway" \ + OPENSHELL_PARITY_CANDIDATE_CLI_BIN="${WORKDIR}/bin/candidate-cli" \ + OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \ + OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN="${WORKDIR}/bin/baseline-driver" \ + OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN="${OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE:-${WORKDIR}/bin/candidate-driver}" \ + OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN="${WORKDIR}/bin/baseline-supervisor" \ + OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN="${WORKDIR}/bin/candidate-supervisor" \ + OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \ + OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \ + OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \ + MISE_TRUSTED_CONFIG_PATHS= \ + bash "${ROOT}/e2e/parity/run.sh" --driver podman "$@" +} + +OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:9 \ +OPENSHELL_GATEWAY_CONFIG=/tmp/untrusted.toml \ +OPENSHELL_COMPUTE_DRIVER=wrong \ +OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \ +OPENSHELL_DRIVERS=wrong \ +OPENSHELL_PODMAN_SOCKET=/tmp/untrusted-podman.sock \ +CONTAINER_HOST=tcp://untrusted.invalid:9999 \ +CONTAINER_CONNECTION=untrusted \ +CONTAINERS_STORAGE_CONF=/tmp/untrusted-storage.conf \ +CONTAINERS_CONF=/tmp/untrusted-containers.conf \ +CONTAINERS_REGISTRIES_CONF=/tmp/untrusted-registries.conf \ +CONTAINERS_REGISTRIES_CONF_DIR=/tmp/untrusted-registries.d \ +CONTAINERS_POLICY=/tmp/untrusted-policy.json \ +PODMAN_CONNECTIONS_CONF=/tmp/untrusted-connections.json \ +DOCKER_HOST=tcp://untrusted.invalid:2375 \ +OPENSHELL_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \ +OPENSHELL_GRPC_ENDPOINT=http://untrusted.invalid:1 \ +OPENSHELL_PODMAN_HOST_GATEWAY_IP=192.0.2.1 \ +OPENSHELL_PODMAN_USERNS=keep-id \ +OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/tmp/untrusted-spiffe.sock \ +OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET=/tmp/untrusted-e2e-spiffe.sock \ +OPENSHELL_APP_ARMOR_PROFILE=Unconfined \ +OPENSHELL_SANDBOX_HTTPS_PROXY=http://untrusted.invalid:8080 \ +OPENSHELL_SANDBOX_NO_PROXY=untrusted.invalid \ +OPENSHELL_SANDBOX_PROXY_AUTH_FILE=/tmp/untrusted-proxy-auth \ +OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE=true \ +OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME=true \ +OPENSHELL_SANDBOX_PROXY_CA_BUNDLE=/tmp/untrusted-proxy-ca \ +OPENSHELL_OTLP_ENDPOINT=http://untrusted.invalid:4317 \ +OPENSHELL_GATEWAY_NAME=untrusted \ +OPENSHELL_COMPUTE_DRIVER_BIND=192.0.2.2:50061 \ +OPENSHELL_COMMUNITY_REGISTRY=untrusted.invalid/community \ + run_harness +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance" +assert_contains "${WORKDIR}/calls" "|${ROOT}" +[ "$(sed -n '1s/|.*//p' "${WORKDIR}/calls")" = baseline ] || fail 'baseline was not invoked first' +[ "$(sed -n '2s/|.*//p' "${WORKDIR}/calls")" = candidate ] || fail 'candidate was not invoked second' +assert_contains "${WORKDIR}/results/baseline.json" "\"source_sha\":\"${HEAD_SHA}\"" +assert_contains "${WORKDIR}/results/baseline.json" '"schema_version":1' +assert_contains "${WORKDIR}/results/candidate.json" '"schema_version":2' +assert_contains "${WORKDIR}/results/candidate.json" "\"source_sha\":\"${HEAD_SHA}\"" +assert_contains "${WORKDIR}/results/candidate.json" '"success":true' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":true' +assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' +assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' +assert_contains "${WORKDIR}/podman-calls" 'pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest' +assert_contains "${WORKDIR}/podman-calls" 'pull alpine:3.22' +assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- ' +assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.' + +set +e +OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \ + run_harness >"${WORKDIR}/mutable-sandbox.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/mutable-sandbox.out" 'must be digest-pinned for parity runs' + +run_harness --scenario external-driver +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/baseline/external-driver|${WORKDIR}/results/artifacts/baseline/supervisor" +assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/candidate/external-driver|${WORKDIR}/results/artifacts/candidate/supervisor" +assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"external-driver"' +assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"external_driver_conformance_smoke"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_profile":"driver-free"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_cargo_features":"--no-default-features --features telemetry"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_image_digest":"sha256:' +assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor@sha256:' +assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"' +assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"conformance_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_origin":"supplied_override"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_dockerfile_sha256"' +assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_sha256"' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"pass"' + +set +e +OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/baseline-driver" \ + run_harness --scenario external-driver >"${WORKDIR}/same-driver.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/same-driver.out" 'requires distinct baseline and candidate driver artifacts' + +cp "${WORKDIR}/bin/baseline-driver" "${WORKDIR}/bin/same-content-driver" +set +e +OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/same-content-driver" \ + run_harness --scenario external-driver >"${WORKDIR}/same-driver-content.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/same-driver-content.out" 'requires different baseline and candidate driver content' + +run_harness --scenario podman-options +assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance|${ROOT}|podman-options" +assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"podman-options"' +assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"podman_options"' +assert_contains "${WORKDIR}/results/baseline.json" '"normalized_result":"baseline.normalized.json"' +assert_contains "${WORKDIR}/results/baseline.normalized.json" '"stable":true' +assert_contains "${WORKDIR}/results/baseline.normalized.json" '"pids_limit":2048' +assert_contains "${WORKDIR}/results/candidate.normalized.json" '"pids_limit":31' +assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output' +assert_contains "${WORKDIR}/results/baseline.log" 'raw output is intentionally not normalized' +assert_contains "${WORKDIR}/results/comparison.json" '"scenario":"podman-options"' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"intentional_change"' +assert_contains "${WORKDIR}/results/comparison.json" '"intentional_change_id":"podman-pid-limit-restored"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":true' + +set +e +OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT=1 run_harness --scenario podman-options >"${WORKDIR}/drift.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false' + +rm -f "${WORKDIR}/results/candidate.normalized.json" +set +e +OPENSHELL_PARITY_TEST_SKIP_RESULT=candidate run_harness --scenario podman-options >"${WORKDIR}/missing-result.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' +assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false' + +set +e +OPENSHELL_PARITY_FAIL_VARIANT=both run_harness >"${WORKDIR}/failure.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/results/baseline.json" '"success":false' +assert_contains "${WORKDIR}/results/candidate.json" '"success":false' +assert_contains "${WORKDIR}/results/comparison.json" '"parity":false' +[ "$(wc -l <"${WORKDIR}/calls")" -eq 12 ] || fail 'candidate did not run after baseline failure' + +set +e +OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT=candidate run_harness >"${WORKDIR}/mutation.out" 2>&1 +status=$? +set -e +assert_status "${status}" 1 +assert_contains "${WORKDIR}/mutation.out" 'candidate gateway changed after it was staged for execution' +assert_contains "${WORKDIR}/results/candidate.json" '"success":false' +assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"' + +set +e +bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/driver.out" 'only --driver podman is supported' + +set +e +bash "${ROOT}/e2e/parity/run.sh" --driver >"${WORKDIR}/option.out" 2>&1 +status=$? +set -e +assert_status "${status}" 2 +assert_contains "${WORKDIR}/option.out" '--driver requires a value' + +echo 'e2e parity deterministic tests passed.' diff --git a/e2e/parity/trace-cli.sh b/e2e/parity/trace-cli.sh new file mode 100755 index 0000000000..372c1b1d3a --- /dev/null +++ b/e2e/parity/trace-cli.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Transparently invoke the staged OpenShell CLI while retaining the exact +# stdout bytes produced by the conformance exec probe. + +set -uo pipefail + +: "${OPENSHELL_PARITY_REAL_CLI:?OPENSHELL_PARITY_REAL_CLI is required}" +: "${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE:?OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE is required}" + +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/openshell-parity-cli.XXXXXX")" +cleanup() { + rm -rf "${tmpdir}" +} +trap cleanup EXIT + +set +e +"${OPENSHELL_PARITY_REAL_CLI}" "$@" >"${tmpdir}/stdout" 2>"${tmpdir}/stderr" +status=$? +set -e + +cat "${tmpdir}/stdout" +cat "${tmpdir}/stderr" >&2 + +if [ "${1:-}" = sandbox ] && [ "${2:-}" = exec ]; then + install -m 0444 "${tmpdir}/stdout" "${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE}" +fi + +exit "${status}" diff --git a/e2e/parity/verify-results.py b/e2e/parity/verify-results.py new file mode 100644 index 0000000000..4ef8edece3 --- /dev/null +++ b/e2e/parity/verify-results.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify retained parity evidence and emit its normalized comparison.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import tomllib +from pathlib import Path +from typing import Any + +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +DIGEST_REFERENCE_RE = re.compile(r"^[^@]+@sha256:([0-9a-f]{64})$") +ARTIFACT_FIELDS = { + "gateway_sha256": "gateway", + "cli_sha256": "cli", + "conformance_sha256": "conformance", + "supervisor_sha256": "supervisor", + "supervisor_dockerfile_sha256": "supervisor.Dockerfile", + "cli_trace_wrapper_sha256": "cli-trace-wrapper", +} +ORACLE_MARKERS = ( + "][smoke/status] completed", + "][smoke/create] completed", + "][smoke/get-ready] completed", + "][smoke/list-visible/0] completed", + "][smoke/exec] completed", + "][smoke/delete] completed", + "][smoke/list-empty/query/0] completed", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as source: + value = json.load(source) + if not isinstance(value, dict): + raise ValueError(f"{path}: expected a JSON object") + return value + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def verify_variant( + results_dir: Path, + variant: str, + expected_sha: str, + schema_version: int, + scenario: str, +) -> dict[str, Any]: + result_path = results_dir / f"{variant}.json" + launch_path = results_dir / f"{variant}.launch.json" + log_path = results_dir / f"{variant}.log" + config_path = results_dir / f"{variant}.gateway.toml" + result = load_json(result_path) + launch = load_json(launch_path) + require(config_path.is_file(), f"{config_path}: retained gateway config is missing") + with config_path.open("rb") as config_file: + config = tomllib.load(config_file) + + require(result.get("variant") == variant, f"{result_path}: variant mismatch") + require( + result.get("source_sha") == expected_sha, f"{result_path}: source SHA mismatch" + ) + require( + result.get("schema_version") == schema_version, + f"{result_path}: schema mismatch", + ) + require(result.get("scenario") == scenario, f"{result_path}: scenario mismatch") + require(result.get("driver") == "podman", f"{result_path}: driver mismatch") + expected_profile = "driver-free" if scenario == "external-driver" else "in-tree" + expected_features = ( + "--no-default-features --features telemetry" + if scenario == "external-driver" + else "default" + ) + require( + result.get("gateway_profile") == expected_profile, + f"{result_path}: gateway profile mismatch", + ) + require( + result.get("gateway_cargo_features") == expected_features, + f"{result_path}: gateway feature profile mismatch", + ) + require(result.get("success") is True, f"{result_path}: parity oracle did not pass") + require( + result.get("gateway_origin") == "built_by_harness", + f"{result_path}: gateway was not built by the harness", + ) + require( + result.get("cli_origin") == "built_by_harness", + f"{result_path}: CLI was not built by the harness", + ) + require( + result.get("conformance_origin") == "built_by_harness", + f"{result_path}: conformance runner was not built by the harness", + ) + require( + result.get("supervisor_origin") == "built_by_harness", + f"{result_path}: supervisor was not built by the harness", + ) + + external = scenario == "external-driver" + expected_driver_origin = "built_by_harness" if external else "not_applicable" + require( + result.get("external_driver_origin") == expected_driver_origin, + f"{result_path}: external driver origin mismatch", + ) + require( + launch.get("schema_version") == schema_version, + f"{launch_path}: schema mismatch", + ) + require( + launch.get("external_compute_driver") is external, + f"{launch_path}: topology mismatch", + ) + expected_transport = "remote_uds" if external else "in_tree" + require( + launch.get("compute_driver_transport") == expected_transport, + f"{launch_path}: transport mismatch", + ) + expected_policy = "missing" if schema_version == 1 else "if_not_present" + require( + launch.get("external_driver_pull_policy") == expected_policy, + f"{launch_path}: pull-policy mismatch", + ) + + openshell = config.get("openshell", {}) + gateway = openshell.get("gateway", {}) + podman_config = openshell.get("drivers", {}).get("podman", {}) + require( + openshell.get("version") == schema_version, f"{config_path}: schema mismatch" + ) + expected_selector = ["podman"] if schema_version == 1 else "podman" + selector_field = "compute_drivers" if schema_version == 1 else "compute_driver" + require( + gateway.get(selector_field) == expected_selector, + f"{config_path}: selected compute driver mismatch", + ) + require(isinstance(podman_config, dict), f"{config_path}: Podman table is missing") + if external: + require( + set(podman_config) == {"socket_path"}, + f"{config_path}: external gateway Podman table is not transport-only", + ) + require( + isinstance(podman_config["socket_path"], str) + and Path(podman_config["socket_path"]).is_absolute(), + f"{config_path}: external driver socket path is not absolute", + ) + else: + required_runtime_fields = { + "socket_path", + "network_name", + "default_image", + "image_pull_policy", + "supervisor_image", + } + require( + set(podman_config) >= required_runtime_fields, + f"{config_path}: in-tree Podman runtime fields are incomplete", + ) + require( + DIGEST_REFERENCE_RE.fullmatch(podman_config["default_image"]) is not None, + f"{config_path}: sandbox image is not digest-pinned", + ) + require( + DIGEST_REFERENCE_RE.fullmatch(podman_config["supervisor_image"]) + is not None, + f"{config_path}: supervisor image is not digest-pinned", + ) + + artifact_fields = dict(ARTIFACT_FIELDS) + if external: + artifact_fields["external_driver_sha256"] = "external-driver" + artifact_hashes: dict[str, str] = {} + for field, filename in artifact_fields.items(): + expected_hash = result.get(field) + require( + isinstance(expected_hash, str) + and SHA256_RE.fullmatch(expected_hash) is not None, + f"{result_path}: invalid {field}", + ) + artifact_path = results_dir / "artifacts" / variant / filename + require( + artifact_path.is_file(), f"{artifact_path}: retained artifact is missing" + ) + actual_hash = sha256(artifact_path) + require( + actual_hash == expected_hash, + f"{artifact_path}: retained artifact hash mismatch", + ) + artifact_hashes[filename] = actual_hash + + image_id = launch.get("supervisor_image_id") + image_digest = launch.get("supervisor_image_digest") + runtime_image = launch.get("supervisor_runtime_image") + require( + isinstance(image_id, str) and SHA256_RE.fullmatch(image_id) is not None, + f"{launch_path}: invalid supervisor image ID", + ) + require( + isinstance(image_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) is not None, + f"{launch_path}: invalid supervisor image digest", + ) + match = ( + DIGEST_REFERENCE_RE.fullmatch(runtime_image) + if isinstance(runtime_image, str) + else None + ) + require( + match is not None, + f"{launch_path}: supervisor runtime image is not digest-pinned", + ) + require( + f"sha256:{match.group(1)}" == image_digest, + f"{launch_path}: runtime reference digest mismatch", + ) + + sandbox_request = launch.get("sandbox_image_request") + sandbox_id = launch.get("sandbox_image_id") + sandbox_digest = launch.get("sandbox_image_digest") + sandbox_runtime = launch.get("sandbox_runtime_image") + sandbox_match = ( + DIGEST_REFERENCE_RE.fullmatch(sandbox_runtime) + if isinstance(sandbox_runtime, str) + else None + ) + require( + isinstance(sandbox_id, str) and SHA256_RE.fullmatch(sandbox_id) is not None, + f"{launch_path}: invalid sandbox image ID", + ) + require( + isinstance(sandbox_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", sandbox_digest) is not None, + f"{launch_path}: invalid sandbox image digest", + ) + require( + sandbox_match is not None + and f"sha256:{sandbox_match.group(1)}" == sandbox_digest, + f"{launch_path}: sandbox runtime image is not digest-pinned", + ) + require( + sandbox_request == sandbox_runtime, + f"{launch_path}: sandbox image request was not the resolved digest reference", + ) + if not external: + require( + podman_config["default_image"] == sandbox_runtime + and podman_config["supervisor_image"] == runtime_image, + f"{config_path}: runtime image references differ from launch evidence", + ) + + base_runtime = launch.get("supervisor_base_runtime_image") + base_runtime_match = ( + DIGEST_REFERENCE_RE.fullmatch(base_runtime) + if isinstance(base_runtime, str) + else None + ) + require( + base_runtime_match is not None, + f"{launch_path}: supervisor base runtime image is not digest-pinned", + ) + for field in ("supervisor_base_image_id", "supervisor_package_manifest_sha256"): + value = launch.get(field) + require( + isinstance(value, str) and SHA256_RE.fullmatch(value) is not None, + f"{launch_path}: invalid {field}", + ) + base_digest = launch.get("supervisor_base_image_digest") + require( + isinstance(base_digest, str) + and re.fullmatch(r"sha256:[0-9a-f]{64}", base_digest) is not None, + f"{launch_path}: invalid supervisor base-image digest", + ) + package_path = results_dir / "artifacts" / variant / "supervisor.packages.txt" + require(package_path.is_file(), f"{package_path}: package manifest is missing") + require( + sha256(package_path) == launch["supervisor_package_manifest_sha256"], + f"{package_path}: package manifest hash mismatch", + ) + artifact_hashes["supervisor.packages.txt"] = sha256(package_path) + + sandbox_alias = launch.get("sandbox_client_image_alias") + require( + isinstance(sandbox_alias, str) + and sandbox_alias == sandbox_runtime.rsplit("@", 1)[0] + ":latest" + and launch.get("sandbox_client_image_alias_id") == sandbox_id, + f"{launch_path}: sandbox client alias is not bound to the pinned image", + ) + + for launch_field, result_field in ( + ("gateway_sha256_before_execution", "gateway_sha256"), + ("cli_sha256_before_execution", "cli_sha256"), + ("conformance_sha256_before_execution", "conformance_sha256"), + ("supervisor_sha256_before_execution", "supervisor_sha256"), + ( + "supervisor_dockerfile_sha256_before_execution", + "supervisor_dockerfile_sha256", + ), + ("cli_trace_wrapper_sha256_before_execution", "cli_trace_wrapper_sha256"), + ): + require( + launch.get(launch_field) == result.get(result_field), + f"{launch_path}: {launch_field} does not bind the staged artifact", + ) + if external: + require( + launch.get("external_driver_sha256_before_execution") + == result.get("external_driver_sha256"), + f"{launch_path}: external driver pre-execution hash mismatch", + ) + gateway_port = launch.get("gateway_port") + callback_endpoint = f"https://host.containers.internal:{gateway_port}" + require( + isinstance(gateway_port, int) + and 0 < gateway_port <= 65535 + and launch.get("external_driver_grpc_endpoint") == callback_endpoint, + f"{launch_path}: external driver callback endpoint is not isolated", + ) + require( + launch.get("external_driver_host_gateway_ip") == "host-gateway" + and launch.get("external_driver_userns") is None + and launch.get("external_driver_spiffe") is False + and launch.get("external_driver_proxy") is False + and launch.get("external_driver_app_armor") is False, + f"{launch_path}: external driver effective configuration is tainted", + ) + driver_environment = launch.get("external_driver_environment") + expected_environment_keys = { + "OPENSHELL_COMPUTE_DRIVER_SOCKET", + "OPENSHELL_PODMAN_SOCKET", + "OPENSHELL_SANDBOX_IMAGE", + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS", + "OPENSHELL_GRPC_ENDPOINT", + "OPENSHELL_GATEWAY_PORT", + "OPENSHELL_NETWORK_NAME", + "OPENSHELL_STOP_TIMEOUT", + "OPENSHELL_SUPERVISOR_IMAGE", + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + "OPENSHELL_ENABLE_BIND_MOUNTS", + } + require( + isinstance(driver_environment, dict) + and set(driver_environment) == expected_environment_keys, + f"{launch_path}: external driver allowlisted environment is incomplete", + ) + require( + driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] + == podman_config["socket_path"], + f"{launch_path}: external driver socket differs from gateway TOML", + ) + podman_socket = driver_environment["OPENSHELL_PODMAN_SOCKET"] + require( + isinstance(podman_socket, str) + and Path(podman_socket).is_absolute() + and podman_socket != podman_config["socket_path"], + f"{launch_path}: external driver Podman socket is not isolated", + ) + require( + driver_environment["OPENSHELL_SANDBOX_IMAGE"] == sandbox_runtime + and driver_environment["OPENSHELL_SANDBOX_IMAGE_PULL_POLICY"] + == expected_policy + and driver_environment["OPENSHELL_HEALTH_CHECK_INTERVAL_SECS"] == 10 + and driver_environment["OPENSHELL_GRPC_ENDPOINT"] == callback_endpoint + and driver_environment["OPENSHELL_GATEWAY_PORT"] == gateway_port + and isinstance(driver_environment["OPENSHELL_NETWORK_NAME"], str) + and driver_environment["OPENSHELL_NETWORK_NAME"] + and isinstance(driver_environment["OPENSHELL_STOP_TIMEOUT"], int) + and driver_environment["OPENSHELL_STOP_TIMEOUT"] >= 0 + and driver_environment["OPENSHELL_SUPERVISOR_IMAGE"] == runtime_image + and driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True, + f"{launch_path}: external driver allowlisted runtime inputs differ", + ) + tls_paths: set[str] = set() + for field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + tls_input = driver_environment[field] + require( + isinstance(tls_input, dict) + and set(tls_input) == {"path", "sha256"} + and isinstance(tls_input["path"], str) + and Path(tls_input["path"]).is_absolute() + and isinstance(tls_input["sha256"], str) + and SHA256_RE.fullmatch(tls_input["sha256"]) is not None, + f"{launch_path}: invalid external driver callback TLS input {field}", + ) + tls_paths.add(tls_input["path"]) + require( + len(tls_paths) == 3, + f"{launch_path}: external driver callback TLS paths are not distinct", + ) + else: + require( + launch.get("external_driver_sha256_before_execution") == "", + f"{launch_path}: unexpected external driver hash", + ) + + require(log_path.is_file(), f"{log_path}: retained raw log is missing") + raw_log = log_path.read_text(encoding="utf-8", errors="replace") + for marker in ORACLE_MARKERS: + require( + re.search(re.escape(marker) + r".*exit 0", raw_log) is not None, + f"{log_path}: missing successful lifecycle oracle marker {marker}", + ) + require( + '"passed": true' in raw_log, + f"{log_path}: missing successful conformance result", + ) + require( + re.search( + r"gateway preflight connected: .*authentication=authenticated(?:\n|$)", + raw_log, + ) + is not None, + f"{log_path}: authenticated gateway preflight is missing", + ) + run_ids = re.findall( + r"^CLI conformance run ID: ([a-z0-9]+)$", raw_log, re.MULTILINE + ) + require( + len(run_ids) == 1, + f"{log_path}: expected exactly one conformance run ID", + ) + exec_stdout_path = results_dir / f"{variant}.exec.stdout" + require( + exec_stdout_path.is_file(), + f"{exec_stdout_path}: retained exec stdout is missing", + ) + expected_exec_stdout = f"openshell-conformance-{run_ids[0]}\n".encode() + require( + exec_stdout_path.read_bytes() == expected_exec_stdout, + f"{exec_stdout_path}: callback exec stdout is not the exact marker", + ) + launch_markers = ( + runtime_image, + image_id, + image_digest, + sandbox_runtime, + sandbox_id, + sandbox_digest, + sandbox_alias, + launch["sandbox_client_image_alias_id"], + launch["supervisor_base_image_id"], + base_digest, + base_runtime, + launch["supervisor_package_manifest_sha256"], + ) + require( + all(marker in raw_log for marker in launch_markers), + f"{log_path}: launch provenance is absent from raw output", + ) + + raw_evidence_hashes = { + result_path.name: sha256(result_path), + launch_path.name: sha256(launch_path), + log_path.name: sha256(log_path), + config_path.name: sha256(config_path), + exec_stdout_path.name: sha256(exec_stdout_path), + } + if external: + driver_log_path = results_dir / f"{variant}.driver.log" + require( + driver_log_path.is_file() and driver_log_path.stat().st_size > 0, + f"{driver_log_path}: retained external driver log is missing or empty", + ) + raw_evidence_hashes[driver_log_path.name] = sha256(driver_log_path) + + return { + "schema_version": schema_version, + "source_sha": expected_sha, + "gateway_profile": result["gateway_profile"], + "gateway_cargo_features": result["gateway_cargo_features"], + "artifact_origins": { + "gateway": result["gateway_origin"], + "cli": result["cli_origin"], + "conformance": result["conformance_origin"], + "external_driver": result["external_driver_origin"], + "supervisor": result["supervisor_origin"], + }, + "artifact_sha256": artifact_hashes, + "launch_attestation": launch, + "raw_evidence_sha256": raw_evidence_hashes, + "artifacts_verified": True, + "raw_output_verified": True, + "success": True, + } + + +def verify_topology( + results_dir: Path, + baseline_sha: str, + candidate_sha: str, + scenario: str, +) -> dict[str, Any]: + comparison_path = results_dir / "comparison.json" + comparison = load_json(comparison_path) + require( + comparison.get("scenario") == scenario, f"{comparison_path}: scenario mismatch" + ) + for field in ("baseline_success", "candidate_success", "parity", "accepted"): + require( + comparison.get(field) is True, f"{comparison_path}: {field} is not true" + ) + require( + comparison.get("classification") == "pass", + f"{comparison_path}: classification is not pass", + ) + + baseline = verify_variant(results_dir, "baseline", baseline_sha, 1, scenario) + candidate = verify_variant(results_dir, "candidate", candidate_sha, 2, scenario) + baseline_launch = baseline["launch_attestation"] + candidate_launch = candidate["launch_attestation"] + for field, label in ( + ("sandbox_image_id", "sandbox image ID"), + ("sandbox_image_digest", "sandbox image digest"), + ("sandbox_runtime_image", "sandbox runtime image"), + ("supervisor_base_image", "supervisor base image"), + ("supervisor_base_image_id", "supervisor base-image ID"), + ("supervisor_base_image_digest", "supervisor base-image digest"), + ("supervisor_package_manifest_sha256", "supervisor package manifest"), + ): + require( + baseline_launch.get(field) == candidate_launch.get(field), + f"baseline and candidate {label} differ", + ) + if scenario == "external-driver": + baseline_driver = results_dir / "artifacts/baseline/external-driver" + candidate_driver = results_dir / "artifacts/candidate/external-driver" + require( + baseline_driver.resolve() != candidate_driver.resolve(), + "external-driver artifacts resolve to the same path", + ) + require( + not baseline_driver.samefile(candidate_driver), + "external-driver artifacts share an inode", + ) + require( + baseline["artifact_sha256"]["external-driver"] + != candidate["artifact_sha256"]["external-driver"], + "external-driver artifacts have identical content", + ) + baseline_env = baseline_launch["external_driver_environment"] + candidate_env = candidate_launch["external_driver_environment"] + for field, label in ( + ("OPENSHELL_COMPUTE_DRIVER_SOCKET", "compute-driver UDS"), + ("OPENSHELL_PODMAN_SOCKET", "Podman API UDS"), + ("OPENSHELL_NETWORK_NAME", "Podman network"), + ): + require( + baseline_env[field] != candidate_env[field], + f"baseline and candidate reuse the same external {label}", + ) + for field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + require( + baseline_env[field]["path"] != candidate_env[field]["path"], + "baseline and candidate reuse the same external callback TLS path", + ) + + return { + "baseline": baseline, + "candidate": candidate, + "comparison_sha256": sha256(comparison_path), + "classification": "pass", + "parity": True, + "accepted": True, + } + + +def verify_four_run_provenance( + in_tree: dict[str, Any], external_uds: dict[str, Any] +) -> None: + variants = [ + in_tree["baseline"]["launch_attestation"], + in_tree["candidate"]["launch_attestation"], + external_uds["baseline"]["launch_attestation"], + external_uds["candidate"]["launch_attestation"], + ] + for fields, label in ( + ( + ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "sandbox_client_image_alias", + "sandbox_client_image_alias_id", + ), + "sandbox artifact", + ), + ( + ( + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_base_runtime_image", + "supervisor_package_manifest_sha256", + ), + "supervisor dependency provenance", + ), + ): + tuples = {tuple(launch.get(field) for field in fields) for launch in variants} + require(len(tuples) == 1, f"the four runs use different {label}") + + for variant in ("baseline", "candidate"): + in_tree_artifacts = in_tree[variant]["artifact_sha256"] + external_artifacts = external_uds[variant]["artifact_sha256"] + for artifact in ( + "cli", + "conformance", + "supervisor", + "supervisor.Dockerfile", + "cli-trace-wrapper", + ): + require( + in_tree_artifacts[artifact] == external_artifacts[artifact], + f"{variant} {artifact} differs across topologies", + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline-sha", required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--in-tree", required=True, type=Path) + parser.add_argument("--external-uds", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + require( + re.fullmatch(r"[0-9a-f]{40}", args.baseline_sha) is not None, + "invalid baseline SHA", + ) + require( + re.fullmatch(r"[0-9a-f]{40}", args.candidate_sha) is not None, + "invalid candidate SHA", + ) + + in_tree = verify_topology( + args.in_tree, args.baseline_sha, args.candidate_sha, "smoke" + ) + external_uds = verify_topology( + args.external_uds, args.baseline_sha, args.candidate_sha, "external-driver" + ) + verify_four_run_provenance(in_tree, external_uds) + + report = { + "manifest_version": 2, + "baseline_commit": args.baseline_sha, + "candidate_commit": args.candidate_sha, + "lane": "local-linux-x86_64-rootless-podman-5.8.2", + "retained_evidence_bundles": { + "in_tree": args.in_tree.as_posix(), + "external_uds": args.external_uds.as_posix(), + }, + "oracle": { + "status": True, + "create": True, + "ready": True, + "list_visible": True, + "callback_exec_exact_marker": True, + "delete": True, + "list_empty": True, + }, + "in_tree": in_tree, + "external_uds": external_uds, + "callback_listener": { + "in_tree_baseline_exec": True, + "in_tree_candidate_exec": True, + "external_baseline_exec": True, + "external_candidate_exec": True, + "classification": "pass", + }, + "classification": "pass", + "accepted": True, + "verification": { + "retained_artifact_hashes_recomputed": True, + "raw_lifecycle_output_inspected": True, + "authenticated_preflight_verified": True, + "exact_callback_exec_stdout_verified": True, + "external_driver_allowlist_verified": True, + "external_driver_logs_retained": True, + "external_uds_isolation_verified": True, + "digest_pinned_supervisor_runtime_verified": True, + "same_immutable_sandbox_verified": True, + "supervisor_dependency_provenance_matched": True, + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/e2e/run.sh b/e2e/run.sh index 875186394c..8d9cee8dac 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -142,7 +142,11 @@ if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then fi gateway_driver="$(python3 -c ' import sys, tomllib -print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) +gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] +driver = gateway.get("compute_driver") +if not driver: + raise SystemExit("gateway config must explicitly select a compute driver") +print(driver) ' "${gateway_config}")" if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 96da2a879f..4bc2cfe1fa 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -256,11 +256,14 @@ e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" cat >"${GATEWAY_CONFIG}" <&2 + return 2 + ;; + esac +} + +e2e_podman_external_driver_pull_policy() { + case "$1" in + 1) printf '%s\n' missing ;; + 2) printf '%s\n' if_not_present ;; + *) echo "ERROR: unsupported Podman config schema version: $1" >&2; return 2 ;; + esac +} + +e2e_podman_toml_string() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '"%s"' "${value}" +} + +# Return the explicitly selected behavioral profile. Keep the default empty so +# ordinary smoke coverage continues to exercise the minimal configuration. +e2e_podman_option_profile() { + case "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" in + "") printf '%s\n' "" ;; + podman-options) printf '%s\n' "podman-options" ;; + *) + echo "ERROR: unsupported OPENSHELL_E2E_PODMAN_OPTION_PROFILE: ${OPENSHELL_E2E_PODMAN_OPTION_PROFILE}" >&2 + return 2 + ;; + esac +} + +# Frozen baseline 74960ebfaeec4673885089ed995fad902459749f does not accept +# Podman app_armor_profile, so it is deliberately not part of this paired profile. + +# Write the minimally configured Podman e2e gateway TOML. The schema-v1 +# branch deliberately uses the frozen-main contract: list driver selection, +# driver-local guest TLS, the old "missing" pull-policy spelling, and zero to +# disable Podman health checks. Schema v2 uses gateway-owned guest TLS and the +# current positive health-check setting from the RPM template. +e2e_write_podman_gateway_config() { + local output=$1 + local schema_version=$2 + local root=$3 + local pki_dir=$4 + local jwt_dir=$5 + local gateway_id=$6 + local external_driver=$7 + local driver_socket=$8 + local network_name=$9 + local gateway_port=${10} + local sandbox_image=${11} + local stop_timeout_secs=${12} + local supervisor_image=${13} + local provider_spiffe_socket=${14} + local podman_socket=${15} + local oidc_mode=${16} + local oidc_issuer=${17} + local configured_with_tls option_profile + + case "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" in + ""|podman-options) option_profile="${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" ;; + *) echo "ERROR: unsupported OPENSHELL_E2E_PODMAN_OPTION_PROFILE: ${OPENSHELL_E2E_PODMAN_OPTION_PROFILE}" >&2; return 2 ;; + esac + + case "${schema_version}" in + 1) + cp "${root}/deploy/rpm/gateway.toml.default.v1" "${output}" + { + e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" + if [ "${oidc_mode}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${oidc_issuer}" ]; then + e2e_write_gateway_oidc_config "${oidc_issuer}" + fi + fi + printf '\n[openshell.drivers.podman]\n' + if [ "${external_driver}" = "1" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" + else + printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" + printf 'gateway_port = %s\n' "${gateway_port}" + printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" + printf 'image_pull_policy = "missing"\n' + if [ "${option_profile}" = "podman-options" ]; then + printf 'sandbox_pids_limit = 31\n' + printf 'health_check_interval_secs = 7\n' + + printf 'sandbox_ssh_socket_path = "/run/openshell/parity-ssh.sock"\n' + else + # In schema v1, zero explicitly disables Podman health checks. + printf 'health_check_interval_secs = 0\n' + fi + printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" + printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" + printf 'guest_tls_ca = %s\n' "$(e2e_podman_toml_string "${pki_dir}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + if [ -n "${provider_spiffe_socket}" ]; then + printf 'provider_spiffe_workload_api_socket = %s\n' "$(e2e_podman_toml_string "${provider_spiffe_socket}")" + fi + if [ -n "${podman_socket}" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" + fi + fi + } >>"${output}" + ;; + 2) + cp "${root}/deploy/rpm/gateway.toml.default" "${output}" + if [ "${external_driver}" = "1" ]; then + # A remote UDS driver owns all runtime options. Keep the selected + # gateway table transport-only so no in-tree setting can be mistaken + # for executed external-driver configuration. + sed -i '/^health_check_interval_secs = /d' "${output}" + elif [ "${option_profile}" = "podman-options" ]; then + sed -i 's/^health_check_interval_secs = .*/health_check_interval_secs = 7/' "${output}" + fi + # The v2 template opens the Podman table. Insert gateway-owned TLS + # before it rather than reopening [openshell.gateway] later. + configured_with_tls="${output}.tls" + while IFS= read -r line; do + if [ "${line}" = "[openshell.drivers.podman]" ]; then + printf 'guest_tls_ca = %s\n' "$(e2e_podman_toml_string "${pki_dir}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(e2e_podman_toml_string "${pki_dir}/client/tls.key")" + fi + printf '%s\n' "${line}" + done <"${output}" >"${configured_with_tls}" + mv "${configured_with_tls}" "${output}" + { + if [ "${external_driver}" = "1" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${driver_socket}")" + else + printf 'network_name = %s\n' "$(e2e_podman_toml_string "${network_name}")" + printf 'gateway_port = %s\n' "${gateway_port}" + printf 'default_image = %s\n' "$(e2e_podman_toml_string "${sandbox_image}")" + printf 'image_pull_policy = "if_not_present"\n' + if [ "${option_profile}" = "podman-options" ]; then + printf 'sandbox_pids_limit = 31\n' + printf 'ssh_socket_path = "/run/openshell/parity-ssh.sock"\n' + fi + printf 'stop_timeout_secs = %s\n' "${stop_timeout_secs}" + printf 'supervisor_image = %s\n' "$(e2e_podman_toml_string "${supervisor_image}")" + printf 'enable_bind_mounts = true\n' + if [ -n "${provider_spiffe_socket}" ]; then + printf 'provider_spiffe_workload_api_socket = %s\n' "$(e2e_podman_toml_string "${provider_spiffe_socket}")" + fi + if [ -n "${podman_socket}" ]; then + printf 'socket_path = %s\n' "$(e2e_podman_toml_string "${podman_socket}")" + fi + fi + e2e_write_gateway_jwt_config "${jwt_dir}" "${gateway_id}" + if [ "${oidc_mode}" != "1" ]; then + e2e_write_gateway_mtls_auth_config + if [ -n "${oidc_issuer}" ]; then + e2e_write_gateway_oidc_config "${oidc_issuer}" + fi + fi + } >>"${output}" + ;; + *) + echo "ERROR: unsupported Podman config schema version: ${schema_version}" >&2 + return 2 + ;; + esac +} diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 0a767576dd..0f64247731 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -16,11 +16,11 @@ # # Sandbox image overrides: # OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE=... -# OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY=Always|IfNotPresent|Never +# OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY=always|if_not_present|never # # The default community sandbox image uses :latest. This wrapper refreshes it -# before starting the gateway, while the Docker driver defaults to IfNotPresent -# so local Dockerfile-built images remain usable. +# before starting the gateway, while the Docker driver defaults to +# if_not_present so local Dockerfile-built images remain usable. # set -euo pipefail @@ -450,7 +450,7 @@ echo "Using Docker supervisor image: ${SUPERVISOR_IMAGE}" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY:-${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY:-${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}}" if ! ensure_sandbox_image_available "${SANDBOX_IMAGE}"; then echo "ERROR: sandbox image '${SANDBOX_IMAGE}' is not available." >&2 exit 2 @@ -501,8 +501,11 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" { - printf '[openshell]\nversion = 1\n\n' - printf '[openshell.gateway]\nlog_level = "info"\n\n' + printf '[openshell]\nversion = 2\n\n' + printf '[openshell.gateway]\nlog_level = "info"\n' + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n\n' "$(toml_string "${PKI_DIR}/client/tls.key")" e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-docker-${HOST_PORT}" if [ "${OIDC_MODE}" != "1" ]; then e2e_write_gateway_mtls_auth_config @@ -514,14 +517,11 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then @@ -532,7 +532,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'sandbox_label = %s\n' "$(toml_string "${E2E_NAMESPACE}")" printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" @@ -560,7 +560,7 @@ GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" - --drivers docker + --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fc3419e182..76fe699bc8 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -28,6 +28,8 @@ fi ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=e2e/support/gateway-common.sh source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck source=e2e/support/podman-gateway-config.sh +source "${ROOT}/e2e/support/podman-gateway-config.sh" require_container_engine_lane() { local lane=$1 @@ -83,6 +85,20 @@ podman_cmd() { with_podman_config podman "$@" } +require_expected_sha256() { + local label=$1 path=$2 expected=$3 actual + [ -n "${expected}" ] || return 0 + if [ ! -f "${path}" ]; then + echo "ERROR: ${label} is missing before execution: ${path}" >&2 + exit 2 + fi + actual="$(sha256sum "${path}" | cut -d' ' -f1)" + if [ "${actual}" != "${expected}" ]; then + echo "ERROR: ${label} hash changed before execution." >&2 + exit 2 + fi +} + WORKDIR_PARENT="${TMPDIR:-/tmp}" WORKDIR_PARENT="${WORKDIR_PARENT%/}" WORKDIR="$(mktemp -d "${WORKDIR_PARENT}/openshell-e2e-podman.XXXXXX")" @@ -100,11 +116,13 @@ GATEWAY_BIN="" CLI_BIN="" GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" +export OPENSHELL_E2E_GATEWAY_LOG="${GATEWAY_LOG}" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" DRIVER_BIN="" DRIVER_PID="" -DRIVER_LOG="${WORKDIR}/podman-driver.log" +DRIVER_LOG="${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE:-${WORKDIR}/podman-driver.log}" +mkdir -p "$(dirname "${DRIVER_LOG}")" DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" @@ -233,17 +251,21 @@ default_podman_socket_path() { } ensure_podman_api_socket() { - if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then - return 0 - fi + if [ "${OPENSHELL_E2E_FORCE_TEMP_PODMAN_SERVICE:-0}" != 1 ]; then + if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then + return 0 + fi - local default_socket - default_socket="$(default_podman_socket_path || true)" - if [ -n "${default_socket}" ] \ - && [ -S "${default_socket}" ] \ - && podman_cmd --url "unix://${default_socket}" info >/dev/null 2>&1; then - export OPENSHELL_PODMAN_SOCKET="${default_socket}" - return 0 + local default_socket + default_socket="$(default_podman_socket_path || true)" + if [ -n "${default_socket}" ] \ + && [ -S "${default_socket}" ] \ + && podman_cmd --url "unix://${default_socket}" info >/dev/null 2>&1; then + export OPENSHELL_PODMAN_SOCKET="${default_socket}" + return 0 + fi + else + unset OPENSHELL_PODMAN_SOCKET fi # `podman system service` is a Linux-only subcommand — the macOS client @@ -313,6 +335,69 @@ resolve_podman_supervisor_image() { ensure_podman_supervisor_image() { local image=$1 + if [ -n "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" ]; then + local dockerfile=${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor} + local context="${WORKDIR}/supervisor-image" arch + case "${image}" in + *:dev|*:latest) + echo "ERROR: supplied supervisor binaries require a unique versioned image tag, not ${image}." >&2 + exit 2 + ;; + *:*) ;; + *) + echo "ERROR: supplied supervisor binaries require an explicit versioned image tag: ${image}." >&2 + exit 2 + ;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + *) echo "ERROR: unsupported supervisor image architecture: $(uname -m)" >&2; exit 2 ;; + esac + if [ ! -x "${OPENSHELL_E2E_SUPERVISOR_BIN}" ]; then + echo "ERROR: supplied supervisor binary is not executable: ${OPENSHELL_E2E_SUPERVISOR_BIN}" >&2 + exit 2 + fi + if [ ! -f "${dockerfile}" ]; then + echo "ERROR: supervisor Dockerfile not found: ${dockerfile}" >&2 + exit 2 + fi + require_expected_sha256 "supervisor binary" "${OPENSHELL_E2E_SUPERVISOR_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256:-}" + require_expected_sha256 "supervisor Dockerfile" "${dockerfile}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256:-}" + mkdir -p "${context}/deploy/docker/.build/prebuilt-binaries/${arch}" + install -m 0555 "${OPENSHELL_E2E_SUPERVISOR_BIN}" \ + "${context}/deploy/docker/.build/prebuilt-binaries/${arch}/openshell-sandbox" + cp "${dockerfile}" "${context}/deploy/docker/Dockerfile.supervisor" + local -a pull_option=() + if [ -n "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-}" ]; then + local dockerfile_base + dockerfile_base="$(awk '$1 == "FROM" { print $2; exit }' "${dockerfile}")" + if [ "${dockerfile_base}" != "${OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE:-}" ] \ + || ! [[ "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: supervisor base-image attestation does not match the Dockerfile." >&2 + exit 2 + fi + echo "Pulling pinned supervisor base image ${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}..." + podman_cmd pull "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" + podman_cmd tag "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE}" "${dockerfile_base}" + pull_option=(--pull=never) + fi + echo "Building Podman supervisor image ${image} from supplied binary..." + ( + cd "${context}" + podman_cmd build \ + "${pull_option[@]}" \ + --build-arg "TARGETARCH=${arch}" \ + --file deploy/docker/Dockerfile.supervisor \ + --target supervisor \ + --tag "${image}" \ + . + ) + return 0 + fi + if [ "${image}" = "openshell/supervisor:dev" ] \ && [ -z "${OPENSHELL_SUPERVISOR_IMAGE:-}" ] \ && [ -z "${CI:-}" ]; then @@ -370,6 +455,13 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then exit $? fi +# Validate the generated configuration dialect before creating runtime resources. +CONFIG_SCHEMA_VERSION="$(e2e_podman_config_schema_version)" +EXTERNAL_DRIVER_PULL_POLICY="$(e2e_podman_external_driver_pull_policy "${CONFIG_SCHEMA_VERSION}")" + +# Validate the opt-in profile before building images or allocating runtime resources. +e2e_podman_option_profile >/dev/null + # Preflight for managed Podman gateway mode. if ! command -v podman >/dev/null 2>&1; then echo "ERROR: podman CLI is required to run Podman-backed e2e tests" >&2 @@ -391,19 +483,90 @@ fi SUPERVISOR_IMAGE="$(resolve_podman_supervisor_image)" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" -echo "Using Podman supervisor image: ${SUPERVISOR_IMAGE}" +SUPERVISOR_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_IMAGE}")" +SUPERVISOR_IMAGE_ID="${SUPERVISOR_IMAGE_ID#sha256:}" +SUPERVISOR_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_IMAGE}")" +if ! [[ "${SUPERVISOR_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve immutable supervisor image ID for ${SUPERVISOR_IMAGE}." >&2 + exit 2 +fi +if ! [[ "${SUPERVISOR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve supervisor image digest for ${SUPERVISOR_IMAGE}." >&2 + exit 2 +fi +# The parity harness forces a temporary Podman API service into the same +# isolated XDG store where this image was built. Address the local image by its +# immutable manifest digest so policy=missing cannot resolve a mutable tag or +# contact a registry for a different artifact. +SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%:*}" +SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE_REPOSITORY}@${SUPERVISOR_IMAGE_DIGEST}" +if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: supervisor runtime image is not digest-pinned: ${SUPERVISOR_RUNTIME_IMAGE}" >&2 + exit 2 +fi +SUPERVISOR_BASE_IMAGE="$(awk '$1 == "FROM" { print $2; exit }' "${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor}")" +SUPERVISOR_BASE_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SUPERVISOR_BASE_IMAGE}")" +SUPERVISOR_BASE_IMAGE_ID="${SUPERVISOR_BASE_IMAGE_ID#sha256:}" +SUPERVISOR_BASE_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SUPERVISOR_BASE_IMAGE}")" +if ! [[ "${SUPERVISOR_BASE_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${SUPERVISOR_BASE_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve supervisor base-image provenance for ${SUPERVISOR_BASE_IMAGE}." >&2 + exit 2 +fi +SUPERVISOR_PACKAGE_MANIFEST="${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE:-${WORKDIR}/supervisor.packages.txt}" +mkdir -p "$(dirname "${SUPERVISOR_PACKAGE_MANIFEST}")" +podman_cmd run --rm --network none --entrypoint /sbin/apk \ + "${SUPERVISOR_RUNTIME_IMAGE}" info -v | LC_ALL=C sort >"${SUPERVISOR_PACKAGE_MANIFEST}" +SUPERVISOR_PACKAGE_MANIFEST_SHA256="$(sha256sum "${SUPERVISOR_PACKAGE_MANIFEST}" | cut -d' ' -f1)" +echo "Using Podman supervisor image: ${SUPERVISOR_RUNTIME_IMAGE} (ID ${SUPERVISOR_IMAGE_ID}, digest ${SUPERVISOR_IMAGE_DIGEST}, base ${SUPERVISOR_BASE_IMAGE} ID ${SUPERVISOR_BASE_IMAGE_ID} digest ${SUPERVISOR_BASE_IMAGE_DIGEST}, packages ${SUPERVISOR_PACKAGE_MANIFEST_SHA256})" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -SANDBOX_IMAGE="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" +SANDBOX_IMAGE_REQUEST="${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ + && ! [[ "${SANDBOX_IMAGE_REQUEST}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: this e2e invocation requires a digest-pinned sandbox image: ${SANDBOX_IMAGE_REQUEST}" >&2 + exit 2 +fi PODMAN_STOP_TIMEOUT_SECS="${OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS:-15}" if ! [[ "${PODMAN_STOP_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then echo "ERROR: OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS must be a non-negative integer." >&2 exit 2 fi -if ! podman_cmd image exists "${SANDBOX_IMAGE}" 2>/dev/null; then - echo "Pulling ${SANDBOX_IMAGE}..." - podman_cmd pull "${SANDBOX_IMAGE}" +if ! podman_cmd image exists "${SANDBOX_IMAGE_REQUEST}" 2>/dev/null; then + echo "Pulling ${SANDBOX_IMAGE_REQUEST}..." + podman_cmd pull "${SANDBOX_IMAGE_REQUEST}" +fi +SANDBOX_IMAGE_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SANDBOX_IMAGE_REQUEST}")" +SANDBOX_IMAGE_ID="${SANDBOX_IMAGE_ID#sha256:}" +SANDBOX_IMAGE_DIGEST="$(podman_cmd image inspect --format '{{.Digest}}' "${SANDBOX_IMAGE_REQUEST}")" +SANDBOX_IMAGE_REPOSITORY="${SANDBOX_IMAGE_REQUEST%%@*}" +case "${SANDBOX_IMAGE_REPOSITORY##*/}" in + *:*) SANDBOX_IMAGE_REPOSITORY="${SANDBOX_IMAGE_REPOSITORY%:*}" ;; +esac +SANDBOX_RUNTIME_IMAGE="${SANDBOX_IMAGE_REPOSITORY}@${SANDBOX_IMAGE_DIGEST}" +if ! [[ "${SANDBOX_IMAGE_ID}" =~ ^[0-9a-f]{64}$ ]] \ + || ! [[ "${SANDBOX_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: could not resolve an immutable sandbox image for ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 +fi +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ] \ + && [ "${SANDBOX_IMAGE_REQUEST}" != "${SANDBOX_RUNTIME_IMAGE}" ]; then + echo "ERROR: sandbox image digest changed while resolving ${SANDBOX_IMAGE_REQUEST}." >&2 + exit 2 +fi +SANDBOX_CLIENT_IMAGE_ALIAS="" +SANDBOX_CLIENT_IMAGE_ALIAS_ID="" +if [ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = "1" ]; then + SANDBOX_CLIENT_IMAGE_ALIAS="${SANDBOX_IMAGE_REPOSITORY}:latest" + podman_cmd tag "${SANDBOX_RUNTIME_IMAGE}" "${SANDBOX_CLIENT_IMAGE_ALIAS}" + SANDBOX_CLIENT_IMAGE_ALIAS_ID="$(podman_cmd image inspect --format '{{.Id}}' "${SANDBOX_CLIENT_IMAGE_ALIAS}")" + SANDBOX_CLIENT_IMAGE_ALIAS_ID="${SANDBOX_CLIENT_IMAGE_ALIAS_ID#sha256:}" + if [ "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" != "${SANDBOX_IMAGE_ID}" ]; then + echo "ERROR: sandbox client alias does not resolve to the pinned sandbox image." >&2 + exit 2 + fi fi +echo "Using Podman sandbox image: ${SANDBOX_RUNTIME_IMAGE} (ID ${SANDBOX_IMAGE_ID}, digest ${SANDBOX_IMAGE_DIGEST}, client alias ${SANDBOX_CLIENT_IMAGE_ALIAS:-none} ID ${SANDBOX_CLIENT_IMAGE_ALIAS_ID:-none})" PKI_DIR="${WORKDIR}/pki" e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" "host.containers.internal" @@ -437,80 +600,125 @@ export OPENSHELL_E2E_SANDBOX_NAMESPACE="${E2E_NAMESPACE}" echo "Starting openshell-gateway on port ${HOST_PORT} (namespace: ${E2E_NAMESPACE})..." e2e_generate_gateway_jwt "${JWT_DIR}" -# Driver-specific options moved from CLI flags into a TOML config table -# (commit 560550d2). Synthesize a minimal config here and pass --config. -# Quote a value as a TOML basic string: see with-docker-gateway.sh for -# the same helper (kept duplicated to avoid sourcing across e2e scripts). -toml_string() { - local value="$1" - value="${value//\\/\\\\}" - value="${value//\"/\\\"}" - printf '"%s"' "${value}" -} - GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" - -# Start from the RPM default template so this e2e test exercises the same TOML -# config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine, -# the driver reserves IPv4 loopback for its callback-only listener, so the -# primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. -# -# We append the driver-specific table and override the port via CLI flag -# (CLI > TOML in the merge precedence) so the test can use an ephemeral port. -cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" -{ - e2e_write_gateway_jwt_config "${JWT_DIR}" "openshell-e2e-podman-${HOST_PORT}" - if [ "${OIDC_MODE}" != "1" ]; then - e2e_write_gateway_mtls_auth_config - if [ -n "${OPENSHELL_OIDC_ISSUER:-}" ]; then - e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" - fi - fi - printf '\n[openshell.drivers.podman]\n' +e2e_write_podman_gateway_config \ + "${GATEWAY_CONFIG}" \ + "${CONFIG_SCHEMA_VERSION}" \ + "${ROOT}" \ + "${PKI_DIR}" \ + "${JWT_DIR}" \ + "openshell-e2e-podman-${HOST_PORT}" \ + "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" \ + "${DRIVER_SOCKET}" \ + "${PODMAN_NETWORK_NAME}" \ + "${HOST_PORT}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ + "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ + "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" \ + "${OPENSHELL_PODMAN_SOCKET:-}" \ + "${OIDC_MODE}" \ + "${OPENSHELL_OIDC_ISSUER:-}" +if [ -n "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE:-}" ]; then + cp "${GATEWAY_CONFIG}" "${OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE}" +fi +EXTERNAL_DRIVER_CALLBACK_ENDPOINT="https://host.containers.internal:${HOST_PORT}" +EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS=10 +EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS=true +EXTERNAL_DRIVER_TLS_CA="${PKI_DIR}/ca.crt" +EXTERNAL_DRIVER_TLS_CERT="${PKI_DIR}/client/tls.crt" +EXTERNAL_DRIVER_TLS_KEY="${PKI_DIR}/client/tls.key" +if [ -n "${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE:-}" ]; then + driver_transport=in_tree + external_driver_grpc_endpoint=null + external_driver_host_gateway_ip=null + external_driver_userns=null + external_driver_spiffe=false + external_driver_proxy=false + external_driver_app_armor=false + external_driver_environment=null if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then - printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" - else - # The Podman driver scopes isolation by network rather than namespace. - printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" - printf 'gateway_port = %s\n' "${HOST_PORT}" - printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = "missing"\n' - # Keep CI teardown bounded while the production Podman driver default stays - # conservative for real user workloads. - printf 'stop_timeout_secs = %s\n' "${PODMAN_STOP_TIMEOUT_SECS}" - printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - printf 'enable_bind_mounts = true\n' - if [ -n "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET:-}" ]; then - printf 'provider_spiffe_workload_api_socket = %s\n' "$(toml_string "${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" - fi - # The in-process Podman driver reads `socket_path` from TOML only — the - # OPENSHELL_PODMAN_SOCKET env var is honoured by the standalone driver - # binary, not the in-process driver used here. Pin the socket to the one - # the harness discovered (e.g. via `podman machine inspect` on macOS) so - # we don't fall back to the driver's stale macOS default. - if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then - printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" - fi + driver_transport=remote_uds + external_driver_grpc_endpoint="\"${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}\"" + external_driver_host_gateway_ip='"host-gateway"' + driver_tls_ca_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CA}" | cut -d' ' -f1)" + driver_tls_cert_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_CERT}" | cut -d' ' -f1)" + driver_tls_key_sha256="$(sha256sum "${EXTERNAL_DRIVER_TLS_KEY}" | cut -d' ' -f1)" + external_driver_environment="$(printf '{\"OPENSHELL_COMPUTE_DRIVER_SOCKET\":\"%s\",\"OPENSHELL_PODMAN_SOCKET\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE\":\"%s\",\"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY\":\"%s\",\"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS\":%s,\"OPENSHELL_GRPC_ENDPOINT\":\"%s\",\"OPENSHELL_GATEWAY_PORT\":%s,\"OPENSHELL_NETWORK_NAME\":\"%s\",\"OPENSHELL_STOP_TIMEOUT\":%s,\"OPENSHELL_SUPERVISOR_IMAGE\":\"%s\",\"OPENSHELL_PODMAN_TLS_CA\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_CERT\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_PODMAN_TLS_KEY\":{\"path\":\"%s\",\"sha256\":\"%s\"},\"OPENSHELL_ENABLE_BIND_MOUNTS\":%s}' \ + "${DRIVER_SOCKET}" \ + "${OPENSHELL_PODMAN_SOCKET:-}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ + "${EXTERNAL_DRIVER_PULL_POLICY}" \ + "${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ + "${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ + "${HOST_PORT}" \ + "${PODMAN_NETWORK_NAME}" \ + "${PODMAN_STOP_TIMEOUT_SECS}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ + "${EXTERNAL_DRIVER_TLS_CA}" \ + "${driver_tls_ca_sha256}" \ + "${EXTERNAL_DRIVER_TLS_CERT}" \ + "${driver_tls_cert_sha256}" \ + "${EXTERNAL_DRIVER_TLS_KEY}" \ + "${driver_tls_key_sha256}" \ + "${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}")" fi -} >> "${GATEWAY_CONFIG}" + printf '{"schema_version":%s,"gateway_port":%s,"external_compute_driver":%s,"compute_driver_transport":"%s","external_driver_pull_policy":"%s","supervisor_image":"%s","supervisor_image_id":"%s","supervisor_image_digest":"%s","supervisor_runtime_image":"%s","supervisor_base_image":"%s","supervisor_base_image_id":"%s","supervisor_base_image_digest":"%s","supervisor_base_runtime_image":"%s","supervisor_package_manifest_sha256":"%s","sandbox_image_request":"%s","sandbox_image_id":"%s","sandbox_image_digest":"%s","sandbox_runtime_image":"%s","sandbox_client_image_alias":"%s","sandbox_client_image_alias_id":"%s","gateway_sha256_before_execution":"%s","cli_sha256_before_execution":"%s","conformance_sha256_before_execution":"%s","external_driver_sha256_before_execution":"%s","supervisor_sha256_before_execution":"%s","supervisor_dockerfile_sha256_before_execution":"%s","cli_trace_wrapper_sha256_before_execution":"%s","external_driver_grpc_endpoint":%s,"external_driver_host_gateway_ip":%s,"external_driver_userns":%s,"external_driver_spiffe":%s,"external_driver_proxy":%s,"external_driver_app_armor":%s,"external_driver_environment":%s}\n' \ + "${CONFIG_SCHEMA_VERSION}" \ + "${HOST_PORT}" \ + "$([ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ] && printf true || printf false)" \ + "${driver_transport}" \ + "${EXTERNAL_DRIVER_PULL_POLICY}" \ + "${SUPERVISOR_IMAGE}" \ + "${SUPERVISOR_IMAGE_ID}" \ + "${SUPERVISOR_IMAGE_DIGEST}" \ + "${SUPERVISOR_RUNTIME_IMAGE}" \ + "${SUPERVISOR_BASE_IMAGE}" \ + "${SUPERVISOR_BASE_IMAGE_ID}" \ + "${SUPERVISOR_BASE_IMAGE_DIGEST}" \ + "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-${SUPERVISOR_BASE_IMAGE}@${SUPERVISOR_BASE_IMAGE_DIGEST}}" \ + "${SUPERVISOR_PACKAGE_MANIFEST_SHA256}" \ + "${SANDBOX_IMAGE_REQUEST}" \ + "${SANDBOX_IMAGE_ID}" \ + "${SANDBOX_IMAGE_DIGEST}" \ + "${SANDBOX_RUNTIME_IMAGE}" \ + "${SANDBOX_CLIENT_IMAGE_ALIAS}" \ + "${SANDBOX_CLIENT_IMAGE_ALIAS_ID}" \ + "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256:-}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_TRACE_WRAPPER_SHA256:-}" \ + "${external_driver_grpc_endpoint}" \ + "${external_driver_host_gateway_ip}" \ + "${external_driver_userns}" \ + "${external_driver_spiffe}" \ + "${external_driver_proxy}" \ + "${external_driver_app_armor}" \ + "${external_driver_environment}" \ + >"${OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE}" +fi if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + require_expected_sha256 "external compute driver" "${DRIVER_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256:-}" + env -i \ OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ - OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ - OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_RUNTIME_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="${EXTERNAL_DRIVER_PULL_POLICY}" \ + OPENSHELL_HEALTH_CHECK_INTERVAL_SECS="${EXTERNAL_DRIVER_HEALTH_CHECK_INTERVAL_SECS}" \ + OPENSHELL_GRPC_ENDPOINT="${EXTERNAL_DRIVER_CALLBACK_ENDPOINT}" \ OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ - OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ - OPENSHELL_PODMAN_TLS_CA="${PKI_DIR}/ca.crt" \ - OPENSHELL_PODMAN_TLS_CERT="${PKI_DIR}/client/tls.crt" \ - OPENSHELL_PODMAN_TLS_KEY="${PKI_DIR}/client/tls.key" \ - OPENSHELL_ENABLE_BIND_MOUNTS=true \ + OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ + OPENSHELL_PODMAN_TLS_CA="${EXTERNAL_DRIVER_TLS_CA}" \ + OPENSHELL_PODMAN_TLS_CERT="${EXTERNAL_DRIVER_TLS_CERT}" \ + OPENSHELL_PODMAN_TLS_KEY="${EXTERNAL_DRIVER_TLS_KEY}" \ + OPENSHELL_ENABLE_BIND_MOUNTS="${EXTERNAL_DRIVER_ENABLE_BIND_MOUNTS}" \ "${DRIVER_BIN}" >"${DRIVER_LOG}" 2>&1 & DRIVER_PID=$! e2e_wait_for_socket \ @@ -519,7 +727,7 @@ fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_drivers comes from the RPM template. Override the loopback address + # compute_driver comes from the RPM template. Override the loopback address # and port so Podman Machine can keep its IPv4 callback listener distinct. --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" @@ -549,7 +757,10 @@ e2e_export_gateway_restart_metadata \ "${GATEWAY_LOG}" \ "${GATEWAY_PID_FILE}" -OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ +require_expected_sha256 "gateway binary" "${GATEWAY_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256:-}" +OPENSHELL_LOCAL_TLS_DIR="${PKI_DIR}" \ +OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_RUNTIME_IMAGE}" \ OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" >"${GATEWAY_LOG}" 2>&1 & GATEWAY_PID=$! @@ -599,5 +810,11 @@ if [ "${elapsed}" -ge "${timeout}" ]; then exit 1 fi +require_expected_sha256 "OpenShell CLI" "${CLI_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_CLI_SHA256:-}" +if [ -n "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256:-}" ]; then + require_expected_sha256 "conformance CLI" "${OPENSHELL_CONFORMANCE_BIN}" \ + "${OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256}" +fi echo "Running e2e command against ${CLI_GATEWAY_ENDPOINT}: $*" "$@" diff --git a/examples/aws-s3-sts.md b/examples/aws-s3-sts.md index ed0c204d6b..94b73c73cc 100644 --- a/examples/aws-s3-sts.md +++ b/examples/aws-s3-sts.md @@ -87,13 +87,11 @@ if your gateway cache directory differs): ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -compute_drivers = ["podman"] -default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +compute_driver = "podman" disable_tls = true -supervisor_image = "localhost/openshell/supervisor:dev" [openshell.gateway.auth] allow_unauthenticated_users = true @@ -106,7 +104,10 @@ gateway_id = "podman-dev" ttl_secs = 3600 [openshell.drivers.podman] -image_pull_policy = "missing" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +supervisor_image = "localhost/openshell/supervisor:dev" +image_pull_policy = "if_not_present" +health_check_interval_secs = 10 ``` If the JWT key files do not exist yet, run `mise run gateway` once to generate @@ -118,7 +119,7 @@ Start the gateway: eval "$(aws configure export-credentials --format env)" ./target/debug/openshell-gateway \ --config .cache/gateway-podman/gateway.toml \ - --port 18080 --log-level info --drivers podman --disable-tls \ + --port 18080 --log-level info --compute-driver podman --disable-tls \ --db-url "sqlite:.cache/gateway-podman/gateway.db?mode=rwc" ``` diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index e25ee0d3cb..8e777f921f 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -325,7 +325,7 @@ generate_gateway_jwt_bundle() { write_gateway_config() { cat >"$GATEWAY_CONFIG" <"$config_path" <"$config_path" <"$GATEWAY_CONFIG" < None: + config = tomllib.loads((FIXTURE_DIR / fixture_name).read_text(encoding="utf-8")) + gateway = config["openshell"]["gateway"] + + assert config["openshell"]["version"] == 2 + assert gateway["compute_driver"] == driver + assert "compute_drivers" not in gateway + assert "sandbox_namespace" not in gateway + + +def test_podman_test_guest_producer_uses_schema_v2() -> None: + tasks = yaml.safe_load(TEST_GUEST_ROLE_PATH.read_text(encoding="utf-8")) + producer = next( + task + for task in tasks + if task.get("name") == "Write development Podman gateway configuration" + ) + content = producer["ansible.builtin.copy"]["content"] + config = tomllib.loads(content) + gateway = config["openshell"]["gateway"] + + assert config["openshell"]["version"] == 2 + assert gateway["compute_driver"] == "podman" + assert "compute_drivers" not in gateway + assert "ttl_secs" not in gateway["gateway_jwt"] + assert "podman" in config["openshell"]["drivers"] + + +def test_docker_e2e_gateway_fixture_uses_canonical_policy_and_label() -> None: + config = tomllib.loads((FIXTURE_DIR / "docker.toml").read_text(encoding="utf-8")) + docker = config["openshell"]["drivers"]["docker"] + + assert docker["image_pull_policy"] == "if_not_present" + assert docker["sandbox_label"] == "openshell-e2e" + assert "sandbox_namespace" not in docker + + +def test_podman_e2e_gateway_fixture_uses_canonical_policy_and_callbacks() -> None: + config = tomllib.loads((FIXTURE_DIR / "podman.toml").read_text(encoding="utf-8")) + podman = config["openshell"]["drivers"]["podman"] + + assert podman["image_pull_policy"] == "if_not_present" + assert podman["health_check_interval_secs"] == 10 + assert podman["ssh_socket_path"] == "/run/openshell/ssh.sock" + assert "sandbox_namespace" not in podman diff --git a/python/openshell/gateway_schema_v2_capability_parity_test.py b/python/openshell/gateway_schema_v2_capability_parity_test.py new file mode 100644 index 0000000000..9d3ff80b65 --- /dev/null +++ b/python/openshell/gateway_schema_v2_capability_parity_test.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate the schema-v2 live capability-parity manifest without a runtime.""" + +from __future__ import annotations + +import re +import tomllib +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" + +REQUIRED_MANIFEST_FIELDS = { + "manifest_version", + "baseline_ref", + "baseline_commit", + "baseline_schema_version", + "candidate_ref", + "candidate_commit", + "candidate_schema_version", + "capabilities", +} +REQUIRED_CAPABILITY_FIELDS = { + "id", + "topics", + "origin_main_access_paths", + "schema_v2_access_paths", + "behavioral_oracle", + "required_environment", + "test_lane", + "status", +} +REQUIRED_TOPICS = { + "auth_tls_jwt", + "configuration_producers", + "credentials", + "database", + "docker", + "external_drivers", + "inference", + "interceptors", + "kubernetes", + "listeners", + "middleware", + "mxc", + "observability", + "packaging_upgrades", + "podman", + "vm", +} +REQUIRED_CAPABILITY_IDS = { + "configuration-source-precedence", + "schema-version-and-strict-layout", + "gateway-identity-and-logging", + "primary-health-and-metrics-listeners", + "database-url-and-persistence-backends", + "ssh-rate-limit-and-policy-posture", + "sandbox-service-routing", + "gateway-listener-tls-and-sni", + "plaintext-listener-mode", + "guest-callback-tls-ownership", + "oidc-bearer-authentication", + "mtls-user-authentication", + "unsafe-unauthenticated-user-mode", + "gateway-minted-sandbox-jwt", + "otlp-observability", + "gateway-interceptor-registration", + "supervisor-middleware-registration", + "provider-profile-sources", + "inference-control-plane-configuration", + "credential-driver-selection-and-kek", + "credential-driver-backend-tables", + "docker-image-and-callback-configuration", + "docker-security-and-provider-configuration", + "podman-image-and-callback-configuration", + "podman-runtime-security-and-health", + "kubernetes-core-placement-and-images", + "kubernetes-workspace-isolation", + "kubernetes-supervisor-topology", + "kubernetes-egress-spiffe-and-security", + "vm-launch-and-resource-configuration", + "vm-guest-security-and-spiffe", + "mxc-windows-driver-configuration", + "external-compute-driver-socket", + "helm-configuration-producer", + "local-launch-script-producers", + "e2e-fixture-producers", + "rpm-schema-upgrade", + "homebrew-debian-and-snap-upgrades", +} +ALLOWED_LANES = { + "deterministic", + "e2e-docker", + "e2e-podman", + "e2e-kubernetes", + "e2e-vm", + "windows-mxc", + "extension-driver", + "auth-oidc", + "observability", + "packaging", +} +# This inventory plans execution. Live results belong in the execution record, +# not in this baseline manifest, so a PASS cannot be accidentally implied. +ALLOWED_STATUSES = {"not_run", "blocked", "planned"} + + +def load_manifest() -> dict[str, Any]: + with MANIFEST_PATH.open("rb") as manifest_file: + return tomllib.load(manifest_file) + + +def require_nonempty_string(value: Any, field: str, entry_id: str) -> None: + assert isinstance(value, str) and value.strip(), f"{entry_id}: {field} is required" + + +def require_string_list(value: Any, field: str, entry_id: str) -> None: + assert isinstance(value, list) and value, f"{entry_id}: {field} must be non-empty" + assert all(isinstance(item, str) and item.strip() for item in value), ( + f"{entry_id}: {field} must contain only non-empty strings" + ) + assert len(value) == len(set(value)), f"{entry_id}: {field} contains duplicates" + + +def validate_manifest(manifest: dict[str, Any]) -> None: + assert set(manifest) == REQUIRED_MANIFEST_FIELDS, ( + "unexpected or missing manifest metadata" + ) + assert manifest["manifest_version"] == 1 + assert manifest["baseline_ref"] == "origin/main" + assert manifest["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert manifest["baseline_schema_version"] == 1 + assert manifest["candidate_ref"] == "HEAD" + assert manifest["candidate_commit"] == "8c868e430e9cd3284d7e274628419ab484ebcee0" + assert manifest["candidate_schema_version"] == 2 + + capabilities = manifest["capabilities"] + assert isinstance(capabilities, list) and capabilities, ( + "capabilities must be non-empty" + ) + ids: list[str] = [] + topics: set[str] = set() + for capability in capabilities: + assert isinstance(capability, dict), "each capability must be a TOML table" + assert set(capability) == REQUIRED_CAPABILITY_FIELDS, ( + "capability has unexpected or missing metadata" + ) + entry_id = capability["id"] + require_nonempty_string(entry_id, "id", "capability") + ids.append(entry_id) + require_string_list(capability["topics"], "topics", entry_id) + topics.update(capability["topics"]) + require_string_list( + capability["origin_main_access_paths"], + "origin_main_access_paths", + entry_id, + ) + require_string_list( + capability["schema_v2_access_paths"], + "schema_v2_access_paths", + entry_id, + ) + require_nonempty_string( + capability["behavioral_oracle"], "behavioral_oracle", entry_id + ) + require_nonempty_string( + capability["required_environment"], "required_environment", entry_id + ) + assert capability["test_lane"] in ALLOWED_LANES, ( + f"{entry_id}: unknown test lane {capability['test_lane']!r}" + ) + assert capability["status"] in ALLOWED_STATUSES, ( + f"{entry_id}: live PASS results are not valid in this planning manifest" + ) + + assert len(ids) == len(set(ids)), "capability IDs must be unique" + assert set(ids) == REQUIRED_CAPABILITY_IDS, ( + "capability inventory is incomplete or stale" + ) + assert topics == REQUIRED_TOPICS, ( + "topic inventory is incomplete or contains an unknown topic" + ) + + +def capability_by_id(manifest: dict[str, Any], capability_id: str) -> dict[str, Any]: + return next( + capability + for capability in manifest["capabilities"] + if capability["id"] == capability_id + ) + + +def test_schema_v2_capability_parity_manifest_is_well_formed() -> None: + validate_manifest(load_manifest()) + + +def test_frozen_comparison_commits_are_full_git_object_ids() -> None: + manifest = load_manifest() + + for field in ("baseline_commit", "candidate_commit"): + commit = manifest[field] + assert len(commit) == 40 + assert all(character in "0123456789abcdef" for character in commit) + + +def test_vm_gateway_inventory_excludes_standalone_driver_only_fields() -> None: + capability = capability_by_id( + load_manifest(), "vm-launch-and-resource-configuration" + ) + gateway_paths = " ".join( + capability["origin_main_access_paths"] + capability["schema_v2_access_paths"] + ) + + gateway_fields = set(re.findall(r"[a-z][a-z0-9_]*", gateway_paths)) + for standalone_field in ("launcher_bin", "log_level", "gpu_enabled", "gpu_mem_mib"): + assert standalone_field not in gateway_fields + assert "driver_dir" in gateway_fields + + +def test_new_docker_capabilities_are_not_attributed_to_origin_main() -> None: + capability = capability_by_id( + load_manifest(), "docker-security-and-provider-configuration" + ) + origin_paths = " ".join(capability["origin_main_access_paths"]) + candidate_paths = " ".join(capability["schema_v2_access_paths"]) + + assert "no origin/main Docker equivalent" in origin_paths + for added_field in ( + "https_proxy", + "provider_spiffe_workload_api_socket", + "app_armor_profile", + ): + assert added_field not in origin_paths + assert added_field in candidate_paths + + +@pytest.mark.parametrize("field", sorted(REQUIRED_MANIFEST_FIELDS - {"capabilities"})) +def test_manifest_rejects_missing_header_metadata(field: str) -> None: + manifest = deepcopy(load_manifest()) + del manifest[field] + + with pytest.raises(AssertionError, match="missing manifest metadata"): + validate_manifest(manifest) + + +@pytest.mark.parametrize("field", sorted(REQUIRED_CAPABILITY_FIELDS - {"id"})) +def test_manifest_rejects_missing_capability_metadata(field: str) -> None: + manifest = deepcopy(load_manifest()) + del manifest["capabilities"][0][field] + + with pytest.raises(AssertionError, match=r"metadata|required|must be"): + validate_manifest(manifest) + + +@pytest.mark.parametrize( + ("field", "value", "match"), + [ + ("topics", [], "must be non-empty"), + ("behavioral_oracle", "", "is required"), + ("required_environment", "", "is required"), + ("test_lane", "not-a-lane", "unknown test lane"), + ], +) +def test_manifest_rejects_malformed_capability_metadata( + field: str, value: Any, match: str +) -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][0][field] = value + + with pytest.raises(AssertionError, match=match): + validate_manifest(manifest) + + +def test_manifest_rejects_duplicate_capability_id() -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][1]["id"] = manifest["capabilities"][0]["id"] + + with pytest.raises(AssertionError, match=r"unique|incomplete"): + validate_manifest(manifest) + + +def test_manifest_does_not_claim_live_pass_results() -> None: + manifest = deepcopy(load_manifest()) + manifest["capabilities"][0]["status"] = "pass" + + with pytest.raises(AssertionError, match="live PASS"): + validate_manifest(manifest) diff --git a/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py new file mode 100644 index 0000000000..801703e041 --- /dev/null +++ b/python/openshell/gateway_schema_v2_compute_boundary_verifier_test.py @@ -0,0 +1,416 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise the retained-artifact verifier used by schema-v2 Step 10.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from types import ModuleType + +REPO_ROOT = Path(__file__).resolve().parents[2] +VERIFIER_PATH = REPO_ROOT / "e2e/parity/verify-results.py" +BASELINE_SHA = "1" * 40 +CANDIDATE_SHA = "2" * 40 +IMAGE_ID = "3" * 64 +IMAGE_DIGEST = f"sha256:{'4' * 64}" +RUNTIME_IMAGE = f"localhost/openshell/supervisor@{IMAGE_DIGEST}" +BASE_RUNTIME_IMAGE = f"docker.io/library/alpine@{IMAGE_DIGEST}" + + +def load_verifier() -> ModuleType: + spec = importlib.util.spec_from_file_location("parity_verifier", VERIFIER_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value) + "\n", encoding="utf-8") + + +def create_variant( + verifier: ModuleType, + results_dir: Path, + variant: str, + source_sha: str, + schema_version: int, +) -> None: + artifact_dir = results_dir / "artifacts" / variant + artifact_dir.mkdir(parents=True) + artifacts = { + "gateway": f"{variant}-gateway", + "cli": f"{variant}-cli", + "conformance": f"{variant}-conformance", + "supervisor": f"{variant}-supervisor", + "supervisor.Dockerfile": f"{variant}-dockerfile", + "cli-trace-wrapper": "shared-cli-trace-wrapper", + "external-driver": f"{variant}-external-driver", + "supervisor.packages.txt": "fixture-package-1.0-r0\n", + } + for filename, content in artifacts.items(): + (artifact_dir / filename).write_text(content, encoding="utf-8") + + write_json( + results_dir / f"{variant}.json", + { + "variant": variant, + "source_sha": source_sha, + "schema_version": schema_version, + "driver": "podman", + "scenario": "external-driver", + "gateway_profile": "driver-free", + "gateway_cargo_features": "--no-default-features --features telemetry", + "gateway_origin": "built_by_harness", + "cli_origin": "built_by_harness", + "conformance_origin": "built_by_harness", + "external_driver_origin": "built_by_harness", + "supervisor_origin": "built_by_harness", + "gateway_sha256": verifier.sha256(artifact_dir / "gateway"), + "cli_sha256": verifier.sha256(artifact_dir / "cli"), + "conformance_sha256": verifier.sha256(artifact_dir / "conformance"), + "supervisor_sha256": verifier.sha256(artifact_dir / "supervisor"), + "supervisor_dockerfile_sha256": verifier.sha256( + artifact_dir / "supervisor.Dockerfile" + ), + "cli_trace_wrapper_sha256": verifier.sha256( + artifact_dir / "cli-trace-wrapper" + ), + "external_driver_sha256": verifier.sha256(artifact_dir / "external-driver"), + "success": True, + }, + ) + selector = ( + 'compute_drivers = ["podman"]' + if schema_version == 1 + else 'compute_driver = "podman"' + ) + (results_dir / f"{variant}.gateway.toml").write_text( + f"""[openshell] +version = {schema_version} +[openshell.gateway] +{selector} +[openshell.drivers.podman] +socket_path = "/tmp/{variant}.sock" +""", + encoding="utf-8", + ) + policy = "missing" if schema_version == 1 else "if_not_present" + package_hash = verifier.sha256(artifact_dir / "supervisor.packages.txt") + result = json.loads((results_dir / f"{variant}.json").read_text(encoding="utf-8")) + write_json( + results_dir / f"{variant}.launch.json", + { + "schema_version": schema_version, + "gateway_port": 18181, + "external_compute_driver": True, + "compute_driver_transport": "remote_uds", + "external_driver_pull_policy": policy, + "supervisor_image_id": IMAGE_ID, + "supervisor_image_digest": IMAGE_DIGEST, + "supervisor_runtime_image": RUNTIME_IMAGE, + "supervisor_base_image": "alpine:fixture", + "supervisor_base_image_id": IMAGE_ID, + "supervisor_base_image_digest": IMAGE_DIGEST, + "supervisor_base_runtime_image": BASE_RUNTIME_IMAGE, + "supervisor_package_manifest_sha256": package_hash, + "sandbox_image_request": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_image_id": IMAGE_ID, + "sandbox_image_digest": IMAGE_DIGEST, + "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_client_image_alias": "example.invalid/sandbox:latest", + "sandbox_client_image_alias_id": IMAGE_ID, + "gateway_sha256_before_execution": result["gateway_sha256"], + "cli_sha256_before_execution": result["cli_sha256"], + "conformance_sha256_before_execution": result["conformance_sha256"], + "external_driver_sha256_before_execution": result["external_driver_sha256"], + "supervisor_sha256_before_execution": result["supervisor_sha256"], + "supervisor_dockerfile_sha256_before_execution": result[ + "supervisor_dockerfile_sha256" + ], + "cli_trace_wrapper_sha256_before_execution": result[ + "cli_trace_wrapper_sha256" + ], + "external_driver_grpc_endpoint": "https://host.containers.internal:18181", + "external_driver_host_gateway_ip": "host-gateway", + "external_driver_userns": None, + "external_driver_spiffe": False, + "external_driver_proxy": False, + "external_driver_app_armor": False, + "external_driver_environment": { + "OPENSHELL_COMPUTE_DRIVER_SOCKET": f"/tmp/{variant}.sock", + "OPENSHELL_PODMAN_SOCKET": f"/tmp/{variant}-podman.sock", + "OPENSHELL_SANDBOX_IMAGE": "example.invalid/sandbox@" + IMAGE_DIGEST, + "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": policy, + "OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10, + "OPENSHELL_GRPC_ENDPOINT": "https://host.containers.internal:18181", + "OPENSHELL_GATEWAY_PORT": 18181, + "OPENSHELL_NETWORK_NAME": f"{variant}-network", + "OPENSHELL_STOP_TIMEOUT": 15, + "OPENSHELL_SUPERVISOR_IMAGE": RUNTIME_IMAGE, + "OPENSHELL_PODMAN_TLS_CA": { + "path": f"/tmp/{variant}-pki/ca.crt", + "sha256": "8" * 64, + }, + "OPENSHELL_PODMAN_TLS_CERT": { + "path": f"/tmp/{variant}-pki/tls.crt", + "sha256": "9" * 64, + }, + "OPENSHELL_PODMAN_TLS_KEY": { + "path": f"/tmp/{variant}-pki/tls.key", + "sha256": "a" * 64, + }, + "OPENSHELL_ENABLE_BIND_MOUNTS": True, + }, + }, + ) + lifecycle = "\n".join( + f"[run fixture{marker} in 1ms: exit 0" for marker in verifier.ORACLE_MARKERS + ) + (results_dir / f"{variant}.log").write_text( + f"CLI conformance run ID: fixture\n" + f"gateway preflight connected: gateway=fixture, authentication=authenticated\n" + f"{lifecycle}\n{RUNTIME_IMAGE} {BASE_RUNTIME_IMAGE} example.invalid/sandbox@{IMAGE_DIGEST} example.invalid/sandbox:latest " + f'{IMAGE_ID} {IMAGE_DIGEST} {package_hash}\n"passed": true\n', + encoding="utf-8", + ) + (results_dir / f"{variant}.exec.stdout").write_bytes( + b"openshell-conformance-fixture\n" + ) + (results_dir / f"{variant}.driver.log").write_text( + "external driver fixture started\n", encoding="utf-8" + ) + + +def create_external_bundle(verifier: ModuleType, results_dir: Path) -> None: + create_variant(verifier, results_dir, "baseline", BASELINE_SHA, 1) + create_variant(verifier, results_dir, "candidate", CANDIDATE_SHA, 2) + write_json( + results_dir / "comparison.json", + { + "scenario": "external-driver", + "baseline_success": True, + "candidate_success": True, + "parity": True, + "accepted": True, + "classification": "pass", + }, + ) + + +def test_verifier_recomputes_retained_artifact_hashes(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + + report = verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + assert report["baseline"]["artifacts_verified"] is True + assert report["candidate"]["raw_output_verified"] is True + + (tmp_path / "artifacts/candidate/gateway").write_text( + "mutated after execution", encoding="utf-8" + ) + with pytest.raises(ValueError, match="retained artifact hash mismatch"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_mutable_sandbox_request(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["sandbox_image_request"] = "example.invalid/sandbox:latest" + write_json(launch_path, launch) + + with pytest.raises( + ValueError, match="request was not the resolved digest reference" + ): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_different_sandbox_artifacts(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + other_id = "5" * 64 + other_digest = f"sha256:{'6' * 64}" + other_runtime = f"example.invalid/sandbox@{other_digest}" + launch.update( + { + "sandbox_image_request": other_runtime, + "sandbox_image_id": other_id, + "sandbox_image_digest": other_digest, + "sandbox_runtime_image": other_runtime, + "sandbox_client_image_alias_id": other_id, + } + ) + launch["external_driver_environment"]["OPENSHELL_SANDBOX_IMAGE"] = other_runtime + write_json(launch_path, launch) + with (tmp_path / "candidate.log").open("a", encoding="utf-8") as log: + log.write(f"{other_id} {other_digest} {other_runtime}\n") + + with pytest.raises(ValueError, match="sandbox image ID differ"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_different_supervisor_packages(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + package_path = tmp_path / "artifacts/candidate/supervisor.packages.txt" + package_path.write_text("fixture-package-2.0-r0\n", encoding="utf-8") + package_hash = verifier.sha256(package_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["supervisor_package_manifest_sha256"] = package_hash + write_json(launch_path, launch) + with (tmp_path / "candidate.log").open("a", encoding="utf-8") as log: + log.write(f"{package_hash}\n") + + with pytest.raises(ValueError, match="supervisor package manifest differ"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_unattested_external_driver_input(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + del launch["external_driver_environment"]["OPENSHELL_STOP_TIMEOUT"] + write_json(launch_path, launch) + + with pytest.raises(ValueError, match="allowlisted environment is incomplete"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_binds_gateway_and_driver_uds(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["external_driver_environment"]["OPENSHELL_COMPUTE_DRIVER_SOCKET"] = ( + "/tmp/different.sock" + ) + write_json(launch_path, launch) + + with pytest.raises(ValueError, match="driver socket differs from gateway TOML"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_reused_external_uds(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + launch_path = tmp_path / "candidate.launch.json" + launch = json.loads(launch_path.read_text(encoding="utf-8")) + launch["external_driver_environment"]["OPENSHELL_COMPUTE_DRIVER_SOCKET"] = ( + "/tmp/baseline.sock" + ) + write_json(launch_path, launch) + config_path = tmp_path / "candidate.gateway.toml" + config_path.write_text( + config_path.read_text(encoding="utf-8").replace( + "/tmp/candidate.sock", "/tmp/baseline.sock" + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="reuse the same external compute-driver UDS"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_requires_exact_exec_stdout(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + (tmp_path / "candidate.exec.stdout").write_text("wrong marker\n", encoding="utf-8") + + with pytest.raises(ValueError, match="stdout is not the exact marker"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_requires_authenticated_preflight(tmp_path: Path) -> None: + verifier = load_verifier() + create_external_bundle(verifier, tmp_path) + log_path = tmp_path / "candidate.log" + log_path.write_text( + log_path.read_text(encoding="utf-8").replace( + "authentication=authenticated", "authentication=unauthenticated" + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="authenticated gateway preflight is missing"): + verifier.verify_topology( + tmp_path, BASELINE_SHA, CANDIDATE_SHA, "external-driver" + ) + + +def test_verifier_rejects_cross_topology_sandbox_drift() -> None: + verifier = load_verifier() + launch = { + "sandbox_image_id": IMAGE_ID, + "sandbox_image_digest": IMAGE_DIGEST, + "sandbox_runtime_image": "example.invalid/sandbox@" + IMAGE_DIGEST, + "sandbox_client_image_alias": "example.invalid/sandbox:latest", + "sandbox_client_image_alias_id": IMAGE_ID, + "supervisor_base_image": "alpine:fixture", + "supervisor_base_image_id": IMAGE_ID, + "supervisor_base_image_digest": IMAGE_DIGEST, + "supervisor_base_runtime_image": BASE_RUNTIME_IMAGE, + "supervisor_package_manifest_sha256": "7" * 64, + } + shared_artifacts = { + "cli": "b" * 64, + "conformance": "c" * 64, + "supervisor": "d" * 64, + "supervisor.Dockerfile": "e" * 64, + "cli-trace-wrapper": "f" * 64, + } + in_tree = { + "baseline": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + "candidate": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + } + external = { + "baseline": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + "candidate": { + "launch_attestation": dict(launch), + "artifact_sha256": dict(shared_artifacts), + }, + } + external["candidate"]["launch_attestation"]["sandbox_image_id"] = "8" * 64 + + with pytest.raises(ValueError, match="four runs use different sandbox artifact"): + verifier.verify_four_run_provenance(in_tree, external) diff --git a/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py b/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py new file mode 100644 index 0000000000..dc83417e85 --- /dev/null +++ b/python/openshell/gateway_schema_v2_cross_cutting_dispositions_test.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate schema-v2 Step 11 cross-cutting capability dispositions.""" + +from __future__ import annotations + +import hashlib +import re +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +DISPOSITIONS_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml" +) +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" +INTENTIONAL_CHANGES_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-intentional-changes.toml" +) + +STEP_11_CAPABILITY_IDS = { + "credential-driver-backend-tables", + "credential-driver-selection-and-kek", + "gateway-interceptor-registration", + "gateway-minted-sandbox-jwt", + "inference-control-plane-configuration", + "mtls-user-authentication", + "oidc-bearer-authentication", + "otlp-observability", + "provider-profile-sources", + "supervisor-middleware-registration", + "unsafe-unauthenticated-user-mode", +} +EXPECTED_SUITES = { + "server-lib": "cargo test -q -p openshell-server --lib", + "gateway-interceptors": "cargo test -q -p openshell-gateway-interceptors", + "supervisor-middleware": "cargo test -q -p openshell-supervisor-middleware", + "otel-test-support": "cargo test -q -p openshell-otel-test-support", + "multiplex-tls-integration": ( + "cargo test -q -p openshell-server --test multiplex_tls_integration" + ), + "edge-tunnel-auth": "cargo test -q -p openshell-server --test edge_tunnel_auth", +} +EXPECTED_CANDIDATE_COMMIT = "363d8540830b2ea294d43198daa2b7a283a2face" +EXPECTED_INTENTIONAL_CHANGES = { + "gateway-jwt-zero-sentinel-removed": "gateway-minted-sandbox-jwt", + "middleware-payload-name-normalized": "supervisor-middleware-registration", +} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def assert_full_sha(value: object, field: str) -> str: + assert isinstance(value, str), f"{field} must be a string" + assert re.fullmatch(r"[0-9a-f]{40}", value), f"{field} must be a full SHA" + return value + + +def test_step_11_dispositions_cover_exact_cross_cutting_capabilities() -> None: + manifest = load_toml(DISPOSITIONS_PATH) + + assert set(manifest) == { + "manifest_version", + "baseline_commit", + "candidate_commit", + "overall_status", + "deterministic_preflight", + "capabilities", + } + assert manifest["manifest_version"] == 1 + assert manifest["overall_status"] == "platform_blocked" + assert ( + assert_full_sha(manifest["baseline_commit"], "baseline_commit") + == load_toml(CAPABILITY_PATH)["baseline_commit"] + ) + assert ( + assert_full_sha(manifest["candidate_commit"], "candidate_commit") + == EXPECTED_CANDIDATE_COMMIT + ) + + capabilities = manifest["capabilities"] + assert {entry["id"] for entry in capabilities} == STEP_11_CAPABILITY_IDS + assert len(capabilities) == len(STEP_11_CAPABILITY_IDS) + for entry in capabilities: + assert set(entry) == {"id", "status", "owner", "lane", "blocker"} + assert entry["status"] == "platform_blocked" + assert all(entry[field].strip() for field in ("owner", "lane")) + assert len(entry["blocker"]) >= 160 + + +def test_step_11_capability_ids_exist_in_inventory() -> None: + inventory_ids = { + entry["id"] for entry in load_toml(CAPABILITY_PATH)["capabilities"] + } + assert inventory_ids >= STEP_11_CAPABILITY_IDS + + +def test_deterministic_preflight_is_not_reported_as_live_parity() -> None: + manifest = load_toml(DISPOSITIONS_PATH) + preflight = manifest["deterministic_preflight"] + + assert set(preflight) == { + "status", + "baseline_attestation", + "baseline_attestation_sha256", + "candidate_attestation", + "candidate_attestation_sha256", + "commands", + "evidence", + } + assert preflight["status"] == "pass" + assert set(preflight["commands"]) == set(EXPECTED_SUITES.values()) + assert len(preflight["commands"]) == len(EXPECTED_SUITES) + assert len(preflight["evidence"]) >= 4 + assert all(item.strip() for item in preflight["evidence"]) + + assert manifest["overall_status"] == "platform_blocked" + assert all( + entry["status"] == "platform_blocked" for entry in manifest["capabilities"] + ) + + attestations = { + "baseline": manifest["baseline_commit"], + "candidate": manifest["candidate_commit"], + } + for variant, source_commit in attestations.items(): + path_field = f"{variant}_attestation" + digest_field = f"{variant}_attestation_sha256" + path = REPO_ROOT / preflight[path_field] + assert path.is_file(), f"missing tracked Step 11 attestation: {path}" + assert re.fullmatch(r"[0-9a-f]{64}", preflight[digest_field]) + assert hashlib.sha256(path.read_bytes()).hexdigest() == preflight[digest_field] + + attestation = path.read_text(encoding="utf-8") + assert "schema_v2_step11_deterministic_attestation_version=1\n" in attestation + assert f"variant={variant}\n" in attestation + assert f"source_commit={source_commit}\n" in attestation + assert "source_tree_clean=true\n" in attestation + assert "cargo_target_scope=checkout-local\n" in attestation + assert "rustc_wrapper=disabled\n" in attestation + assert attestation.endswith("\nattestation_complete=true\n") + suite_sections = {} + for section in attestation.split("\n=== suite:")[1:]: + suite, separator, body = section.partition(" ===\n") + assert separator + assert suite not in suite_sections + suite_sections[suite] = body + assert set(suite_sections) == set(EXPECTED_SUITES) + for suite, command in EXPECTED_SUITES.items(): + section = suite_sections[suite] + assert section.startswith(f"command={command} \n--- output ---\n") + assert section.count("--- exit_status:0 ---") == 1 + assert "--- exit_status:" not in section.replace( + "--- exit_status:0 ---", "" + ) + + +def test_step_11_representation_changes_remain_in_intentional_change_ledger() -> None: + changes = { + entry["id"]: entry + for entry in load_toml(INTENTIONAL_CHANGES_PATH)["intentional_changes"] + } + + for change_id, capability_id in EXPECTED_INTENTIONAL_CHANGES.items(): + assert change_id in changes + change = changes[change_id] + assert change["parity_disposition"] == "intentional_change" + assert capability_id in change["validation_capability_ids"] diff --git a/python/openshell/gateway_schema_v2_intentional_changes_test.py b/python/openshell/gateway_schema_v2_intentional_changes_test.py new file mode 100644 index 0000000000..84e8f6968b --- /dev/null +++ b/python/openshell/gateway_schema_v2_intentional_changes_test.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate the explicit schema-v2 intentional-change ledger.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +LEDGER_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-intentional-changes.toml" +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" + +REQUIRED_HEADER_FIELDS = { + "ledger_version", + "issue", + "baseline_commit", + "candidate_start_commit", + "intentional_changes", +} +REQUIRED_CHANGE_FIELDS = { + "id", + "category", + "origin_main_contract", + "schema_v2_contract", + "migration", + "rationale", + "parity_disposition", + "validation_capability_ids", +} +REQUIRED_CHANGE_IDS = { + "canonical-image-pull-policy", + "docker-sandbox-label-rename", + "driver-table-exclusive-ownership", + "gateway-jwt-zero-sentinel-removed", + "guest-tls-centralized", + "kubernetes-fields-relocated", + "middleware-payload-name-normalized", + "package-default-only-auto-migration", + "podman-health-zero-sentinel-removed", + "podman-pid-limit-restored", + "podman-ssh-socket-rename", + "sandbox-pid-zero-sentinel-removed", + "schema-version-cutover", + "singular-compute-driver-selector", + "vm-grpc-endpoint-rename", + "vm-sandbox-identity-selection", +} +ALLOWED_CATEGORIES = { + "bug_fix", + "cardinality", + "default_behavior", + "migration_policy", + "ownership", + "relocation", + "rename", + "rename_with_alias", + "schema_cutover", + "sentinel_removal", + "type_normalization", +} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def require_nonempty_string(value: Any, field: str, change_id: str) -> None: + assert isinstance(value, str) and value.strip(), f"{change_id}: {field} is required" + + +def test_intentional_change_ledger_is_complete_and_well_formed() -> None: + ledger = load_toml(LEDGER_PATH) + + assert set(ledger) == REQUIRED_HEADER_FIELDS + assert ledger["ledger_version"] == 1 + assert ledger["issue"] == 2792 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert ledger["candidate_start_commit"] == ( + "8c868e430e9cd3284d7e274628419ab484ebcee0" + ) + + changes = ledger["intentional_changes"] + assert isinstance(changes, list) and changes + ids: list[str] = [] + for change in changes: + assert set(change) == REQUIRED_CHANGE_FIELDS + change_id = change["id"] + require_nonempty_string(change_id, "id", "intentional change") + ids.append(change_id) + assert change["category"] in ALLOWED_CATEGORIES + assert change["parity_disposition"] == "intentional_change" + for field in ( + "origin_main_contract", + "schema_v2_contract", + "migration", + "rationale", + ): + require_nonempty_string(change[field], field, change_id) + capability_ids = change["validation_capability_ids"] + assert isinstance(capability_ids, list) and capability_ids + assert len(capability_ids) == len(set(capability_ids)) + + assert len(ids) == len(set(ids)) + assert set(ids) == REQUIRED_CHANGE_IDS + + +def test_every_intentional_change_links_to_known_capabilities() -> None: + ledger = load_toml(LEDGER_PATH) + capabilities = load_toml(CAPABILITY_PATH)["capabilities"] + known_capability_ids = {capability["id"] for capability in capabilities} + + for change in ledger["intentional_changes"]: + assert set(change["validation_capability_ids"]) <= known_capability_ids, ( + f"{change['id']}: unknown validation capability" + ) + + +def test_ledger_does_not_hide_known_unresolved_parity_gaps() -> None: + ledger = load_toml(LEDGER_PATH) + change_ids = {change["id"] for change in ledger["intentional_changes"]} + + assert "legacy-environment-selector-upgrade" not in change_ids + assert "tls-require-client-auth-ignored" not in change_ids + assert "debian-snap-v1-upgrade" not in change_ids + + +def test_singular_selector_ledger_preserves_auto_detection() -> None: + ledger = load_toml(LEDGER_PATH) + selector = next( + change + for change in ledger["intentional_changes"] + if change["id"] == "singular-compute-driver-selector" + ) + + assert "omission retains built-in auto-detection" in selector["schema_v2_contract"] + assert "unsupported multi-driver operation" in selector["rationale"] + + +def test_operator_edited_package_configuration_is_never_auto_rewritten() -> None: + ledger = load_toml(LEDGER_PATH) + package_policy = next( + change + for change in ledger["intentional_changes"] + if change["id"] == "package-default-only-auto-migration" + ) + + contract = package_policy["schema_v2_contract"] + assert "byte-identical package defaults" in contract + assert "Debian and Snap preserve every legacy file" in contract + assert "fail closed through read-only package preflight" in contract + assert "manual-migration guidance" in contract + assert "no safe provenance marker" in contract + assert "preserve every edited file" in package_policy["migration"] + assert "manual schema-v2 conversion" in package_policy["migration"] diff --git a/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py b/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py new file mode 100644 index 0000000000..deb413b0da --- /dev/null +++ b/python/openshell/gateway_schema_v2_kubernetes_option_dispositions_test.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate field-level Step 8 Kubernetes parity dispositions.""" + +import json +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +LEDGER = ( + ROOT + / "e2e" + / "configs" + / "gateway" + / "schema-v2-kubernetes-option-dispositions.toml" +) +CAPABILITIES = ROOT / "e2e" / "configs" / "gateway" / "schema-v2-capability-parity.toml" +LIVE_RESULTS = ROOT / "e2e" / "configs" / "gateway" / "schema-v2-live-results.toml" +KUBERNETES_CAPABILITY_IDS = { + "kubernetes-core-placement-and-images", + "kubernetes-workspace-isolation", + "kubernetes-supervisor-topology", + "kubernetes-egress-spiffe-and-security", +} +FIELD_GROUP = re.compile( + r"\[openshell\.(?Pgateway|drivers\.kubernetes(?:\.(?P[a-z_]+))?)\]" + r"\.\{(?P[^}]+)\}" +) + + +def load_ledger() -> dict: + with LEDGER.open("rb") as handle: + return tomllib.load(handle) + + +def manifest_kubernetes_fields() -> set[str]: + with CAPABILITIES.open("rb") as handle: + capabilities = tomllib.load(handle)["capabilities"] + fields: set[str] = set() + for capability in capabilities: + if capability["id"] not in KUBERNETES_CAPABILITY_IDS: + continue + for access_path in [ + *capability["origin_main_access_paths"], + *capability["schema_v2_access_paths"], + ]: + for match in FIELD_GROUP.finditer(access_path): + table = match.group("table") + if table == "gateway" and "inherited by Kubernetes" not in access_path: + continue + prefix = ( + f"{match.group('subtable')}." if match.group("subtable") else "" + ) + fields.update( + f"{prefix}{field.strip()}" + for field in match.group("fields").split(",") + ) + return fields + + +def test_kubernetes_ledger_covers_every_manifest_driver_field_once() -> None: + ledger = load_ledger() + coverage = ledger["coverage"] + observed = [field for entry in coverage for field in entry["fields"]] + + assert set(observed) == manifest_kubernetes_fields() + assert len(observed) == len(set(observed)) + + +def test_kubernetes_core_pass_is_paired_and_qualified_checks_stay_blocked() -> None: + ledger = load_ledger() + core = next(entry for entry in ledger["coverage"] if entry["status"] == "pass") + + assert core["id"] == "shared-combined-core" + assert len(core["evidence"]) >= 3 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert len(ledger["validated_candidate_commit"]) == 40 + + comparison_path = ROOT / ledger["core_comparison"] + comparison = json.loads(comparison_path.read_text()) + assert comparison == { + "accepted": True, + "baseline_commit": ledger["baseline_commit"], + "baseline_success": True, + "candidate_commit": ledger["validated_candidate_commit"], + "candidate_success": True, + "classification": "pass", + "parity": True, + } + with LIVE_RESULTS.open("rb") as handle: + live_results = tomllib.load(handle)["result"] + live_core = next( + result + for result in live_results + if result["id"] == "kubernetes-core-option-parity" + ) + assert live_core["status"] == comparison["classification"] + assert live_core["validated_baseline_commit"] == comparison["baseline_commit"] + assert live_core["validated_candidate_commit"] == comparison["candidate_commit"] + + blocked = [ + entry + for entry in [*ledger["coverage"], *ledger["qualified_value"]] + if entry["status"] == "platform_blocked" + ] + assert blocked + for entry in blocked: + assert entry["owner"] + assert entry["lane"] + assert entry["requirement"] + + +def test_environment_qualified_security_checks_are_not_claimed_by_core() -> None: + ledger = load_ledger() + blocked_fields = { + field + for entry in ledger["coverage"] + if entry["status"] == "platform_blocked" + for field in entry["fields"] + } + qualified = {entry["id"] for entry in ledger["qualified_value"]} + + assert { + "enable_user_namespaces", + "https_proxy", + "proxy_auth_secret_name", + "provider_spiffe_workload_api_socket_path", + "sidecar.proxy_uid", + } <= blocked_fields + assert { + "apparmor-enforcement", + "gpu-resource-combinations", + "image-volume-sideload", + "managed-and-operator-workspace-values", + "runtime-class-isolation", + "sidecar-topology-value", + } <= qualified diff --git a/python/openshell/gateway_schema_v2_live_results_test.py b/python/openshell/gateway_schema_v2_live_results_test.py new file mode 100644 index 0000000000..3cb52c5d62 --- /dev/null +++ b/python/openshell/gateway_schema_v2_live_results_test.py @@ -0,0 +1,449 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate machine-readable schema-v2 live parity results.""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +RESULTS_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-live-results.toml" +CAPABILITY_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-capability-parity.toml" +COMPUTE_BOUNDARY_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-compute-boundary-comparison.json" +) +CROSS_CUTTING_DISPOSITIONS_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-cross-cutting-dispositions.toml" +) + +REQUIRED_HEADER_FIELDS = { + "manifest_version", + "baseline_commit", + "candidate_start_commit", + "result", +} +REQUIRED_STEP_5_IDS = { + "portable-lifecycle-docker", + "portable-lifecycle-kubernetes", + "portable-lifecycle-mxc", + "portable-lifecycle-podman", + "portable-lifecycle-vm", +} +REQUIRED_STEP_6_IDS = { + "gateway-tls-client-auth-policy", + "gateway-wide-process-options", +} +REQUIRED_STEP_7_IDS = { + "docker-driver-option-parity", + "podman-driver-option-parity", + "podman-qualified-security-option-parity", +} +REQUIRED_STEP_8_IDS = { + "kubernetes-core-option-parity", + "kubernetes-qualified-option-parity", +} +REQUIRED_STEP_9_IDS = { + "vm-guest-security-and-spiffe", + "vm-launch-and-resource-configuration", +} +REQUIRED_STEP_10_IDS = {"compute-driver-boundary-parity"} +REQUIRED_STEP_11_IDS = { + "credential-driver-backend-tables", + "credential-driver-selection-and-kek", + "gateway-interceptor-registration", + "gateway-minted-sandbox-jwt", + "inference-control-plane-configuration", + "mtls-user-authentication", + "oidc-bearer-authentication", + "otlp-observability", + "provider-profile-sources", + "supervisor-middleware-registration", + "unsafe-unauthenticated-user-mode", +} +REQUIRED_STEP_12_IDS = { + "debian-package-upgrade", + "homebrew-package-upgrade", + "rpm-package-upgrade", + "snap-package-refresh", +} +EXPECTED_EXECUTED_CANDIDATE_COMMITS = { + "portable-lifecycle-podman": "a3860084d019ed2ac979e3eaa1ddf085a96b773c", + "gateway-wide-process-options": "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea", + "gateway-tls-client-auth-policy": "e6aac1aa7c624c5df43535ab4fbe2bc6f9697dea", + "podman-driver-option-parity": "e09070d4ba0b30f0ac278fbb9938c1520ecff696", + "kubernetes-core-option-parity": "0f08b5822e4da98c9ced3d4b0f2bf4f30dae28fd", + "compute-driver-boundary-parity": "4a39da510e4d278a24dd60291149519c9a570b46", +} +STEP_10_CANDIDATE_COMMIT = EXPECTED_EXECUTED_CANDIDATE_COMMITS[ + "compute-driver-boundary-parity" +] +STEP_10_REPORT_SHA256 = ( + "65541eec5f642461a88b04b5459474fd7a475adeb7071a65a53fe183caad6a01" +) +STEP_10_EVIDENCE_BUNDLES = { + "in_tree": "target/parity/step10-intree-4a39da51", + "external_uds": "target/parity/step10-external-4a39da51", +} +ALLOWED_STATUSES = { + "pass", + "intentional_change", + "regression", + "platform_blocked", +} +BASE_FIELDS = {"id", "step", "capability", "driver", "status", "lane"} +PASS_FIELDS = { + "validated_baseline_commit", + "validated_candidate_commit", + "evidence", +} +BLOCKED_FIELDS = {"owner", "blocker"} + + +def load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as toml_file: + return tomllib.load(toml_file) + + +def assert_full_sha(value: object, field: str) -> str: + assert isinstance(value, str), f"{field} must be a string" + assert len(value) == 40 and all(char in "0123456789abcdef" for char in value), ( + f"{field} must be a full lowercase SHA" + ) + return value + + +def test_live_results_manifest_is_well_formed() -> None: + manifest = load_toml(RESULTS_PATH) + + assert set(manifest) == REQUIRED_HEADER_FIELDS + assert manifest["manifest_version"] == 1 + baseline = assert_full_sha(manifest["baseline_commit"], "baseline_commit") + assert_full_sha(manifest["candidate_start_commit"], "candidate_start_commit") + assert baseline == load_toml(CAPABILITY_PATH)["baseline_commit"] + + results = manifest["result"] + assert isinstance(results, list) and results + ids: list[str] = [] + for result in results: + assert set(result) >= BASE_FIELDS + result_id = result["id"] + assert isinstance(result_id, str) and result_id.strip() + ids.append(result_id) + assert result["status"] in ALLOWED_STATUSES + assert isinstance(result["step"], int) and 1 <= result["step"] <= 15 + for field in ("capability", "driver", "lane"): + assert isinstance(result[field], str) and result[field].strip(), ( + f"{result_id}: {field} is required" + ) + + assert len(ids) == len(set(ids)) + + +def test_step_5_covers_every_in_tree_compute_driver() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 5 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_5_IDS + assert {result["driver"] for result in results} == { + "docker", + "kubernetes", + "mxc", + "podman", + "vm", + } + + +def test_executed_results_pin_commits_and_evidence() -> None: + manifest = load_toml(RESULTS_PATH) + executed = { + result["id"]: result + for result in manifest["result"] + if result["status"] in {"pass", "intentional_change"} + } + assert set(executed) == set(EXPECTED_EXECUTED_CANDIDATE_COMMITS) + + for result_id, result in executed.items(): + assert set(result) >= PASS_FIELDS, result_id + assert result["validated_baseline_commit"] == manifest["baseline_commit"] + candidate = assert_full_sha( + result["validated_candidate_commit"], + f"{result_id}.validated_candidate_commit", + ) + assert candidate == EXPECTED_EXECUTED_CANDIDATE_COMMITS[result_id] + assert isinstance(result["evidence"], list) and result["evidence"] + assert all( + isinstance(item, str) and item.strip() for item in result["evidence"] + ) + # These immutable SHAs bind the original live-validation artifacts. + # A later history-preserving source rebase can intentionally make those + # exact execution commits non-ancestors without changing the evidence. + + +def test_step_6_records_gateway_option_and_tls_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 6 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_6_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["gateway-wide-process-options"] == "pass" + assert statuses["gateway-tls-client-auth-policy"] == "intentional_change" + + +def test_step_7_records_driver_option_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 7 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_7_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["podman-driver-option-parity"] == "intentional_change" + assert statuses["docker-driver-option-parity"] == "platform_blocked" + assert statuses["podman-qualified-security-option-parity"] == "platform_blocked" + + +def test_step_8_records_kubernetes_option_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 8 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_8_IDS + statuses = {result["id"]: result["status"] for result in results} + assert statuses["kubernetes-core-option-parity"] == "pass" + assert statuses["kubernetes-qualified-option-parity"] == "platform_blocked" + + +def test_step_9_records_vm_runtime_dispositions() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 9 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_9_IDS + assert all(result["status"] == "platform_blocked" for result in results) + assert all(result["driver"] == "vm" for result in results) + + +def test_step_10_records_verified_compute_boundary_parity(tmp_path: Path) -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 10 + ] + assert {result["id"] for result in results} == REQUIRED_STEP_10_IDS + assert results[0]["status"] == "pass" + + with COMPUTE_BOUNDARY_PATH.open(encoding="utf-8") as report_file: + report = json.load(report_file) + assert report["manifest_version"] == 2 + assert report["baseline_commit"] == load_toml(RESULTS_PATH)["baseline_commit"] + assert report["candidate_commit"] == results[0]["validated_candidate_commit"] + assert report["candidate_commit"] == STEP_10_CANDIDATE_COMMIT + assert report["retained_evidence_bundles"] == STEP_10_EVIDENCE_BUNDLES + assert ( + hashlib.sha256(COMPUTE_BOUNDARY_PATH.read_bytes()).hexdigest() + == STEP_10_REPORT_SHA256 + ) + assert report["classification"] == "pass" + assert report["accepted"] is True + assert all(report["oracle"].values()) + assert all(report["verification"].values()) + + launch_attestations = [] + for topology_name in ("in_tree", "external_uds"): + topology = report[topology_name] + assert topology["classification"] == "pass" + assert topology["parity"] is True + assert topology["accepted"] is True + assert re.fullmatch(r"[0-9a-f]{64}", topology["comparison_sha256"]) + baseline_launch = topology["baseline"]["launch_attestation"] + candidate_launch = topology["candidate"]["launch_attestation"] + for field in ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_package_manifest_sha256", + ): + assert baseline_launch[field] == candidate_launch[field] + launch_attestations.extend((baseline_launch, candidate_launch)) + + for variant_name, schema_version in (("baseline", 1), ("candidate", 2)): + variant = topology[variant_name] + assert variant["schema_version"] == schema_version + assert variant["success"] is True + assert variant["artifacts_verified"] is True + assert variant["raw_output_verified"] is True + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) + for digest in variant["artifact_sha256"].values() + ) + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) + for digest in variant["raw_evidence_sha256"].values() + ) + launch = variant["launch_attestation"] + supervisor_digest = launch["supervisor_image_digest"] + assert re.fullmatch(r"sha256:[0-9a-f]{64}", supervisor_digest) + assert launch["supervisor_runtime_image"].endswith(f"@{supervisor_digest}") + sandbox_digest = launch["sandbox_image_digest"] + assert re.fullmatch(r"sha256:[0-9a-f]{64}", sandbox_digest) + assert launch["sandbox_image_request"] == launch["sandbox_runtime_image"] + assert launch["sandbox_runtime_image"].endswith(f"@{sandbox_digest}") + + for field in ( + "sandbox_image_id", + "sandbox_image_digest", + "sandbox_runtime_image", + "supervisor_base_image", + "supervisor_base_image_id", + "supervisor_base_image_digest", + "supervisor_base_runtime_image", + "supervisor_package_manifest_sha256", + "supervisor_dockerfile_sha256_before_execution", + ): + assert len({launch[field] for launch in launch_attestations}) == 1, field + + external_sockets = [] + for variant_name in ("baseline", "candidate"): + external_variant = report["external_uds"][variant_name] + external_launch = external_variant["launch_attestation"] + assert external_launch["compute_driver_transport"] == "remote_uds" + assert external_launch["external_compute_driver"] is True + assert external_launch["external_driver_grpc_endpoint"].startswith("https://") + assert external_launch["external_driver_host_gateway_ip"] == "host-gateway" + assert external_launch["external_driver_userns"] is None + assert external_launch["external_driver_spiffe"] is False + assert external_launch["external_driver_proxy"] is False + assert external_launch["external_driver_app_armor"] is False + driver_environment = external_launch["external_driver_environment"] + assert driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"] + assert driver_environment["OPENSHELL_PODMAN_SOCKET"] + assert driver_environment["OPENSHELL_GRPC_ENDPOINT"].startswith("https://") + assert driver_environment["OPENSHELL_ENABLE_BIND_MOUNTS"] is True + for tls_field in ( + "OPENSHELL_PODMAN_TLS_CA", + "OPENSHELL_PODMAN_TLS_CERT", + "OPENSHELL_PODMAN_TLS_KEY", + ): + assert re.fullmatch( + r"[0-9a-f]{64}", driver_environment[tls_field]["sha256"] + ) + external_sockets.append(driver_environment["OPENSHELL_COMPUTE_DRIVER_SOCKET"]) + assert f"{variant_name}.driver.log" in external_variant["raw_evidence_sha256"] + assert f"{variant_name}.exec.stdout" in external_variant["raw_evidence_sha256"] + assert len(set(external_sockets)) == 2 + + evidence_paths = { + name: REPO_ROOT / relative_path + for name, relative_path in STEP_10_EVIDENCE_BUNDLES.items() + } + present = {name: path.is_dir() for name, path in evidence_paths.items()} + assert len(set(present.values())) == 1, "Step 10 retained evidence is incomplete" + if all(present.values()): + reproduced = tmp_path / "schema-v2-compute-boundary-comparison.json" + subprocess.run( + [ + "python3", + str(REPO_ROOT / "e2e/parity/verify-results.py"), + "--baseline-sha", + report["baseline_commit"], + "--candidate-sha", + STEP_10_CANDIDATE_COMMIT, + "--in-tree", + STEP_10_EVIDENCE_BUNDLES["in_tree"], + "--external-uds", + STEP_10_EVIDENCE_BUNDLES["external_uds"], + "--output", + str(reproduced), + ], + cwd=REPO_ROOT, + check=True, + ) + assert reproduced.read_bytes() == COMPUTE_BOUNDARY_PATH.read_bytes() + + +def test_step_11_records_cross_cutting_live_lane_dispositions() -> None: + results = { + result["id"]: result + for result in load_toml(RESULTS_PATH)["result"] + if result["step"] == 11 + } + dispositions = { + entry["id"]: entry + for entry in load_toml(CROSS_CUTTING_DISPOSITIONS_PATH)["capabilities"] + } + + assert set(results) == REQUIRED_STEP_11_IDS + assert set(dispositions) == REQUIRED_STEP_11_IDS + for result_id, result in results.items(): + disposition = dispositions[result_id] + assert result["status"] == "platform_blocked" + for field in ("status", "owner", "lane", "blocker"): + assert result[field] == disposition[field] + + +def test_step_12_keeps_real_package_upgrade_lanes_blocked() -> None: + results = [ + result for result in load_toml(RESULTS_PATH)["result"] if result["step"] == 12 + ] + + assert {result["id"] for result in results} == REQUIRED_STEP_12_IDS + assert all(result["status"] == "platform_blocked" for result in results) + by_id = {result["id"]: result for result in results} + + rpm = by_id["rpm-package-upgrade"] + assert rpm["owner"] == "OpenShell RPM package upgrade CI lane" + assert rpm["lane"] == "fedora-rpm-prior-release-upgrade" + assert "prior RPM" in rpm["blocker"] + assert "installed and upgraded" in rpm["blocker"] + + debian = by_id["debian-package-upgrade"] + assert debian["owner"] == "OpenShell Debian package upgrade CI lane" + assert debian["lane"] == "ubuntu-debian-prior-release-upgrade" + assert "installed prior Debian artifact" in debian["blocker"] + assert "source-tree tests" in debian["blocker"] + + snap = by_id["snap-package-refresh"] + assert snap["owner"] == "OpenShell Snap refresh CI lane" + assert snap["lane"] == "ubuntu-snap-prior-release-refresh" + assert "refresh confined Snap revisions" in snap["blocker"] + assert "source-tree tests" in snap["blocker"] + + homebrew = by_id["homebrew-package-upgrade"] + assert homebrew["owner"] == "OpenShell macOS Homebrew package upgrade CI lane" + assert homebrew["lane"] == "macos-homebrew-prior-release-upgrade" + assert "prior-release upgrade" in homebrew["blocker"] + assert "Linux host" in homebrew["blocker"] + + +def test_platform_blocked_results_name_owner_lane_and_blocker() -> None: + for result in load_toml(RESULTS_PATH)["result"]: + if result["status"] != "platform_blocked": + continue + + assert set(result) >= BLOCKED_FIELDS, result["id"] + assert isinstance(result["owner"], str) and result["owner"].strip() + assert isinstance(result["blocker"], str) and len(result["blocker"]) >= 80 + + +def test_step_5_records_only_executed_podman_as_pass() -> None: + statuses = { + result["driver"]: result["status"] + for result in load_toml(RESULTS_PATH)["result"] + if result["step"] == 5 + } + + assert statuses["podman"] == "pass" + assert all( + status == "platform_blocked" + for driver, status in statuses.items() + if driver != "podman" + ) diff --git a/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py new file mode 100644 index 0000000000..76c8f08bd4 --- /dev/null +++ b/python/openshell/gateway_schema_v2_parity_gap_dispositions_test.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate schema-v2 parity-gap dispositions and release blockers.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +GAP_LEDGER_PATH = ( + REPO_ROOT / "e2e/configs/gateway/schema-v2-parity-gap-dispositions.toml" +) +LIVE_RESULTS_PATH = REPO_ROOT / "e2e/configs/gateway/schema-v2-live-results.toml" + +REQUIRED_HEADER_FIELDS = { + "ledger_version", + "issue", + "baseline_commit", + "candidate_start_commit", + "gaps", +} +REQUIRED_GAP_FIELDS = { + "id", + "severity", + "parity_relation", + "disposition", + "origin_main_behavior", + "candidate_behavior", + "impact", + "resolution", + "validation", + "owner_step", +} +REQUIRED_GAP_IDS = { + "debian-snap-v1-upgrade", + "gateway-owned-guest-tls", + "generated-e2e-selector-shape", + "legacy-environment-selector-upgrade", + "multi-driver-runtime-loss", + "rpm-exact-default-migration", + "tls-require-client-auth-ignored", + "unselected-driver-validation-claim", +} +ALLOWED_SEVERITIES = {"blocker", "major", "minor", "none"} +ALLOWED_RELATIONS = { + "documentation_gap", + "non_finding", + "preexisting_inaccessible_option", + "regression", + "upgrade_regression", +} +ALLOWED_DISPOSITIONS = { + "documentation_fix_required", + "must_fix_before_parity", + "must_fix_before_release_gate", + "must_fix_or_remove_claim", + "no_action", + "resolved", +} + + +def load_ledger() -> dict[str, Any]: + with GAP_LEDGER_PATH.open("rb") as ledger_file: + return tomllib.load(ledger_file) + + +def test_gap_disposition_ledger_is_complete_and_well_formed() -> None: + ledger = load_ledger() + + assert set(ledger) == REQUIRED_HEADER_FIELDS + assert ledger["ledger_version"] == 1 + assert ledger["issue"] == 2792 + assert ledger["baseline_commit"] == "74960ebfaeec4673885089ed995fad902459749f" + assert ledger["candidate_start_commit"] == ( + "8c868e430e9cd3284d7e274628419ab484ebcee0" + ) + + gaps = ledger["gaps"] + assert isinstance(gaps, list) and gaps + ids: list[str] = [] + for gap in gaps: + assert set(gap) == REQUIRED_GAP_FIELDS + gap_id = gap["id"] + assert isinstance(gap_id, str) and gap_id.strip() + ids.append(gap_id) + assert gap["severity"] in ALLOWED_SEVERITIES + assert gap["parity_relation"] in ALLOWED_RELATIONS + assert gap["disposition"] in ALLOWED_DISPOSITIONS + assert isinstance(gap["owner_step"], int) and 1 <= gap["owner_step"] <= 15 + for field in ( + "origin_main_behavior", + "candidate_behavior", + "impact", + "resolution", + "validation", + ): + assert isinstance(gap[field], str) and gap[field].strip(), ( + f"{gap_id}: {field} is required" + ) + + assert len(ids) == len(set(ids)) + assert set(ids) == REQUIRED_GAP_IDS + + +def test_every_blocker_has_a_required_fix_and_validation() -> None: + for gap in load_ledger()["gaps"]: + if gap["severity"] != "blocker": + continue + + assert gap["disposition"].startswith("must_fix") + assert gap["owner_step"] in {6, 12} + assert len(gap["resolution"]) >= 80 + assert len(gap["validation"]) >= 80 + + +def test_non_findings_do_not_require_product_changes() -> None: + for gap in load_ledger()["gaps"]: + if gap["parity_relation"] == "non_finding": + assert gap["severity"] == "none" + assert gap["disposition"] == "no_action" + + +def test_security_sensitive_tls_gap_records_reviewed_resolution() -> None: + tls_gap = next( + gap + for gap in load_ledger()["gaps"] + if gap["id"] == "tls-require-client-auth-ignored" + ) + + assert tls_gap["parity_relation"] == "preexisting_inaccessible_option" + assert tls_gap["disposition"] == "resolved" + assert "Security review" in tls_gap["resolution"] + assert "rejects require_client_auth" in tls_gap["candidate_behavior"] + + +def test_step_6_gap_dispositions_are_resolved() -> None: + step_6_gaps = [gap for gap in load_ledger()["gaps"] if gap["owner_step"] == 6] + + assert {gap["id"] for gap in step_6_gaps} == { + "legacy-environment-selector-upgrade", + "tls-require-client-auth-ignored", + "unselected-driver-validation-claim", + } + assert all(gap["severity"] == "none" for gap in step_6_gaps) + assert all(gap["disposition"] == "resolved" for gap in step_6_gaps) + + +def test_step_12_product_gaps_are_resolved_without_claiming_live_package_parity() -> ( + None +): + step_12_gaps = [gap for gap in load_ledger()["gaps"] if gap["owner_step"] == 12] + assert {gap["id"] for gap in step_12_gaps} == { + "debian-snap-v1-upgrade", + "rpm-exact-default-migration", + } + + debian_snap = next( + gap for gap in step_12_gaps if gap["id"] == "debian-snap-v1-upgrade" + ) + assert debian_snap["severity"] == "none" + assert debian_snap["disposition"] == "resolved" + assert "source-free, read-only preflight" in debian_snap["candidate_behavior"] + assert "fail closed" in debian_snap["resolution"] + assert "manual migration" in debian_snap["resolution"] + assert "no safe default-provenance marker" in debian_snap["resolution"] + assert "platform-blocked" in debian_snap["validation"] + + with LIVE_RESULTS_PATH.open("rb") as results_file: + package_results = { + result["id"]: result + for result in tomllib.load(results_file)["result"] + if result["step"] == 12 + } + for result_id in ( + "debian-package-upgrade", + "homebrew-package-upgrade", + "rpm-package-upgrade", + "snap-package-refresh", + ): + assert package_results[result_id]["status"] == "platform_blocked" + + +def test_legacy_environment_resolution_preserves_singular_semantics() -> None: + legacy_gap = next( + gap + for gap in load_ledger()["gaps"] + if gap["id"] == "legacy-environment-selector-upgrade" + ) + + assert "one non-empty OPENSHELL_DRIVERS value" in legacy_gap["resolution"] + assert "rejects multiple values" in legacy_gap["resolution"] + assert "conflicting canonical and legacy values" in legacy_gap["resolution"] diff --git a/python/openshell/release_formula_test.py b/python/openshell/release_formula_test.py index 62c27115b5..9a4948116f 100644 --- a/python/openshell/release_formula_test.py +++ b/python/openshell/release_formula_test.py @@ -53,7 +53,7 @@ def test_generate_homebrew_formula_uses_tagged_macos_driver_asset_without_defaul "v0.0.10/openshell-driver-vm-aarch64-apple-darwin.tar.gz" ) in formula assert 'sha256 "' + "b" * 64 + '"' in formula - assert "OPENSHELL_DRIVERS: " not in formula + assert "OPENSHELL_COMPUTE_DRIVER: " not in formula assert 'OPENSHELL_GATEWAY_CONFIG: "#{var}/openshell/gateway.toml"' not in formula assert "init-gateway-config.sh" not in formula assert 'gateway_config = var/"openshell/gateway.toml"' in formula @@ -64,12 +64,31 @@ def test_generate_homebrew_formula_uses_tagged_macos_driver_asset_without_defaul flags=re.DOTALL, ) assert generated_config is not None + assert "version = 2" in generated_config.group("contents") assert "[openshell.gateway]" in generated_config.group("contents") assert "bind_address =" not in generated_config.group("contents") - assert 'bind_address = "[::1]:17670"' in formula + + legacy_empty_config = re.search( + r"legacy_empty_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_empty_config is not None + assert "version = 1" in legacy_empty_config.group("contents") + assert "bind_address =" not in legacy_empty_config.group("contents") + + legacy_ipv6_config = re.search( + r"legacy_ipv6_gateway_config_contents = <<~TOML\n(?P.*?)\n TOML", + formula, + flags=re.DOTALL, + ) + assert legacy_ipv6_config is not None + assert "version = 1" in legacy_ipv6_config.group("contents") + assert 'bind_address = "[::1]:17670"' in legacy_ipv6_config.group("contents") + assert "gateway_config.read == legacy_empty_gateway_config_contents ||" in formula assert "gateway_config.read == legacy_ipv6_gateway_config_contents" in formula assert "gateway_config.write gateway_config_contents" in formula - assert '# compute_drivers = ["vm"]' not in formula + assert '# compute_driver = "vm"' not in formula assert ( "openshell gateway add https://localhost:17670 --local --name openshell" in formula @@ -142,12 +161,15 @@ def test_snap_docker_connect_hook_restarts_gateway() -> None: ) -def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: +def test_rpm_spec_seeds_and_migrates_gateway_defaults() -> None: repo_root = Path(__file__).resolve().parents[2] spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") assert "init-gateway-config.sh" not in spec assert "init-pki.sh" not in spec + assert "migrate-gateway-config.sh" in spec + assert "gateway.toml.default.v1" in spec + assert "%{name}-gateway-migrate-config" in spec assert "Environment=OPENSHELL_LOCAL_TLS_DIR=%%h/.local/state/openshell/tls" in spec assert ( "openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR}" @@ -155,7 +177,7 @@ def test_rpm_spec_uses_gateway_defaults_without_config_helper() -> None: ) assert "EnvironmentFile=-%%E/openshell/gateway.env" in spec assert "%%S/openshell/tls" not in spec - assert "Environment=OPENSHELL_DRIVERS" not in spec + assert "Environment=OPENSHELL_COMPUTE_DRIVER" not in spec assert "Environment=OPENSHELL_BIND_ADDRESS" not in spec assert "Environment=OPENSHELL_PODMAN_TLS_CA" not in spec assert "ExecStart=/usr/bin/openshell-gateway" in spec @@ -180,3 +202,50 @@ def test_deb_user_service_uses_gateway_defaults_without_config_helper() -> None: assert "ExecStart=/usr/bin/openshell-gateway" in unit assert "--config" not in unit assert "--db-url" not in unit + + +def test_rpm_migration_exec_start_pre_argument_order() -> None: + repo_root = Path(__file__).resolve().parents[2] + spec = (repo_root / "openshell.spec").read_text(encoding="utf-8") + + assert ( + "ExecStartPre=%{_libexecdir}/%{name}-gateway-migrate-config " + "%%E/openshell/gateway.toml " + "/usr/share/openshell-gateway/gateway.toml.default " + "/usr/share/openshell-gateway/gateway.toml.default.v1" + ) in spec + + +def test_schema_v2_debian_and_snap_preflight_wiring() -> None: + repo_root = Path(__file__).resolve().parents[2] + unit = (repo_root / "deploy/deb/openshell-gateway.service").read_text( + encoding="utf-8" + ) + wrapper = (repo_root / "tasks/scripts/snap-gateway-wrapper.sh").read_text( + encoding="utf-8" + ) + package_deb = (repo_root / "tasks/scripts/package-deb.sh").read_text( + encoding="utf-8" + ) + preflight = "ExecStartPre=/usr/bin/openshell-gateway config preflight" + certs = "ExecStartPre=/usr/bin/openshell-gateway generate-certs" + assert preflight in unit + assert unit.index(preflight) < unit.index(certs) + assert "EnvironmentFile=-%E/openshell/gateway.env" in unit + assert "ExecStart=/usr/bin/openshell-gateway" in unit + assert "$src_dir/openshell-gateway.service" in package_deb + assert "$pkgroot/usr/lib/systemd/user/openshell-gateway.service" in package_deb + assert 'if [ -n "${OPENSHELL_GATEWAY_CONFIG:-}" ]; then' in wrapper + assert ( + 'elif [ -e "$CANONICAL_CONFIG_FILE" ] || [ -L "$CANONICAL_CONFIG_FILE" ]; then' + in wrapper + ) + assert wrapper.count('"${SNAP}/bin/openshell-gateway" config preflight') == 4 + assert 'config preflight -- "$@"' in wrapper + assert 'config preflight -- --config "$CANONICAL_CONFIG_FILE" "$@"' in wrapper + assert ( + 'exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@"' + in wrapper + ) + assert wrapper.count('exec "${SNAP}/bin/openshell-gateway" "$@"') == 3 + assert '[ -f "$CANONICAL_CONFIG_FILE" ]' not in wrapper diff --git a/python/openshell/rpm_gateway_config_migration_test.py b/python/openshell/rpm_gateway_config_migration_test.py new file mode 100644 index 0000000000..d622f36bb1 --- /dev/null +++ b/python/openshell/rpm_gateway_config_migration_test.py @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +MIGRATOR = REPO_ROOT / "deploy/rpm/migrate-gateway-config.sh" +CURRENT = REPO_ROOT / "deploy/rpm/gateway.toml.default" +LEGACY = REPO_ROOT / "deploy/rpm/gateway.toml.default.v1" +SHIPPED_V1_FIXTURE = b"""# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Default gateway configuration for RPM installs. +# +# This file is seeded to ~/.config/openshell/gateway.toml on first start +# of the openshell-gateway.service systemd user unit. Edit that copy to +# customize. This file is not read directly at runtime. +# +# Configuration precedence (highest to lowest): +# CLI flag > OPENSHELL_* env var > TOML file > built-in default +# +# To override settings without editing this file, set OPENSHELL_* variables +# in ~/.config/openshell/gateway.env or run: +# systemctl --user edit openshell-gateway + +[openshell] +version = 1 + +[openshell.gateway] +# Keep the primary listener on the built-in 127.0.0.1:17670 default. The +# Podman driver reports the callback interface it needs, and the gateway +# adds a separate listener scoped to that interface. + +# Pin to the Podman compute driver. Without this, the gateway auto-detects +# in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver +# selection if Docker is also installed on the host. +compute_drivers = ["podman"] +""" + + +def run_migrator( + destination: Path, *, check: bool = True +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(CURRENT), str(LEGACY)], + check=check, + text=True, + capture_output=True, + ) + + +def test_migrator_seeds_missing_config_and_is_idempotent(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_replaces_only_exact_legacy_default(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + shipped_v1 = SHIPPED_V1_FIXTURE + assert b'compute_drivers = ["podman"]' in shipped_v1 + assert LEGACY.read_bytes() == shipped_v1 + destination.write_bytes(shipped_v1) + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_edited_legacy_and_current_configs(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\n" + destination.write_text(edited, encoding="utf-8") + + run_migrator(destination) + assert destination.read_text(encoding="utf-8") == edited + + destination.write_bytes(CURRENT.read_bytes()) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + + +def test_migrator_refuses_symlink_destination(tmp_path: Path) -> None: + target = tmp_path / "target.toml" + target.write_text("operator-owned\n", encoding="utf-8") + destination = tmp_path / "gateway.toml" + destination.symlink_to(target) + + result = run_migrator(destination, check=False) + + assert result.returncode != 0 + assert "non-regular gateway config" in result.stderr + assert target.read_text(encoding="utf-8") == "operator-owned\n" + + +def test_shipped_rpm_defaults_have_expected_schema_shapes() -> None: + import tomllib + + current = tomllib.loads(CURRENT.read_text(encoding="utf-8")) + legacy = tomllib.loads(LEGACY.read_text(encoding="utf-8")) + + assert current["openshell"]["version"] == 2 + assert current["openshell"]["gateway"]["compute_driver"] == "podman" + assert "compute_drivers" not in current["openshell"]["gateway"] + assert current["openshell"]["drivers"]["podman"]["health_check_interval_secs"] == 10 + assert LEGACY.read_bytes() == SHIPPED_V1_FIXTURE + assert legacy["openshell"]["version"] == 1 + assert legacy["openshell"]["gateway"]["compute_drivers"] == ["podman"] + assert "compute_driver" not in legacy["openshell"]["gateway"] + + +def test_migrator_rejects_wrong_argument_count() -> None: + result = subprocess.run( + ["sh", str(MIGRATOR)], text=True, capture_output=True, check=False + ) + + assert result.returncode == 2 + assert result.stderr == ( + f"usage: {MIGRATOR} DESTINATION CURRENT_DEFAULT LEGACY_DEFAULT\n" + ) + + +def test_migrator_seeds_with_mode_0644(tmp_path: Path) -> None: + destination = tmp_path / "config/openshell/gateway.toml" + + run_migrator(destination) + + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_preserves_operator_content_and_mode(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + edited = LEGACY.read_text(encoding="utf-8") + "# operator edit\\n" + destination.write_text(edited, encoding="utf-8") + destination.chmod(0o600) + + run_migrator(destination) + + assert destination.read_text(encoding="utf-8") == edited + assert destination.stat().st_mode & 0o777 == 0o600 + + destination.write_bytes(CURRENT.read_bytes()) + destination.chmod(0o640) + run_migrator(destination) + assert destination.read_bytes() == CURRENT.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o640 + + +def test_migrator_handles_paths_containing_spaces(tmp_path: Path) -> None: + package = tmp_path / "package defaults" + package.mkdir() + current = package / "current default.toml" + legacy = package / "legacy default.toml" + current.write_bytes(CURRENT.read_bytes()) + legacy.write_bytes(LEGACY.read_bytes()) + destination = tmp_path / "operator config/gateway config.toml" + + subprocess.run( + ["sh", str(MIGRATOR), str(destination), str(current), str(legacy)], + check=True, + ) + + assert destination.read_bytes() == current.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o644 + + +def test_migrator_rejects_missing_and_nonregular_sources(tmp_path: Path) -> None: + for source_name in ("current", "legacy"): + for source_kind in ("missing", "directory", "symlink"): + current = tmp_path / f"{source_name}-{source_kind}-current.toml" + legacy = tmp_path / f"{source_name}-{source_kind}-legacy.toml" + current.write_bytes(CURRENT.read_bytes()) + legacy.write_bytes(LEGACY.read_bytes()) + source = current if source_name == "current" else legacy + if source_kind == "missing": + source.unlink() + elif source_kind == "directory": + source.unlink() + source.mkdir() + else: + source.unlink() + source.symlink_to(CURRENT) + + result = subprocess.run( + [ + "sh", + str(MIGRATOR), + str(tmp_path / f"{source_name}-{source_kind}-destination.toml"), + str(current), + str(legacy), + ], + check=False, + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + assert ( + f"gateway config migration source is not a regular file: {source}" + in result.stderr + ) + + +def test_migrator_refuses_directory_destination(tmp_path: Path) -> None: + destination = tmp_path / "gateway.toml" + destination.mkdir() + + result = run_migrator(destination, check=False) + + assert result.returncode == 1 + assert ( + f"refusing to replace non-regular gateway config: {destination}" + in result.stderr + ) diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index d6d4750f25..8007236d7a 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -8,16 +8,16 @@ state: implemented ## Summary -Introduce a TOML-based configuration file for the OpenShell gateway that unifies all gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file, while preserving full backwards compatibility with the existing CLI flags and `OPENSHELL_*` environment variables. +Introduce a TOML-based configuration file for the OpenShell gateway that unifies gateway settings — core server options, TLS, OIDC, observability listeners, and per-driver parameters — under a single structured file. CLI flags and supported `OPENSHELL_*` environment variables retain higher precedence. Schema version 2 intentionally rejects legacy file fields and locations. ## Motivation -The gateway today is configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This works for simple single-node deployments but breaks down as deployments grow: +Before this RFC, the gateway was configured exclusively through CLI flags and `OPENSHELL_*` environment variables. This worked for simple single-node deployments but broke down as deployments grew: -- **Too many flags** — the gateway has ~40 configurable parameters today (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests are hard to read, diff, and audit. -- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers all live in the same flat CLI namespace, with no structural separation. Most flags only apply to one driver, but there is no way to express that in CLI form. -- **Helm friction** — The chart's `statefulset.yaml` already carries a long `env:` block of `OPENSHELL_*` variables that each map to a `values.yaml` key. A config file can be mounted as a single `ConfigMap` and reduces the chart's templating surface significantly. -- **Secrets management** — Injecting secrets (TLS material paths, database URL, OIDC settings) via environment variables is functional but not idiomatic for Kubernetes. A file-based format opens the door to projected secrets and volume mounts that compose cleanly with the non-secret config. +- **Too many flags** — the gateway exposed roughly 40 configurable parameters (TLS, OIDC, four compute drivers, three listeners). Long `docker run` commands and `args:` arrays in Kubernetes manifests were hard to read, diff, and audit. +- **Driver coupling** — Docker, Podman, Kubernetes, and VM drivers shared one flat CLI namespace with no structural separation. Most flags applied to only one driver, but CLI syntax did not express that ownership. +- **Helm friction** — The chart's `statefulset.yaml` carried a long `env:` block of `OPENSHELL_*` variables that each mapped to a `values.yaml` key. A mounted configuration file reduces the chart's templating surface. +- **Secrets management** — Environment-only configuration did not compose naturally with Kubernetes `ConfigMap` and projected `Secret` volumes. ## Non-goals @@ -48,7 +48,7 @@ The file path is provided via: OPENSHELL_GATEWAY_CONFIG=/path/to/gateway.toml ``` -The file must have a `.toml` extension. A missing path is a hard error; an empty existing file is treated as "no configuration" — the gateway falls back to defaults and to whatever the CLI/env supply. +The file must have a `.toml` extension. A missing path is a hard error. A configured file must declare the exact supported schema version; an empty existing file is rejected. ### TOML schema @@ -58,7 +58,7 @@ The file is rooted at an `[openshell]` table. This namespacing reserves room for ```toml [openshell] -version = 1 # optional; reserved for future schema migrations +version = 2 # required schema version # ────────────────────────────────────────────────────────────────────────────── # Gateway-wide settings @@ -72,10 +72,9 @@ metrics_bind_address = "0.0.0.0:9090" # optional; omit to disable # Logging log_level = "info" -# Compute drivers — list of driver names whose [openshell.drivers.] -# tables should be activated. When empty, the gateway auto-detects a driver -# (kubernetes → podman → docker). VM is never auto-detected. -compute_drivers = ["kubernetes"] +# Compute driver — exactly one driver may be active. When omitted, the gateway +# auto-detects a driver (kubernetes → podman → docker). VM is never auto-detected. +compute_driver = "kubernetes" # Note: database_url is a secret and must be supplied via OPENSHELL_DB_URL # (or --db-url) — it is NOT permitted in the file. @@ -88,12 +87,17 @@ server_sans = ["openshell", "*.dev.openshell.localhost"] enable_loopback_service_http = true # ────────────────────────────────────────────────────────────────────────────── -# TLS / mTLS — when omitted, the gateway listens plaintext (sets --disable-tls) +# TLS / mTLS — package-managed local TLS may supply listener defaults. # ────────────────────────────────────────────────────────────────────────────── -# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. When true, the gateway -# ignores the [openshell.gateway.tls] table below. +# Mirrors --disable-tls / OPENSHELL_DISABLE_TLS. Set true explicitly for a +# plaintext listener; guest TLS fields must then be omitted. disable_tls = false +# Gateway-owned TLS bundle injected into the selected local driver. +guest_tls_ca = "/etc/openshell/certs/ca.pem" +guest_tls_cert = "/etc/openshell/certs/client.pem" +guest_tls_key = "/etc/openshell/certs/client-key.pem" + [openshell.gateway.tls] cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" @@ -113,15 +117,15 @@ scopes_claim = "" # empty disables scope enforcement # ────────────────────────────────────────────────────────────────────────────── # Compute drivers — each table is owned and parsed by its driver crate. -# Only tables for drivers listed in compute_drivers are activated. +# Only the selected or auto-detected driver's table is activated. # ────────────────────────────────────────────────────────────────────────────── [openshell.drivers.kubernetes] namespace = "openshell" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" +image_pull_policy = "if_not_present" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" -supervisor_image_pull_policy = "IfNotPresent" +supervisor_image_pull_policy = "if_not_present" grpc_endpoint = "https://host.openshell.internal:8080" client_tls_secret_name = "openshell-sandbox-tls" host_gateway_ip = "10.0.0.1" @@ -129,26 +133,20 @@ ssh_socket_path = "/run/openshell/ssh.sock" [openshell.drivers.docker] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "IfNotPresent" -sandbox_namespace = "docker-dev" +image_pull_policy = "if_not_present" +sandbox_label = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.podman] socket_path = "/run/podman/podman.sock" default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" -image_pull_policy = "missing" # Podman vocabulary: always | missing | never | newer +image_pull_policy = "if_not_present" # always | if_not_present | never | newer supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" network_name = "openshell" stop_timeout_secs = 10 -guest_tls_ca = "/etc/openshell/certs/ca.pem" -guest_tls_cert = "/etc/openshell/certs/client.pem" -guest_tls_key = "/etc/openshell/certs/client-key.pem" [openshell.drivers.vm] state_dir = "/var/lib/openshell/vm" @@ -157,9 +155,6 @@ grpc_endpoint = "https://host.containers.internal:8080" vcpus = 2 mem_mib = 2048 krun_log_level = 1 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` ### Driver configuration @@ -167,12 +162,12 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" Each `[openshell.drivers.]` table is extracted from the parsed file and handed to the driver's initialization function as a raw TOML value. The driver is then responsible for: 1. **Parsing** — deserializing the table into its own typed config struct (e.g. `KubernetesComputeConfig`, `DockerComputeConfig`, `PodmanComputeConfig`, `VmComputeConfig`). -2. **Validation** — applying cross-field checks specific to that driver (e.g. requiring TLS triplets when sandbox-side mTLS is enabled). +2. **Validation** — applying cross-field checks specific to that driver. Gateway-owned guest TLS paths are validated as one bundle and injected only into the selected local driver before this step. 3. **Consumption** — using the resulting struct to initialize internal state. Driver authors define and own their config schema. Adding a new driver does not require changes to the gateway's core `Config` struct or to this RFC. -`[openshell.drivers.]` tables for drivers not listed in `compute_drivers` (and not the auto-detected driver) are parsed for syntax but not activated. +`[openshell.drivers.]` tables for drivers other than the selected or auto-detected driver are parsed for syntax but not activated. ### Merge semantics @@ -207,25 +202,26 @@ Deserialization uses `#[serde(deny_unknown_fields)]` at every table level. An un The following cross-field validations are applied after merging file + env + CLI: - `bind_address`, `health_bind_address`, and `metrics_bind_address` must all use distinct ports when set. -- When `[openshell.gateway.tls]` is present, all three of `cert_path`, `key_path`, and `client_ca_path` must be present (either from the file or from CLI/env). Partial TLS configuration is an error. +- Gateway listener TLS requires `cert_path` and `key_path`; `client_ca_path` is required only for listener client-certificate verification. TLS-enabled Docker, Podman, and VM drivers also require a complete gateway-owned guest CA, certificate, and key bundle. Kubernetes projects guest TLS through a Secret instead. - `database_url` must be non-empty after merging env + CLI — every supported driver requires it. The field is not accepted from the file (see Secrets above). -- `compute_drivers` may be empty; in that case the gateway falls back to auto-detection. If the list contains a driver name with no matching `[openshell.drivers.]` table, the driver runs with its built-in defaults. +- `compute_driver` selects exactly one driver. When omitted, the gateway falls back to auto-detection. A custom driver requires a named table with `socket_path`, unless startup supplies an explicit socket override. The legacy `compute_drivers` list is rejected. -### Backwards compatibility +### Schema compatibility -The existing CLI interface is fully preserved. All flags continue to work exactly as before. The `--config` flag is new and additive. `OPENSHELL_DB_URL` remains a required process input (it is not accepted from the file). +Schema version 2 requires `version = 2`, a singular `compute_driver` when a driver is selected, and driver-owned fields under `[openshell.drivers.]`. Legacy schema versions and `compute_drivers` lists are rejected. `OPENSHELL_DB_URL` remains a required process input and is not accepted from the file. ### Example: minimal Kubernetes deployment ```toml [openshell] -version = 1 +version = 2 [openshell.gateway] -bind_address = "0.0.0.0:8080" -compute_drivers = ["kubernetes"] +bind_address = "0.0.0.0:8080" +compute_driver = "kubernetes" # database_url comes from env (e.g. valueFrom.secretKeyRef). -# No [openshell.gateway.tls] → plaintext listener (gateway runs behind Envoy / ingress). +# The gateway runs plaintext behind Envoy / ingress. +disable_tls = true [openshell.drivers.kubernetes] namespace = "agents" @@ -236,12 +232,7 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ### Helm integration -The Helm chart today renders a long `env:` block in `templates/statefulset.yaml`, with each `OPENSHELL_*` variable mapped to a `values.yaml` key. This RFC's adoption replaces that block with: - -1. A new `gateway.config` value tree (TOML-shaped YAML) in `values.yaml`. -2. A new `ConfigMap` template that renders the values into a TOML document via Helm's `tpl`. -3. A volume mount of the `ConfigMap` at `/etc/openshell/gateway.toml` and a `--config` flag in the gateway container's `args`. -4. Continued use of a `Secret`-backed `env:` entry for `OPENSHELL_DB_URL` (which never lives in the `ConfigMap`), plus optional projections for TLS material paths. The CLI/env precedence above means any `Secret`-backed env var also wins over a value in the `ConfigMap`. +The Helm chart renders schema-v2 gateway TOML into a `ConfigMap`, mounts it at `/etc/openshell/gateway.toml`, and starts the gateway with that file. Secret process inputs such as `OPENSHELL_DB_URL` remain `Secret`-backed environment entries and retain higher precedence. Kubernetes projects sandbox guest TLS through its configured Secret rather than placing host guest-certificate paths in the gateway TOML. ```yaml # values.yaml excerpt @@ -250,7 +241,7 @@ gateway: bind_address: "0.0.0.0:8080" health_bind_address: "0.0.0.0:8081" metrics_bind_address: "0.0.0.0:9090" - compute_drivers: ["kubernetes"] + compute_driver: "kubernetes" drivers: kubernetes: namespace: agents @@ -260,23 +251,15 @@ gateway: The chart owners can migrate one section at a time: `OPENSHELL_*` env vars and the `ConfigMap` coexist during the transition, with env continuing to override the file. -## Implementation plan - -No part of this RFC has shipped yet. The work breaks down as: +## Implementation -1. **Add a config-file loader to `openshell-server`** — define a `GatewayConfigFile` struct that mirrors the schema above, parse it with `serde` + `toml`, and merge it into `openshell_core::Config` plus the per-driver structs in `compute/`. -2. **Wire the merge into `cli.rs`** — add `--config` / `OPENSHELL_GATEWAY_CONFIG`, gate each existing flag's "apply from file" path on clap `ValueSource::DefaultValue`, and run cross-field validation after the merge. -3. **Per-driver deserialization** — give each driver crate (`openshell-driver-{kubernetes,docker,podman,vm}`) a `from_toml` (or `serde::Deserialize`) entry point so the gateway can hand each driver its own table. -4. **Test coverage** — file parsing, env-overrides-file, CLI-overrides-env, partial TLS error, port-collision error, unknown-field rejection, missing driver table fallback. -5. **Helm chart migration** — add `gateway.config` value tree, render the `ConfigMap`, mount it, switch the gateway container to `--config`. Keep the `OPENSHELL_*` env names available as opt-in overrides for secrets. -6. **Example file** — ship the per-driver examples on the published docs reference at `docs/reference/gateway-config.mdx`. -7. **Architecture doc update** — reflect the new config sources and precedence in `architecture/gateway.md`. +The implemented gateway loader parses TOML with `serde`, merges file values below environment and CLI sources, and rejects unknown fields. Each compute driver deserializes only its named table. Helm renders schema-v2 TOML into a ConfigMap, while secret process inputs remain environment-backed. Package templates, examples, tests, and the gateway architecture documentation use the same canonical schema. ## Risks -- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Mitigate by treating field renames as breaking, keeping the `version` field reserved for schema migrations, and surfacing rename errors clearly. +- **Serde `deny_unknown_fields` is strict** — any field name change in `openshell_core::Config` or in a driver's config struct becomes a breaking change for anyone using the file. Treat field renames as versioned schema changes and surface migration errors clearly. - **Secrets in the file** — `database_url` is excluded from the schema entirely (env / CLI only). OIDC settings remain allowed in the file because none of them are credentials in isolation. Operators should still prefer env-var injection for any field that would live in a `Secret` rather than a `ConfigMap` (TLS material paths, restricted-environment OIDC issuers, etc.). Documentation must call this out prominently. -- **Partial TLS configuration** — the hard error on partial TLS config is the right UX, but the error message must clearly identify which source (file vs. CLI/env) is missing which field, since the file's `[openshell.gateway.tls]` table is all-or-nothing while the CLI flags are independent. +- **Partial TLS configuration** — listener and guest TLS are separate complete-bundle contracts. Startup rejects partial bundles and identifies the missing configuration before constructing a driver. - **Driver schema drift** — once each driver owns its own TOML table, driver releases can change field names independently of the gateway. The gateway's `version` field does not protect against driver-side breakage; document driver-config stability separately. ## Alternatives @@ -295,8 +278,7 @@ No part of this RFC has shipped yet. The work breaks down as: ## Open questions -1. **Schema versioning** — the `version` field is reserved but not acted on. Should the parser reject files with `version > 1`, or just warn? Define this before the first stable release. -2. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. +1. **Directory-based config (`conf.d` pattern)** — a `--config-dir` flag that globs all `*.toml` files in a directory, sorts them alphabetically, and deep-merges them in order (later files win per key). CLI/env overrides still sit above everything. This maps cleanly to Kubernetes: a base `ConfigMap` as `10-base.toml`, driver config as `20-kubernetes.toml`, and credentials from a projected `Secret` as `90-credentials.toml` — all mounted into the same directory without a monolithic file. This is the approach taken by cri-o and kubelet, inspired by systemd's `conf.d` convention. - Deferred to a follow-on: the single `--config` file is sufficient for v1, and the directory loader can be added without any schema changes. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (e.g. `compute_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. -3. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). OIDC settings are allowed for v1 since the listed fields are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. + Deferred to a follow-on: the single `--config` file is sufficient for the current schema, and the directory loader can be added without changing the file schema. Before implementing, three design decisions must be settled: (a) whether `--config` and `--config-dir` are mutually exclusive or composable (and if so which takes lower precedence); (b) whether a later file's array value (for example `credential_drivers`) replaces or appends — replace is simpler and less surprising; (c) `deny_unknown_fields` validation must apply to the final merged result rather than each individual file, since partial drop-in files won't contain all sections. +2. **OIDC secret hygiene (revisit)** — `database_url` is excluded from the file schema (resolved). Schema version 2 allows the listed OIDC fields because they are identifiers, not credentials. If we add OIDC fields that *are* credentials in the future (e.g. a client secret for confidential-client flows), they should join the env-only list at that point. Re-evaluate once the OIDC surface stabilises. diff --git a/rfc/0011-multi-player-design/README.md b/rfc/0011-multi-player-design/README.md index 32a2f584e1..5fcb67fe51 100644 --- a/rfc/0011-multi-player-design/README.md +++ b/rfc/0011-multi-player-design/README.md @@ -755,13 +755,14 @@ use the workspace to select the target Kubernetes namespace instead of encoding it in the resource name. The label-based lookup and annotation patterns established here carry over unchanged. -**Docker and Podman drivers.** The Docker driver's `sandbox_namespace` label -provides a foundation for workspace mapping, but the driver currently uses a -single configured namespace rather than per-sandbox values. The driver contract -must be updated so that workspace flows through `DriverSandbox` and the driver -applies it as the container label filter. The same applies to Podman and other -local drivers — workspace isolation is enforced at the gateway level and does -not require Kubernetes. +**Docker and Podman drivers.** The Docker driver's `sandbox_label` +configuration value is stored in the `openshell.sandbox_namespace` container +label and provides a foundation for workspace mapping, but the driver currently +uses a single configured value rather than per-sandbox values. The driver +contract must be updated so that workspace flows through `DriverSandbox` and +the driver applies it as the container label filter. The same applies to Podman +and other local drivers — workspace isolation is enforced at the gateway level +and does not require Kubernetes. ### Compute Driver Trust Model diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 7d47b18065..c1b7d28082 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -85,12 +85,28 @@ Before debugging the compute platform, inspect gateway logs for failures in depe For out-of-tree compute drivers, confirm the selected driver name and socket agree across CLI flags or `gateway.toml`, and that the operator-owned driver is running before the gateway starts: ```bash -rg -n 'compute_drivers|socket_path' /etc/openshell/gateway.toml +rg -n '^version|compute_driver|socket_path|guest_tls_' /etc/openshell/gateway.toml stat /run/openshell/.sock journalctl -u --no-pager --lines=200 journalctl -u openshell-gateway --no-pager --lines=200 ``` +Gateway configuration requires `[openshell] version = 2`, a singular +`compute_driver` selector, and driver-owned settings under +`[openshell.drivers.]`. The gateway rejects legacy `compute_drivers` and +`--drivers` selectors rather than silently migrating them. One valid, nonempty +`OPENSHELL_DRIVERS` value remains a deprecated environment-only alias when the +canonical selector is absent; the gateway selects that driver with a warning. +Empty, invalid, comma-delimited, or conflicting values fail startup. Homebrew +and RPM package startup migrates only exact package-generated v1 defaults. If +an upgraded package still reports an unsupported version, inspect the active prefix or `~/.config/openshell/gateway.toml`; an edited v1 file must +follow the published schema-v2 migration steps and must not be overwritten. +Guest TLS CA, certificate, and key paths are the exception to driver ownership: +configure the complete bundle under `[openshell.gateway]`, and the gateway +injects it only into the selected local driver. TLS-enabled Docker, Podman, and +VM drivers fail startup when neither those paths nor the package-managed local +bundle is available; Kubernetes projects its bundle through a Secret. + Custom names use `[openshell.drivers.].socket_path`. A launch-time `--compute-driver-socket` override may also use `docker`, `podman`, `kubernetes`, or `vm`; the endpoint then takes precedence over built-in construction. First-party standalone drivers require the socket parent directory to be owned by the driver's effective UID, force its mode to `0700`, create the socket with mode `0600`, and accept only peers with that same UID. Check the parent and socket separately with `stat`; a gateway running under a different UID cannot connect even when filesystem permissions or group membership would otherwise allow it. Operator-supplied drivers must provide equivalent access control appropriate to their implementation. Check gateway logs for connection errors, `GetCapabilities` failures, or an unexpected advertised driver name. The advertised name is diagnostic metadata; negotiated features control optional behavior. The gateway does not create or supervise operator-supplied driver processes or sockets. For configured gateway interceptors, inspect `[[openshell.gateway.interceptors]]`, their Unix or network endpoints, and gateway startup logs: @@ -282,6 +298,20 @@ release. Look for failed installs, unexpected values, missing namespace, wrong image tag, TLS settings that do not match the registered endpoint, and scheduling failures. +The chart mounts the `gateway.toml` ConfigMap key directly at +`/etc/openshell/gateway.toml` as a read-only `subPath` file. This avoids the +atomic-writer symlink exposed by a ConfigMap directory mount because the gateway +rejects symlinked configuration. A checksum pod-template annotation rolls the +workload when the ConfigMap changes. If config preflight reports a symlink or +nonregular path, inspect the rendered mount and confirm the workload rolled to +the current chart revision: + +```bash +kubectl -n openshell get deployment,statefulset -o yaml | rg -n 'gateway-config|mountPath|subPath|checksum/gateway-config' +kubectl -n openshell rollout status /openshell +kubectl -n openshell logs -c openshell-gateway --tail=200 +``` + `server.telemetryEnabled` renders `OPENSHELL_TELEMETRY_ENABLED` on the gateway pod, and the gateway propagates the effective value to sandbox supervisors. @@ -611,6 +641,11 @@ Use the VM driver logs and host diagnostics available in the user's environment. - The VM driver process is running and reachable by the gateway. - The runtime rootfs exists and matches the expected architecture. +- `mke2fs` or `mkfs.ext4` and `debugfs` from e2fsprogs are installed; explicit + `sandbox_uid`/`sandbox_gid` does not remove this prerequisite. +- A persisted overlay identity error is resolved from its owner marker, overlay + upper layer, prepared rootfs, explicit config, or current image. Do not assign + `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. - The sandbox supervisor can establish its callback connection to the gateway. @@ -725,3 +760,24 @@ When handing results back to the user, include: - Service exposure status. - Sandbox workload status. - The exact command that failed and the shortest fix. + +## Package Configuration Preflight + +For a Debian, Ubuntu, or Snap gateway that stops before certificate generation or +daemon startup, validate the selected configuration without starting the service: + +```shell +openshell-gateway config preflight [--path PATH | -- GATEWAY_ARGS...] +``` + +Without a path, preflight validates a nonempty `OPENSHELL_GATEWAY_CONFIG` or an +auto-discovered XDG config; no config succeeds. An explicit missing path, legacy +schema-v1 file, malformed TOML, symlink, or nonregular file fails before gateway +startup. It also applies read-only effective-config checks for driver selection +and configuration, sockets, rate limits, TLS, interceptors, and supervisor +middleware. If the file omits the selector, preflight validates configured tables +for auto-detectable drivers without running socket or process-based detection +probes. Arguments after `--` validate the effective daemon invocation, +including its command-line overrides. Preflight preserves every failed file. Do +not advise users to delete or rewrite it automatically; back it up and follow the +manual schema-v2 migration in the Gateway Configuration reference. diff --git a/snapcraft.yaml b/snapcraft.yaml index f567c63c47..35ec3b1d0c 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -87,9 +87,9 @@ apps: # Operators must manually restart the service after a refresh if needed. refresh-mode: endure # The wrapper sets OPENSHELL_DISABLE_TLS=true and OPENSHELL_DB_URL to - # use $SNAP_COMMON/gateway.db. If $SNAP_COMMON/gateway.toml exists it is - # passed to the gateway as --config, allowing operators to override - # settings without rebuilding the snap. + # use $SNAP_COMMON/gateway.db. Before startup it validates the selected + # operator-provided config without creating or rewriting it. A nonempty + # OPENSHELL_GATEWAY_CONFIG takes precedence over gateway.toml. environment: XDG_DATA_HOME: "$SNAP_COMMON" XDG_RUNTIME_DIR: "$SNAP_COMMON" diff --git a/tasks/parity.toml b/tasks/parity.toml new file mode 100644 index 0000000000..a44012ac33 --- /dev/null +++ b/tasks/parity.toml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +["test:e2e-parity"] +description = "Run deterministic schema-v1/schema-v2 parity harness tests" +run = "bash e2e/parity/test.sh" +hide = true + +["e2e:parity:podman"] +description = "Compare frozen schema-v1 and current schema-v2 conformance against Podman (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman" + +["e2e:parity:podman-options"] +description = "Compare paired Podman sandbox option semantics across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman --scenario podman-options" + +["e2e:parity:podman-external-driver"] +description = "Compare paired external Podman driver lifecycle semantics across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/run.sh --driver podman --scenario external-driver" + +["e2e:parity:gateway-options"] +description = "Compare process-level gateway option behavior across schema v1 and v2 (opt-in live test)" +run = "bash e2e/parity/gateway-options.sh" + +["test:e2e-parity:kubernetes-options"] +description = "Run deterministic Kubernetes schema parity harness tests" +run = "bash e2e/parity/kubernetes-options-test.sh" +hide = true + +["e2e:parity:kubernetes-options"] +description = "Compare Kubernetes driver options on an explicitly supplied disposable kind cluster" +run = "bash e2e/parity/kubernetes-options.sh" diff --git a/tasks/scripts/codex_security_range_test.py b/tasks/scripts/codex_security_range_test.py index dd77fe7693..abf5026cdc 100644 --- a/tasks/scripts/codex_security_range_test.py +++ b/tasks/scripts/codex_security_range_test.py @@ -18,7 +18,24 @@ def _git(repo: Path, *args: str) -> str: - return subprocess.check_output(["git", *args], cwd=repo).decode("utf-8").strip() + # Temporary fixture repositories must not inherit developer-wide signing + # requirements: these tests intentionally create disposable commits and + # lightweight tags without prompting for a private key. + return ( + subprocess.check_output( + [ + "git", + "-c", + "commit.gpgsign=false", + "-c", + "tag.gpgsign=false", + *args, + ], + cwd=repo, + ) + .decode("utf-8") + .strip() + ) def _commit(repo: Path, name: str) -> None: diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 5a6fb62887..8771728a89 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -24,12 +24,16 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_DOCKER_GATEWAY_NAME:-docker-dev}" STATE_DIR="${OPENSHELL_DOCKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-docker}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-docker-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" @@ -211,11 +215,11 @@ mkdir -p "${STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" < only on a Docker host with AppArmor enabled. +app_armor_profile = "Unconfined" EOF +# Keep the local task's proxy inputs aligned with [openshell.drivers.docker]. +# Credentials stay in the referenced root-owned file; do not echo their value. +if [[ -n "${OPENSHELL_SANDBOX_HTTPS_PROXY+x}" ]]; then + printf 'https_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_HTTPS_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_SANDBOX_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET+x}" ]]; then + printf 'provider_spiffe_workload_api_socket = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET}")" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -255,6 +283,6 @@ exec "${GATEWAY_BIN}" \ --config "${CONFIG_PATH}" \ --port "${PORT}" \ --log-level "${LOG_LEVEL}" \ - --drivers docker \ + --compute-driver docker \ --disable-tls \ --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index ab166865ef..ccf30ca104 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -21,12 +21,16 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" PORT="${OPENSHELL_SERVER_PORT:-18080}" GATEWAY_NAME="${OPENSHELL_PODMAN_GATEWAY_NAME:-podman-dev}" STATE_DIR="${OPENSHELL_PODMAN_GATEWAY_STATE_DIR:-${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-podman}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-podman-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -90,32 +94,6 @@ ensure_podman_supervisor_image() { fi } -podman_pull_policy() { - case "$1" in - Always|always) echo "always" ;; - IfNotPresent|ifnotpresent|missing|"") echo "missing" ;; - Never|never) echo "never" ;; - Newer|newer) echo "newer" ;; - *) - echo "ERROR: unsupported Podman image pull policy '$1'" >&2 - exit 2 - ;; - esac -} - -# Escape a value for embedding in a double-quoted TOML basic string, so -# quotes, backslashes, or control characters in an environment value cannot -# corrupt gateway.toml or inject extra configuration keys. -toml_escape() { - local s=$1 - s=${s//\\/\\\\} - s=${s//\"/\\\"} - s=${s//$'\n'/\\n} - s=${s//$'\r'/\\r} - s=${s//$'\t'/\\t} - printf '%s' "${s}" -} - port_is_in_use() { local port=$1 if command_available lsof; then @@ -215,12 +193,11 @@ CONFIG_PATH="${STATE_DIR}/gateway.toml" install -m 600 /dev/null "${CONFIG_PATH}" cat >"${CONFIG_PATH}" <&2 + return 2 + ;; + esac +} diff --git a/tasks/scripts/gateway-toml.sh b/tasks/scripts/gateway-toml.sh new file mode 100644 index 0000000000..f9b1e1916b --- /dev/null +++ b/tasks/scripts/gateway-toml.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shell helpers shared by local gateway launch scripts. This file is sourced, +# rather than executed, so helpers must not change shell options or process +# environment at import time. + +# Escape a value for a TOML basic string. This supports operator-provided paths +# and URLs without allowing a quote or control character to add TOML fields. +toml_escape() { + local value=$1 + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '%s' "${value}" +} diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 80d723eb11..7c38541e77 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -33,13 +33,17 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" PORT="${OPENSHELL_SERVER_PORT:-18081}" GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-vm-dev}" STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" VM_BOOTSTRAP_IMAGE="${OPENSHELL_VM_BOOTSTRAP_IMAGE:-}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}" +# VM currently has no image-pull-policy setting in its driver configuration; unlike +# Docker, Podman, and Kubernetes launch paths it intentionally does not normalize this input. LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_DIR_DEFAULT="${ROOT}/target/debug" @@ -336,11 +340,11 @@ chmod 700 "${VM_DRIVER_STATE_DIR}" CONFIG_PATH="${STATE_DIR}/gateway.toml" cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_NO_PROXY+x}" ]]; then + printf 'no_proxy = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_NO_PROXY}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE+x}" ]]; then + printf 'proxy_auth_file = "%s"\n' "$(toml_escape "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_FILE}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE+x}" ]]; then + printf 'proxy_auth_allow_insecure = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_AUTH_ALLOW_INSECURE}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME+x}" ]]; then + printf 'proxy_connect_by_hostname = %s\n' "${OPENSHELL_VM_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME}" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT+x}" ]]; then + printf 'provider_spiffe_workload_api_tcp_endpoint = "%s"\n' "$(toml_escape "${OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_TCP_ENDPOINT}")" >>"${CONFIG_PATH}" +fi +if [[ -n "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP+x}" ]]; then + printf 'provider_spiffe_allow_guest_tcp = %s\n' "${OPENSHELL_PROVIDER_SPIFFE_ALLOW_GUEST_TCP}" >>"${CONFIG_PATH}" +fi + append_local_otlp_config_if_available "${CONFIG_PATH}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" @@ -385,7 +414,7 @@ GATEWAY_ARGS=( --config "${CONFIG_PATH}" --port "${PORT}" --log-level "${LOG_LEVEL}" - --drivers vm + --compute-driver vm --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index 906379df79..34bc143fb6 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -10,13 +10,15 @@ # # VM/MicroVM is intentionally explicit-only because it requires runtime setup. # Use either: -# OPENSHELL_DRIVERS=vm mise run gateway +# OPENSHELL_COMPUTE_DRIVER=vm mise run gateway # mise run gateway:vm set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" +GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" usage() { cat <<'EOF' @@ -33,7 +35,7 @@ Options: -h, --help Show this help. Environment: - OPENSHELL_DRIVERS Driver override used by openshell-gateway. + OPENSHELL_COMPUTE_DRIVER Driver override used by openshell-gateway. OPENSHELL_GATEWAY_NAME Gateway name for delegated or Kubernetes runs. OPENSHELL_BIND_ADDRESS Gateway listener address. Defaults to 127.0.0.1, or ::1 for Podman Machine on macOS. @@ -104,7 +106,7 @@ detect_driver() { fi echo "ERROR: no compute driver detected." >&2 - echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_DRIVERS." >&2 + echo " Start Podman or Docker, run inside Kubernetes, or set OPENSHELL_COMPUTE_DRIVER." >&2 exit 2 } @@ -171,17 +173,17 @@ while [[ "$#" -gt 0 ]]; do esac done -if [[ -n "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - echo "ERROR: use either --driver or OPENSHELL_DRIVERS, not both" >&2 +if [[ -n "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + echo "ERROR: use either --driver or OPENSHELL_COMPUTE_DRIVER, not both" >&2 exit 2 fi -if [[ -z "${explicit_driver}" && -n "${OPENSHELL_DRIVERS:-}" ]]; then - if [[ "${OPENSHELL_DRIVERS}" == *,* ]]; then - echo "ERROR: mise run gateway supports one driver; got OPENSHELL_DRIVERS=${OPENSHELL_DRIVERS}" >&2 +if [[ -z "${explicit_driver}" && -n "${OPENSHELL_COMPUTE_DRIVER:-}" ]]; then + if [[ "${OPENSHELL_COMPUTE_DRIVER}" == *,* ]]; then + echo "ERROR: mise run gateway supports one driver; got OPENSHELL_COMPUTE_DRIVER=${OPENSHELL_COMPUTE_DRIVER}" >&2 exit 2 fi - explicit_driver="$(normalize_driver "${OPENSHELL_DRIVERS}")" + explicit_driver="$(normalize_driver "${OPENSHELL_COMPUTE_DRIVER}")" fi DRIVER="${explicit_driver:-$(detect_driver)}" @@ -206,7 +208,7 @@ GATEWAY_NAME="${OPENSHELL_GATEWAY_NAME:-${DRIVER}-dev}" STATE_DIR="${OPENSHELL_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-${DRIVER}}" SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-${DRIVER}-dev}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" -SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +SANDBOX_IMAGE_PULL_POLICY="$(normalize_image_pull_policy "${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-if_not_present}")" GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-}" LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" PRIMARY_BIND_IP="${OPENSHELL_BIND_ADDRESS:-127.0.0.1}" @@ -253,12 +255,11 @@ fi cat >"${CONFIG_PATH}" <>"${CONFIG_PATH}" < sqlite:$SNAP_COMMON/gateway.db (overridable) # - OPENSHELL_DISABLE_TLS -> true -# If $SNAP_COMMON/gateway.toml exists, passes it as --config so operators -# can override settings without rebuilding the snap. +# It validates, but never creates or rewrites, an operator-provided config +# before starting the gateway. set -eu @@ -14,8 +14,65 @@ CANONICAL_CONFIG_FILE="${SNAP_COMMON}/gateway.toml" export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${SNAP_COMMON}/gateway.db?mode=rwc}" export OPENSHELL_DISABLE_TLS="${OPENSHELL_DISABLE_TLS:-true}" -if [ -z "${OPENSHELL_GATEWAY_CONFIG:-}" ] && [ -f "$CANONICAL_CONFIG_FILE" ]; then - exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@" +# Mirror clap's CLI-over-environment precedence so preflight always inspects +# the same file the daemon will load. Reject ambiguous duplicate selectors +# before either command runs. +cli_config="" +config_seen=false +expect_config_path=false +options_done=false +for argument in "$@"; do + if [ "$options_done" = true ]; then + continue + fi + if [ "$expect_config_path" = true ]; then + case "$argument" in + -*) + echo "openshell-gateway: --config requires a nonempty path" >&2 + exit 2 + ;; + esac + if [ "$config_seen" = true ]; then + echo "openshell-gateway: duplicate --config option" >&2 + exit 2 + fi + cli_config=$argument + config_seen=true + expect_config_path=false + continue + fi + case "$argument" in + --) + options_done=true + ;; + --config) + expect_config_path=true + ;; + --config=*) + if [ "$config_seen" = true ]; then + echo "openshell-gateway: duplicate --config option" >&2 + exit 2 + fi + cli_config=${argument#--config=} + config_seen=true + ;; + esac +done +if [ "$expect_config_path" = true ] || { [ "$config_seen" = true ] && [ -z "$cli_config" ]; }; then + echo "openshell-gateway: --config requires a nonempty path" >&2 + exit 2 fi -exec "${SNAP}/bin/openshell-gateway" "$@" +if [ "$config_seen" = true ]; then + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" + exec "${SNAP}/bin/openshell-gateway" "$@" +elif [ -n "${OPENSHELL_GATEWAY_CONFIG:-}" ]; then + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" + exec "${SNAP}/bin/openshell-gateway" "$@" +elif [ -e "$CANONICAL_CONFIG_FILE" ] || [ -L "$CANONICAL_CONFIG_FILE" ]; then + "${SNAP}/bin/openshell-gateway" config preflight -- --config "$CANONICAL_CONFIG_FILE" "$@" + exec "${SNAP}/bin/openshell-gateway" --config "$CANONICAL_CONFIG_FILE" "$@" +else + "${SNAP}/bin/openshell-gateway" config preflight -- "$@" + exec "${SNAP}/bin/openshell-gateway" "$@" +fi diff --git a/tasks/scripts/test-gateway-config.sh b/tasks/scripts/test-gateway-config.sh new file mode 100755 index 0000000000..850cccc0bb --- /dev/null +++ b/tasks/scripts/test-gateway-config.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +UV="$(mise which uv)" +trap 'rm -rf "${WORK}"' EXIT + +# shellcheck source=tasks/scripts/gateway-toml.sh +source "${ROOT}/tasks/scripts/gateway-toml.sh" + +raw=$'quote" slash\\ newline\ncarriage\rtab\t' +printf 'value = "' > "${WORK}/escaped.toml" +toml_escape "${raw}" >> "${WORK}/escaped.toml" +printf '"\n' >> "${WORK}/escaped.toml" + +printf '%s\n' 'import sys, tomllib' 'from pathlib import Path' 'assert tomllib.loads(Path(sys.argv[1]).read_text())["value"] == sys.argv[2]' > "${WORK}/check_escape.py" +"${UV}" run --no-project python "${WORK}/check_escape.py" "${WORK}/escaped.toml" "${raw}" + +mkdir -p "${WORK}/bin" +ln -s /usr/bin/true "${WORK}/bin/mise" +ln -s /usr/bin/false "${WORK}/bin/lsof" +ln -s /bin/bash "${WORK}/bin/gateway" + +printf '%s\n' 'exec() {' ' config=""' ' while [ "$#" -gt 0 ]; do' ' if [ "$1" = "--config" ]; then config="$2"; shift 2; else shift; fi' ' done' ' cp "${config}" "${CAPTURED_CONFIG}"' ' builtin exit 0' '}' > "${WORK}/fake-gateway-env" + +CAPTURED_CONFIG="${WORK}/generated.toml" +BASH_ENV="${WORK}/fake-gateway-env" CAPTURED_CONFIG="${CAPTURED_CONFIG}" PATH="${WORK}/bin:${PATH}" KUBERNETES_SERVICE_HOST=fixture OPENSHELL_GATEWAY_BIN=/bin/true OPENSHELL_GATEWAY_STATE_DIR="${WORK}/state" OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent OPENSHELL_GRPC_ENDPOINT=https://callback.example.test:9443 bash "${ROOT}/tasks/scripts/gateway.sh" + +printf '%s\n' 'import sys, tomllib' 'from pathlib import Path' 'config = tomllib.loads(Path(sys.argv[1]).read_text())' 'gateway = config["openshell"]["gateway"]' 'driver = config["openshell"]["drivers"]["kubernetes"]' 'assert config["openshell"]["version"] == 2' 'assert gateway["compute_driver"] == "kubernetes"' 'assert "compute_drivers" not in gateway' 'assert driver["image_pull_policy"] == "if_not_present"' 'assert driver["grpc_endpoint"] == "https://callback.example.test:9443"' > "${WORK}/check_generated.py" +"${UV}" run --no-project python "${WORK}/check_generated.py" "${CAPTURED_CONFIG}" +echo "gateway generated-TOML tests passed" diff --git a/tasks/scripts/test-gateway-pull-policy.sh b/tasks/scripts/test-gateway-pull-policy.sh new file mode 100755 index 0000000000..7859b7c1d8 --- /dev/null +++ b/tasks/scripts/test-gateway-pull-policy.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=tasks/scripts/gateway-pull-policy.sh +source "${ROOT}/tasks/scripts/gateway-pull-policy.sh" + +assert_policy() { + local input=$1 + local expected=$2 + local actual + actual="$(normalize_image_pull_policy "${input}")" + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %q -> %q, got %q\n' "${input}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +for input in always Always ALWAYS; do + assert_policy "${input}" always +done +for input in if_not_present IfNotPresent ifnotpresent IFNOTPRESENT missing MISSING; do + assert_policy "${input}" if_not_present +done +for input in never Never NEVER; do + assert_policy "${input}" never +done +for input in newer Newer NEWER; do + assert_policy "${input}" newer +done + +if normalize_image_pull_policy sometimes >/dev/null 2>&1; then + echo "unsupported policy unexpectedly succeeded" >&2 + exit 1 +fi + +for script in gateway.sh gateway-docker.sh gateway-podman.sh; do + if ! grep -q 'normalize_image_pull_policy' "${ROOT}/tasks/scripts/${script}"; then + echo "${script} does not normalize image pull policy" >&2 + exit 1 + fi +done + +echo "gateway pull-policy tests passed" + +# Empty and whitespace values are invalid rather than silently falling back to a default. +# Callers must supply one of the documented names. +assert_rejected_policy() { + local input=$1 + if normalize_image_pull_policy "${input}" >/dev/null 2>&1; then + printf 'unsupported policy unexpectedly succeeded: %q\n' "${input}" >&2 + exit 1 + fi +} + +assert_rejected_policy "" +assert_rejected_policy " " +assert_rejected_policy $'\t' +assert_rejected_policy " if_not_present" +assert_rejected_policy "if_not_present " + +# The VM driver does not render an image-pull-policy field, so it is intentionally +# excluded from Docker/Podman/Kubernetes normalization. Do not add it until it consumes it. +if grep -Fq 'gateway-pull-policy.sh' "${ROOT}/tasks/scripts/gateway-vm.sh"; then + echo "gateway-vm.sh unexpectedly normalizes an unused image pull policy" >&2 + exit 1 +fi +if ! grep -Fq 'intentionally does not normalize this input' "${ROOT}/tasks/scripts/gateway-vm.sh"; then + echo "gateway-vm.sh does not document its pull-policy exclusion" >&2 + exit 1 +fi + +echo "gateway pull-policy edge-case tests passed" diff --git a/tasks/scripts/test-packaging-assets.sh b/tasks/scripts/test-packaging-assets.sh index a03f60559a..ac7b2b190a 100755 --- a/tasks/scripts/test-packaging-assets.sh +++ b/tasks/scripts/test-packaging-assets.sh @@ -59,4 +59,53 @@ assert_contains \ 'ExecStartPre=/usr/bin/openshell-gateway generate-certs --output-dir ${OPENSHELL_LOCAL_TLS_DIR} --server-san host.openshell.internal' assert_not_contains "$spec" '%%S/openshell/tls' +# Schema-v2 package startup wiring. +snap_wrapper="${ROOT}/tasks/scripts/snap-gateway-wrapper.sh" +package_deb="${ROOT}/tasks/scripts/package-deb.sh" +assert_file_exists "$snap_wrapper" +assert_file_exists "$package_deb" +assert_contains "$service" "ExecStartPre=/usr/bin/openshell-gateway config preflight" +assert_contains "$package_deb" "\$src_dir/openshell-gateway.service" +assert_contains "$package_deb" "\$pkgroot/usr/lib/systemd/user/openshell-gateway.service" +assert_contains "$snap_wrapper" "if [ -n \"\${OPENSHELL_GATEWAY_CONFIG:-}\" ]; then" +assert_contains \ + "$snap_wrapper" \ + "elif [ -e \"\$CANONICAL_CONFIG_FILE\" ] || [ -L \"\$CANONICAL_CONFIG_FILE\" ]; then" +assert_contains "$snap_wrapper" "config preflight -- --config \"\$CANONICAL_CONFIG_FILE\" \"\$@\"" +assert_not_contains "$snap_wrapper" "[ -f \"\$CANONICAL_CONFIG_FILE\" ]" +bash "$ROOT/tasks/scripts/test-snap-gateway-wrapper.sh" "$snap_wrapper" +if ! awk '/config preflight/ { seen = 1 } /generate-certs/ { exit !seen }' "$service"; then + echo "FAIL: Debian preflight must precede certificate generation" >&2 + exit 1 +fi + +# Build a throwaway package when Debian tooling is available to prove the +# staged unit comes from deploy/deb/. Other hosts retain the static source-to- +# destination assertion above; the real Debian upgrade lane remains required. +if command -v dpkg-deb >/dev/null 2>&1; then + package_work=$(mktemp -d "${TMPDIR:-/tmp}/openshell-package-assets.XXXXXX") + trap 'rm -rf "$package_work"' EXIT + mkdir -p "$package_work/bin" "$package_work/output" + for binary in openshell openshell-gateway openshell-driver-vm; do + printf '#!/bin/sh\nexit 0\n' >"$package_work/bin/$binary" + chmod +x "$package_work/bin/$binary" + done + OPENSHELL_CLI_BINARY="$package_work/bin/openshell" \ + OPENSHELL_GATEWAY_BINARY="$package_work/bin/openshell-gateway" \ + OPENSHELL_DRIVER_VM_BINARY="$package_work/bin/openshell-driver-vm" \ + OPENSHELL_DEB_VERSION=0.0.0 \ + OPENSHELL_DEB_ARCH=amd64 \ + OPENSHELL_OUTPUT_DIR="$package_work/output" \ + "$package_deb" >/dev/null + dpkg-deb --fsys-tarfile "$package_work/output/openshell_0.0.0_amd64.deb" \ + | tar -xOf - ./usr/lib/systemd/user/openshell-gateway.service \ + >"$package_work/staged.service" + if ! cmp -s "$service" "$package_work/staged.service"; then + echo "FAIL: package-deb did not stage the current Debian service" >&2 + exit 1 + fi +else + echo "SKIP: dpkg-deb unavailable; Debian artifact staging requires its assigned lane" +fi + echo "packaging asset tests passed" diff --git a/tasks/scripts/test-snap-gateway-wrapper.sh b/tasks/scripts/test-snap-gateway-wrapper.sh new file mode 100755 index 0000000000..733ec796b5 --- /dev/null +++ b/tasks/scripts/test-snap-gateway-wrapper.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +wrapper_input=${1:?Usage: test-snap-gateway-wrapper.sh } +wrapper_dir=$(cd "$(dirname "$wrapper_input")" && pwd) +wrapper="${wrapper_dir}/$(basename "$wrapper_input")" +work=$(mktemp -d "${TMPDIR:-/tmp}/openshell snap wrapper.XXXXXX") +trap 'rm -rf "$work"' EXIT + +snap="$work/snap" +common="$work/common" +log="$work/calls" +expected="$work/expected" +mkdir -p "$snap/bin" "$common" + +cat >"$snap/bin/openshell-gateway" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >>"$FAKE_GATEWAY_LOG" +printf 'env:%s|%s|%s\n' \ + "${OPENSHELL_GATEWAY_CONFIG:-}" \ + "${OPENSHELL_DB_URL:-}" \ + "${OPENSHELL_DISABLE_TLS:-}" >>"$FAKE_GATEWAY_LOG" +if [ "${1:-}" = config ] && [ "${2:-}" = preflight ]; then + if [ "${FAKE_PREFLIGHT_FAIL:-}" = 1 ]; then + exit 42 + fi + if [ "${FAKE_REJECT_UNPAIRED_RATE:-}" = 1 ]; then + case " $* " in + *" --grpc-rate-limit-requests "*) + case " $* " in + *" --grpc-rate-limit-window-seconds "*) ;; + *) exit 43 ;; + esac + ;; + esac + fi +fi +EOF +chmod +x "$snap/bin/openshell-gateway" + +run_wrapper() { + local config=$1 + local fail=${2:-} + if [ "$config" = unset ]; then + env -u OPENSHELL_GATEWAY_CONFIG \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL="$fail" \ + "$wrapper" --trace + else + env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$config" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL="$fail" \ + "$wrapper" --trace + fi +} + +assert_log() { + printf '%s\n' "$1" >"$expected" + if ! cmp -s "$expected" "$log"; then + echo "FAIL: unexpected call sequence" >&2 + diff -u "$expected" "$log" >&2 + exit 1 + fi +} + +override="$work/override.toml" +printf 'operator override\n' >"$override" +cp "$override" "$work/override-before" +: >"$log" +run_wrapper "$override" +assert_log "config preflight -- --trace +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--trace +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/override-before" "$override" + +cli_config="$work/cli.toml" +printf 'CLI override\n' >"$cli_config" +cp "$cli_config" "$work/cli-before" +: >"$log" +env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" --trace --config "$cli_config" +assert_log "config preflight -- --trace --config $cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--trace --config $cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/cli-before" "$cli_config" + +: >"$log" +if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_PREFLIGHT_FAIL=1 \ + "$wrapper" --config="$cli_config"; then + echo "FAIL: CLI-selected config preflight failure reached gateway start" >&2 + exit 1 +fi +assert_log "config preflight -- --config=$cli_config +env:$override|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/cli-before" "$cli_config" + +: >"$log" +if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + FAKE_REJECT_UNPAIRED_RATE=1 \ + "$wrapper" --grpc-rate-limit-requests 10; then + echo "FAIL: invalid daemon overrides reached gateway start" >&2 + exit 1 +fi +assert_log "config preflight -- --grpc-rate-limit-requests 10 +env:$override|sqlite:$common/gateway.db?mode=rwc|true" + +for invalid_selector in terminator nested-config; do + : >"$log" + if [ "$invalid_selector" = terminator ]; then + invalid_args=(--config --) + else + invalid_args=(--config "--config=$cli_config") + fi + if env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" "${invalid_args[@]}"; then + echo "FAIL: invalid $invalid_selector selector reached gateway execution" >&2 + exit 1 + fi + if [ -s "$log" ]; then + echo "FAIL: invalid $invalid_selector selector reached preflight" >&2 + exit 1 + fi +done + +: >"$log" +env \ + SNAP="$snap" \ + SNAP_COMMON="$common" \ + OPENSHELL_GATEWAY_CONFIG="$override" \ + FAKE_GATEWAY_LOG="$log" \ + "$wrapper" --config=--dash-leading +assert_log "config preflight -- --config=--dash-leading +env:$override|sqlite:$common/gateway.db?mode=rwc|true +--config=--dash-leading +env:$override|sqlite:$common/gateway.db?mode=rwc|true" + +canonical="$common/gateway.toml" +printf 'valid schema-v2\n' >"$canonical" +cp "$canonical" "$work/canonical-before" +: >"$log" +run_wrapper unset +assert_log "config preflight -- --config $canonical --trace +env:|sqlite:$common/gateway.db?mode=rwc|true +--config $canonical --trace +env:|sqlite:$common/gateway.db?mode=rwc|true" +cmp -s "$work/canonical-before" "$canonical" + +rm "$canonical" +: >"$log" +run_wrapper unset +assert_log "config preflight -- --trace +env:|sqlite:$common/gateway.db?mode=rwc|true +--trace +env:|sqlite:$common/gateway.db?mode=rwc|true" + +assert_preflight_failure() { + local name=$1 + : >"$log" + if run_wrapper unset 1; then + echo "FAIL: $name reached gateway start" >&2 + exit 1 + fi + assert_log "config preflight -- --config $canonical --trace +env:|sqlite:$common/gateway.db?mode=rwc|true" +} + +printf 'legacy version = 1\n' >"$canonical" +cp "$canonical" "$work/legacy-before" +assert_preflight_failure legacy +cmp -s "$work/legacy-before" "$canonical" + +printf 'not valid TOML = [\n' >"$canonical" +cp "$canonical" "$work/malformed-before" +assert_preflight_failure malformed +cmp -s "$work/malformed-before" "$canonical" + +rm "$canonical" +ln -s "$work/missing-target" "$canonical" +readlink "$canonical" >"$work/link-before" +assert_preflight_failure broken-symlink +readlink "$canonical" >"$work/link-after" +cmp -s "$work/link-before" "$work/link-after" + +rm "$canonical" +mkdir "$canonical" +printf 'nonregular marker\n' >"$canonical/marker" +cp "$canonical/marker" "$work/marker-before" +assert_preflight_failure nonregular +cmp -s "$work/marker-before" "$canonical/marker" + +echo "Snap gateway wrapper tests passed" diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 7d0b05334d..6d66fbb5d4 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -54,10 +54,10 @@ start_gateway() { mkdir -p "$STATE_DIR" cat >"$config" <