From 90d208fa5e1221c68f069119444ff978adf20ed9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 8 Aug 2026 05:27:23 +0000 Subject: [PATCH 1/3] feat(controller): add persistent sandbox workspaces --- cli/src/commands/add.test.ts | 69 + cli/src/commands/add.ts | 76 ++ controller/src/crd.rs | 175 ++- controller/src/crd_validations.rs | 54 +- controller/src/helm_drift.rs | 55 +- controller/src/reconciler/mod.rs | 1199 ++++++++++++++++- controller/src/reconciler/tests.rs | 779 ++++++++++- controller/src/status/conditions.rs | 18 + controller/src/status/mod.rs | 155 ++- deploy/helm/kars/templates/crd.yaml | 54 +- deploy/helm/kars/templates/rbac.yaml | 12 +- docs/api/conditions.md | 13 + docs/api/crd-reference.md | 24 +- docs/api/lifecycle.md | 6 +- docs/cli-reference.md | 14 + docs/getting-started.md | 13 + .../2026-08-07-persistent-workspace-design.md | 616 +++++++++ ...-07-persistent-workspace-implementation.md | 130 ++ docs/runtimes/CONTRACT.md | 3 +- docs/security.md | 10 +- sandbox-images/openclaw/Dockerfile | 3 +- sandbox-images/openclaw/entrypoint.sh | 19 +- .../openclaw/workspace-bootstrap.sh | 87 ++ 23 files changed, 3523 insertions(+), 61 deletions(-) create mode 100644 docs/plans/2026-08-07-persistent-workspace-design.md create mode 100644 docs/plans/2026-08-07-persistent-workspace-implementation.md create mode 100644 sandbox-images/openclaw/workspace-bootstrap.sh diff --git a/cli/src/commands/add.test.ts b/cli/src/commands/add.test.ts index 417e48867..d5f61c879 100644 --- a/cli/src/commands/add.test.ts +++ b/cli/src/commands/add.test.ts @@ -36,6 +36,12 @@ interface AddOptions { openaiApiKey?: string; learnEgress: boolean; skills?: string; + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; + workspaceBootstrap?: string; + workspaceOverwrite?: "IfMissing" | "Always"; } function defaultOptions(overrides: Partial = {}): AddOptions { @@ -113,6 +119,28 @@ function buildSandboxManifest(name: string, options: AddOptions) { np.egressMode = "Learn"; } + if (options.workspaceStorage || options.workspaceExistingClaim) { + const workspace = options.workspaceExistingClaim + ? { existingClaim: options.workspaceExistingClaim } + : { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy ?? "Retain", + }; + (sandbox.spec as Record).storage = { workspace }; + } + + if (options.workspaceBootstrap) { + const runtime = (sandbox.spec as any).runtime; + runtime.openclaw.workspace = { + bootstrapConfigMapRef: { name: options.workspaceBootstrap }, + overwritePolicy: options.workspaceOverwrite ?? "IfMissing", + }; + } + return sandbox; } @@ -180,6 +208,47 @@ describe("KarsSandbox manifest generation", () => { }); }); + it("configures a dynamically provisioned workspace PVC", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ + workspaceStorage: "20Gi", + workspaceStorageClass: "managed-csi", + workspaceRetainPolicy: "Retain", + }), + ); + expect((manifest.spec as any).storage.workspace).toEqual({ + size: "20Gi", + storageClassName: "managed-csi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }); + }); + + it("references an existing workspace claim without dynamic fields", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ workspaceExistingClaim: "restored-workspace" }), + ); + expect((manifest.spec as any).storage.workspace).toEqual({ + existingClaim: "restored-workspace", + }); + }); + + it("configures OpenClaw workspace bootstrap", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ + workspaceBootstrap: "my-agent-workspace", + workspaceOverwrite: "Always", + }), + ); + expect((manifest.spec as any).runtime.openclaw.workspace).toEqual({ + bootstrapConfigMapRef: { name: "my-agent-workspace" }, + overwritePolicy: "Always", + }); + }); + it("uses default model gpt-4.1 with azure/ prefix", () => { const manifest = buildSandboxManifest("a", defaultOptions()); const spec = manifest.spec as any; diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index b46b459a5..a7f0d5beb 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -26,6 +26,14 @@ export function addCommand(): Command { .option("--isolation ", "Isolation level: standard | enhanced | confidential", "enhanced") .option("--image ", "Custom sandbox image (default: from Helm values; OpenClaw runtime only)") + // ── Workspace storage (all runtimes; bootstrap is OpenClaw-only) ─── + .option("--workspace-storage ", "Create a persistent workspace PVC, e.g. 10Gi") + .option("--workspace-storage-class ", "StorageClass for a generated workspace PVC") + .option("--workspace-existing-claim ", "Use an existing PVC in the generated sandbox namespace") + .option("--workspace-retain-policy ", "Generated PVC deletion policy: Retain | Delete", "Retain") + .option("--workspace-bootstrap ", "[OpenClaw only] ConfigMap containing workspace bootstrap files") + .option("--workspace-overwrite ", "[OpenClaw only] Bootstrap policy: IfMissing | Always", "IfMissing") + // ── Inference budget (all runtimes) ──────────────────────────────── .option("--token-budget-daily ", "Daily token budget (0 = unlimited)", "0") .option("--token-budget-per-request ", "Per-request token limit (0 = unlimited)", "0") @@ -72,6 +80,7 @@ export function addCommand(): Command { .addHelpText("after", ` Flag groups (see --help for details): Core: --runtime, --model, --isolation, --image + Workspace: --workspace-storage, --workspace-existing-claim, --workspace-bootstrap Inference budget: --token-budget-* Governance / net: --governance, --trust-threshold, --policy-profile, --learn-egress Foundry agent: --agent-instructions, --agent-tools @@ -129,6 +138,40 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. console.error(chalk.red(`\n Error: --maf-language is only valid with --runtime microsoft-agent-framework.\n`)); process.exit(1); } + if (options.workspaceStorage && options.workspaceExistingClaim) { + console.error(chalk.red(`\n Error: --workspace-storage and --workspace-existing-claim are mutually exclusive.\n`)); + process.exit(1); + } + if (options.workspaceStorageClass && !options.workspaceStorage) { + console.error(chalk.red(`\n Error: --workspace-storage-class requires --workspace-storage .\n`)); + process.exit(1); + } + if (options.workspaceExistingClaim && options.workspaceRetainPolicy !== "Retain") { + console.error(chalk.red(`\n Error: --workspace-retain-policy applies only to generated PVCs; existing claims are always externally managed.\n`)); + process.exit(1); + } + if (!(["Retain", "Delete"] as const).includes(options.workspaceRetainPolicy)) { + console.error(chalk.red(`\n Error: --workspace-retain-policy must be Retain or Delete.\n`)); + process.exit(1); + } + if (runtimeKind !== "OpenClaw" && options.workspaceBootstrap) { + console.error(chalk.red(`\n Error: --workspace-bootstrap is only valid with --runtime openclaw.\n`)); + process.exit(1); + } + if (!(["IfMissing", "Always"] as const).includes(options.workspaceOverwrite)) { + console.error(chalk.red(`\n Error: --workspace-overwrite must be IfMissing or Always.\n`)); + process.exit(1); + } + if (!options.workspaceBootstrap && options.workspaceOverwrite !== "IfMissing") { + console.error(chalk.red(`\n Error: --workspace-overwrite requires --workspace-bootstrap .\n`)); + process.exit(1); + } + if (options.workspaceStorage && options.workspaceRetainPolicy === "Delete") { + console.log(chalk.yellow(" ⚠ Workspace retain policy is Delete: deleting the KarsSandbox will delete its PVC and data.")); + } + if (!options.workspaceStorage && !options.workspaceExistingClaim) { + console.log(chalk.yellow(" ⚠ Workspace is ephemeral (emptyDir): Pod recreation or suspension deletes sessions and files.")); + } const runtimeBlock = buildRuntimeBlock({ kind: runtimeKind, @@ -139,6 +182,13 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. byoContractVersion: options.byoContractVersion, mafLanguage: options.mafLanguage as "python" | "dotnet", }); + if (options.workspaceBootstrap) { + const openclaw = runtimeBlock.openclaw as Record; + openclaw.workspace = { + bootstrapConfigMapRef: { name: options.workspaceBootstrap }, + overwritePolicy: options.workspaceOverwrite, + }; + } const sandbox: Record = { apiVersion: "kars.azure.com/v1alpha1", @@ -174,6 +224,20 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. }, }; + if (options.workspaceStorage || options.workspaceExistingClaim) { + const workspace = options.workspaceExistingClaim + ? { existingClaim: options.workspaceExistingClaim } + : { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy, + }; + (sandbox.spec as Record).storage = { workspace }; + } + // Add Foundry agent config if provided if (options.agentInstructions || options.agentTools) { const agentSpec: Record = {}; @@ -591,6 +655,18 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. console.log(chalk.dim(` Namespace: ${namespace}`)); console.log(chalk.dim(` Model: ${options.model}`)); console.log(chalk.dim(` Isolation: ${options.isolation}`)); + if (options.workspaceExistingClaim) { + console.log(chalk.dim(` Workspace: existing PVC ${options.workspaceExistingClaim} (externally managed)`)); + } else if (options.workspaceStorage) { + console.log(chalk.dim( + ` Workspace: ${name}-workspace (${options.workspaceStorage}, ${options.workspaceStorageClass || "default StorageClass"}, ${options.workspaceRetainPolicy})`, + )); + } else { + console.log(chalk.yellow(" Workspace: ephemeral emptyDir")); + } + if (options.workspaceBootstrap) { + console.log(chalk.dim(` Bootstrap: ${options.workspaceBootstrap} (${options.workspaceOverwrite})`)); + } if (options.channels) { console.log(chalk.dim(` Channels: ${options.channels}`)); } diff --git a/controller/src/crd.rs b/controller/src/crd.rs index 8eb458c17..d3c677b44 100644 --- a/controller/src/crd.rs +++ b/controller/src/crd.rs @@ -72,6 +72,11 @@ pub struct KarsSandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_ref: Option, + /// Optional per-sandbox storage configuration. When omitted, the runtime + /// workspace remains an ephemeral `emptyDir` for backward compatibility. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + /// Network policy pub network_policy: Option, @@ -128,9 +133,9 @@ pub struct KarsSandboxSpec { /// to `replicas: 0` and stamps the K8s `Suspended=True` Condition /// with reason `SuspendedBySpec`. The namespace, NetworkPolicy, /// ServiceAccount, governance ConfigMaps, and any Azure - /// federated-identity binding are preserved byte-identical, so - /// flipping back to `Some(false)` (or unsetting) restores the - /// agent in-place without losing state. + /// federated-identity binding are preserved byte-identical. Runtime files + /// survive the transition only when `spec.storage.workspace` uses a PVC; + /// the backward-compatible `emptyDir` workspace is deleted with the Pod. /// /// Distinct from `Suspended=True / Reason=OverlayMode` (induced by /// `spec.upstreamCompatibility.sigsAgentSandbox=overlay`), which @@ -155,6 +160,94 @@ pub struct KarsSandboxSpec { pub mesh_auth: Option, } +/// Per-sandbox storage resources. +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SandboxStorageSpec { + /// Runtime workspace mounted at `/sandbox`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, +} + +/// Persistent workspace configuration. +/// +/// An existing claim is a pure reference. Otherwise omitted dynamic fields +/// resolve to 10Gi, ReadWriteOnce, and Retain. +#[derive(Debug, Serialize, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub existing_claim: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_class_name: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub access_modes: Vec, + #[serde(default)] + pub retain_policy: WorkspaceRetainPolicy, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct RawWorkspaceStorageSpec { + existing_claim: Option, + size: Option, + storage_class_name: Option, + access_modes: Option>, + retain_policy: Option, +} + +impl<'de> Deserialize<'de> for WorkspaceStorageSpec { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = RawWorkspaceStorageSpec::deserialize(deserializer)?; + let references_existing_claim = raw.existing_claim.is_some(); + Ok(Self { + existing_claim: raw.existing_claim, + size: raw + .size + .or_else(|| (!references_existing_claim).then(|| "10Gi".to_string())), + storage_class_name: raw.storage_class_name, + access_modes: raw.access_modes.unwrap_or_else(|| { + if references_existing_claim { + Vec::new() + } else { + vec![PersistentVolumeAccessMode::ReadWriteOnce] + } + }), + retain_policy: raw.retain_policy.unwrap_or_default(), + }) + } +} + +impl Default for WorkspaceStorageSpec { + fn default() -> Self { + Self { + existing_claim: None, + size: Some("10Gi".to_string()), + storage_class_name: None, + access_modes: vec![PersistentVolumeAccessMode::ReadWriteOnce], + retain_policy: WorkspaceRetainPolicy::Retain, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum PersistentVolumeAccessMode { + ReadWriteOnce, + ReadWriteOncePod, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum WorkspaceRetainPolicy { + #[default] + Retain, + Delete, +} + /// Per-sandbox mesh authentication mode. /// /// Two terminal modes are supported: @@ -849,6 +942,9 @@ pub struct OpenClawConfig { pub version: Option, pub image: Option, pub config: Option, + /// Declarative initialization for selected OpenClaw workspace files. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, /// Extra environment variables injected into the openclaw container as `key: value` /// pairs. Used by the controller to propagate offload parameters /// (`OFFLOAD_REQUEST_ID`, `OFFLOAD_PARENT_AMID`, `OFFLOAD_TASK`, @@ -857,6 +953,22 @@ pub struct OpenClawConfig { pub extra_env: Option>, } +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct OpenClawWorkspaceSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bootstrap_config_map_ref: Option, + #[serde(default)] + pub overwrite_policy: WorkspaceOverwritePolicy, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum WorkspaceOverwritePolicy { + #[default] + IfMissing, + Always, +} + #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SandboxConfig { @@ -1393,12 +1505,69 @@ mod tests { // empty-name on apply. assert!(spec.inference_ref.name.is_empty()); assert!(spec.network_policy.is_none()); + assert!(spec.storage.is_none()); assert!(spec.agent.is_none()); assert!(spec.governance.is_none()); assert!(spec.azure_services.is_none()); assert!(spec.resources.is_none()); } + #[test] + fn dynamic_workspace_storage_defaults_and_uses_camel_case() { + let workspace: WorkspaceStorageSpec = serde_json::from_value(serde_json::json!({})) + .expect("empty workspace storage uses safe defaults"); + assert_eq!(workspace.size.as_deref(), Some("10Gi")); + assert_eq!( + workspace.access_modes, + vec![PersistentVolumeAccessMode::ReadWriteOnce] + ); + assert_eq!(workspace.retain_policy, WorkspaceRetainPolicy::Retain); + assert!(workspace.existing_claim.is_none()); + + let value = serde_json::to_value(SandboxStorageSpec { + workspace: Some(workspace), + }) + .unwrap(); + let workspace = &value["workspace"]; + assert_eq!(workspace["size"], "10Gi"); + assert_eq!( + workspace["accessModes"], + serde_json::json!(["ReadWriteOnce"]) + ); + assert_eq!(workspace["retainPolicy"], "Retain"); + assert!(workspace.get("existingClaim").is_none()); + } + + #[test] + fn existing_claim_and_openclaw_bootstrap_round_trip() { + let storage: SandboxStorageSpec = serde_json::from_value(serde_json::json!({ + "workspace": { "existingClaim": "restored-workspace" } + })) + .unwrap(); + let workspace = storage.workspace.expect("workspace storage"); + assert_eq!( + workspace.existing_claim.as_deref(), + Some("restored-workspace") + ); + + let cfg: OpenClawConfig = serde_json::from_value(serde_json::json!({ + "workspace": { + "bootstrapConfigMapRef": { "name": "teaching-agent-workspace" }, + "overwritePolicy": "IfMissing" + } + })) + .unwrap(); + let workspace = cfg.workspace.expect("OpenClaw workspace config"); + assert_eq!( + workspace.bootstrap_config_map_ref.unwrap().name, + "teaching-agent-workspace" + ); + assert_eq!( + workspace.overwrite_policy, + WorkspaceOverwritePolicy::IfMissing + ); + } + #[test] fn sandbox_config_security_defaults() { let cfg = SandboxConfig::default(); diff --git a/controller/src/crd_validations.rs b/controller/src/crd_validations.rs index bf121b5cd..d626a5da9 100644 --- a/controller/src/crd_validations.rs +++ b/controller/src/crd_validations.rs @@ -752,18 +752,52 @@ pub fn kars_sre_action_crd() -> CustomResourceDefinition { .expect("kube-rs derive must produce a spec property on KarsSREAction") } -/// `KarsSandbox` CRD as produced by the kube-rs derive. -/// -/// Currently no `kars_sandbox_validations()` helper exists — `KarsSandbox` -/// has historically relied on its hand-written -/// `deploy/helm/kars/templates/crd.yaml` (with rich CEL rules baked -/// in there) rather than rule-injection in code. This helper is exposed -/// so future drift tests / dumpers can compare the kube-rs-derived -/// schema to the hand-written one without each call site reimplementing -/// the `KarsSandbox::crd()` invocation. +fn inject_kars_sandbox_workspace_validations( + mut crd: CustomResourceDefinition, +) -> Option { + let root = crd + .spec + .versions + .first_mut()? + .schema + .as_mut()? + .open_api_v3_schema + .as_mut()?; + let workspace = root + .properties + .as_mut()? + .get_mut("spec")? + .properties + .as_mut()? + .get_mut("storage")? + .properties + .as_mut()? + .get_mut("workspace")?; + workspace.x_kubernetes_validations = Some(vec![ + ValidationRule { + rule: "!has(self.existingClaim) || (!has(self.size) && !has(self.storageClassName) && !has(self.accessModes) && !has(self.retainPolicy))".into(), + message: Some( + "existingClaim is mutually exclusive with dynamic provisioning fields".into(), + ), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ValidationRule { + rule: "!has(self.size) || quantity(self.size).isGreaterThan(quantity('0'))".into(), + message: Some("workspace size must be a positive Kubernetes quantity".into()), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ]); + Some(crd) +} + +/// `KarsSandbox` CRD with workspace storage CEL injected into the generated +/// schema. The hand-written Helm CRD carries the same rule. #[must_use] pub fn kars_sandbox_crd() -> CustomResourceDefinition { - crate::crd::KarsSandbox::crd() + inject_kars_sandbox_workspace_validations(crate::crd::KarsSandbox::crd()) + .expect("kube-rs derive must produce spec.storage.workspace") } #[cfg(test)] diff --git a/controller/src/helm_drift.rs b/controller/src/helm_drift.rs index 7d37ab7b4..e65e407bc 100644 --- a/controller/src/helm_drift.rs +++ b/controller/src/helm_drift.rs @@ -33,9 +33,14 @@ #[cfg(test)] use crate::crd_validations::{ a2a_agent_crd, egress_approval_crd, inference_policy_crd, kars_eval_crd, kars_memory_crd, - kars_sre_action_crd, mcp_server_crd, tool_policy_crd, trust_graph_crd, + kars_sandbox_crd, kars_sre_action_crd, mcp_server_crd, tool_policy_crd, trust_graph_crd, }; +const KARSSANDBOX_HELM_CRD_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../deploy/helm/kars/templates/crd.yaml" +); + const MCP_HELM_CRD_PATH: &str = concat!( env!("CARGO_MANIFEST_DIR"), "/../deploy/helm/kars/templates/crd-mcpserver.yaml" @@ -106,6 +111,7 @@ fn canonical_form(value: &serde_json::Value) -> serde_json::Value { #[cfg(test)] mod tests { use super::*; + use serde::Deserialize; /// One-shot dumper. Run via: /// @@ -181,6 +187,53 @@ mod tests { assert_helm_matches_rust(MCP_HELM_CRD_PATH, rust_crd_value, "mcpserver"); } + #[test] + fn helm_and_rust_expose_workspace_storage_and_bootstrap() { + let rust = serde_json::to_value(kars_sandbox_crd()).expect("rust CRD serializes"); + let helm_text = std::fs::read_to_string(KARSSANDBOX_HELM_CRD_PATH) + .expect("KarsSandbox Helm CRD exists"); + let helm: serde_json::Value = serde_yaml::Deserializer::from_str(&helm_text) + .next() + .map(serde_json::Value::deserialize) + .expect("KarsSandbox document exists") + .expect("KarsSandbox Helm CRD parses"); + + const SPEC: &str = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties"; + for (label, crd) in [("Rust", &rust), ("Helm", &helm)] { + assert!( + crd.pointer(&format!("{SPEC}/storage/properties/workspace")) + .is_some(), + "{label} schema must expose spec.storage.workspace" + ); + assert!( + crd.pointer(&format!( + "{SPEC}/runtime/properties/openclaw/properties/workspace" + )) + .is_some(), + "{label} schema must expose runtime.openclaw.workspace" + ); + } + + for (label, crd) in [("Rust", &rust), ("Helm", &helm)] { + let validations = crd + .pointer(&format!( + "{SPEC}/storage/properties/workspace/x-kubernetes-validations" + )) + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("{label} workspace CEL validations")); + assert!(validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("existingClaim") && rule.contains("size")) + })); + assert!(validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("quantity(self.size)")) + })); + } + } + #[test] fn helm_toolpolicy_crd_matches_rust_schema() { let rust_crd_value = diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index ce919983d..0268d7819 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -14,7 +14,8 @@ use anyhow::Result; use futures::StreamExt; use k8s_openapi::api::{ apps::v1::Deployment, - core::v1::{ConfigMap, Namespace, Secret, Service, ServiceAccount}, + batch::v1::{CronJob, Job}, + core::v1::{ConfigMap, Namespace, PersistentVolumeClaim, Pod, Secret, Service, ServiceAccount}, networking::v1::NetworkPolicy, rbac::v1::ClusterRoleBinding, }; @@ -41,6 +42,746 @@ pub(crate) mod trustgraph_mount; use mcp_egress::mcp_egress_rule; +#[derive(Debug, Clone, PartialEq)] +struct WorkspaceStoragePlan { + volume: serde_json::Value, + claim: Option, +} + +fn build_workspace_storage_plan( + sandbox_name: &str, + sandbox_namespace: &str, + storage: Option<&crate::crd::SandboxStorageSpec>, + sandbox_uid: Option<&str>, +) -> WorkspaceStoragePlan { + let Some(workspace) = storage.and_then(|storage| storage.workspace.as_ref()) else { + return WorkspaceStoragePlan { + volume: json!({"name": "sandbox-data", "emptyDir": {}}), + claim: None, + }; + }; + + if let Some(existing_claim) = workspace.existing_claim.as_deref() { + return WorkspaceStoragePlan { + volume: json!({ + "name": "sandbox-data", + "persistentVolumeClaim": {"claimName": existing_claim} + }), + claim: None, + }; + } + + let claim_name = format!("{sandbox_name}-workspace"); + let retain_policy = match workspace.retain_policy { + crate::crd::WorkspaceRetainPolicy::Retain => "Retain", + crate::crd::WorkspaceRetainPolicy::Delete => "Delete", + }; + let mut metadata = json!({ + "name": claim_name, + "namespace": sandbox_namespace, + "labels": { + "app.kubernetes.io/managed-by": "kars-controller", + "kars.azure.com/sandbox": sandbox_name, + "kars.azure.com/storage-role": "workspace" + }, + "annotations": { + "kars.azure.com/retain-policy": retain_policy + } + }); + if let Some(uid) = sandbox_uid { + metadata["annotations"]["kars.azure.com/sandbox-uid"] = json!(uid); + } + + let access_modes = workspace + .access_modes + .iter() + .map(|mode| match mode { + crate::crd::PersistentVolumeAccessMode::ReadWriteOnce => "ReadWriteOnce", + crate::crd::PersistentVolumeAccessMode::ReadWriteOncePod => "ReadWriteOncePod", + }) + .collect::>(); + let mut spec = json!({ + "accessModes": access_modes, + "resources": { + "requests": { + "storage": workspace.size.as_deref().unwrap_or("10Gi") + } + } + }); + if let Some(storage_class_name) = workspace.storage_class_name.as_deref() { + spec["storageClassName"] = json!(storage_class_name); + } + + WorkspaceStoragePlan { + volume: json!({ + "name": "sandbox-data", + "persistentVolumeClaim": {"claimName": claim_name} + }), + claim: Some(json!({ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": metadata, + "spec": spec + })), + } +} + +fn validate_workspace_storage_spec( + workspace: &crate::crd::WorkspaceStorageSpec, +) -> Result<(), String> { + let mut errors = Vec::new(); + if workspace.existing_claim.is_some() { + if workspace.size.is_some() + || workspace.storage_class_name.is_some() + || !workspace.access_modes.is_empty() + || workspace.retain_policy != crate::crd::WorkspaceRetainPolicy::Retain + { + errors.push( + "existingClaim is mutually exclusive with size, storageClassName, accessModes, and retainPolicy" + .to_string(), + ); + } + } else { + if workspace.size.as_deref().is_none_or(str::is_empty) { + errors.push("dynamic workspace size must not be empty".to_string()); + } + if workspace.access_modes.len() != 1 { + errors.push("dynamic workspace accessModes must contain exactly one mode".to_string()); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +fn validate_workspace_claim(claim: &PersistentVolumeClaim) -> Result<(), String> { + let access_modes = claim + .spec + .as_ref() + .and_then(|spec| spec.access_modes.as_ref()) + .cloned() + .unwrap_or_default(); + if access_modes.len() != 1 + || !matches!( + access_modes.first().map(String::as_str), + Some("ReadWriteOnce" | "ReadWriteOncePod") + ) + { + return Err(format!( + "workspace PVC `{}` must use exactly one of ReadWriteOnce or ReadWriteOncePod", + claim.name_any() + )); + } + if claim + .status + .as_ref() + .and_then(|status| status.phase.as_deref()) + == Some("Lost") + { + return Err(format!("workspace PVC `{}` is Lost", claim.name_any())); + } + if claim + .spec + .as_ref() + .and_then(|spec| spec.volume_mode.as_deref()) + .is_some_and(|mode| mode != "Filesystem") + { + return Err(format!( + "workspace PVC `{}` must use Filesystem volumeMode", + claim.name_any() + )); + } + Ok(()) +} + +fn validate_dynamic_claim_provenance( + claim: &PersistentVolumeClaim, + sandbox_uid: Option<&str>, +) -> Result<(), String> { + let recorded_uid = claim + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get("kars.azure.com/sandbox-uid")) + .map(String::as_str); + if sandbox_uid.is_some() && recorded_uid == sandbox_uid { + return Ok(()); + } + Err(format!( + "workspace PVC `{}` belongs to a different sandbox instance; use spec.storage.workspace.existingClaim for explicit recovery", + claim.name_any() + )) +} + +fn workspace_storage_status_conditions( + sandbox: &KarsSandbox, + claim_phase: Option<&str>, +) -> Vec { + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let generation = sandbox.metadata.generation; + let storage = match claim_phase { + None => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::EMPTY_DIR, + "workspace uses ephemeral emptyDir storage", + generation, + ), + Some("Bound") => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CLAIM_BOUND, + "workspace PVC is Bound", + generation, + ), + Some(_) => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CLAIM_PENDING, + "workspace PVC is waiting to bind", + generation, + ), + }; + + if storage.status == crate::status::conditions::status::TRUE { + vec![storage] + } else { + vec![ + storage, + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_READY), + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace storage", + generation, + ), + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_PROGRESSING), + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CREATING, + "waiting for workspace storage", + generation, + ), + ] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum WorkspaceBootstrapState { + NotConfigured, + Pending, + Ready, + Failed(String), +} + +fn workspace_bootstrap_state(pods: &[Pod]) -> WorkspaceBootstrapState { + let mut saw_ready = false; + let mut saw_pending = false; + for pod in pods { + if pod.metadata.deletion_timestamp.is_some() { + continue; + } + let Some(statuses) = pod + .status + .as_ref() + .and_then(|status| status.init_container_statuses.as_ref()) + else { + saw_pending = true; + continue; + }; + let Some(status) = statuses + .iter() + .find(|status| status.name == "workspace-bootstrap") + else { + saw_pending = true; + continue; + }; + if let Some(terminated) = status + .state + .as_ref() + .and_then(|state| state.terminated.as_ref()) + { + if terminated.exit_code == 0 { + saw_ready = true; + continue; + } + return WorkspaceBootstrapState::Failed( + terminated + .message + .clone() + .or_else(|| terminated.reason.clone()) + .unwrap_or_else(|| format!("exit code {}", terminated.exit_code)), + ); + } + if let Some(terminated) = status + .last_state + .as_ref() + .and_then(|state| state.terminated.as_ref()) + && terminated.exit_code != 0 + && status.restart_count > 0 + { + return WorkspaceBootstrapState::Failed( + terminated + .message + .clone() + .or_else(|| terminated.reason.clone()) + .unwrap_or_else(|| format!("exit code {}", terminated.exit_code)), + ); + } + if let Some(waiting) = status + .state + .as_ref() + .and_then(|state| state.waiting.as_ref()) + && matches!( + waiting.reason.as_deref(), + Some("CreateContainerError" | "RunContainerError") + ) + { + return WorkspaceBootstrapState::Failed( + waiting + .message + .clone() + .or_else(|| waiting.reason.clone()) + .unwrap_or_else(|| "bootstrap container could not start".to_string()), + ); + } + saw_pending = true; + } + if saw_ready && !saw_pending { + WorkspaceBootstrapState::Ready + } else { + WorkspaceBootstrapState::Pending + } +} + +fn workspace_bootstrap_status_conditions( + sandbox: &KarsSandbox, + state: &WorkspaceBootstrapState, +) -> Vec { + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let generation = sandbox.metadata.generation; + let condition = |condition_type: &str, status: &str, reason: &str, message: &str| { + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, condition_type), + condition_type, + status, + reason, + message, + generation, + ) + }; + + match state { + WorkspaceBootstrapState::NotConfigured => Vec::new(), + WorkspaceBootstrapState::Ready => vec![condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::RECONCILED, + "workspace bootstrap completed", + )], + WorkspaceBootstrapState::Pending => vec![ + condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap init container", + ), + condition( + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap", + ), + condition( + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap", + ), + ], + WorkspaceBootstrapState::Failed(message) => vec![ + condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + message, + ), + condition( + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + "workspace bootstrap failed", + ), + condition( + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::FAILED, + "workspace bootstrap cannot make progress", + ), + condition( + crate::status::conditions::TYPE_DEGRADED, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + message, + ), + ], + } +} + +#[derive(Debug, Clone, PartialEq)] +struct WorkspaceBootstrapPlan { + volume: serde_json::Value, + init_container: serde_json::Value, +} + +fn build_workspace_bootstrap_plan( + config: &crate::crd::OpenClawConfig, + image: &str, + image_pull_policy: &str, + config_map_uid: &str, + resource_version: &str, +) -> Option { + let workspace = config.workspace.as_ref()?; + let config_map = workspace.bootstrap_config_map_ref.as_ref()?; + let overwrite_policy = match workspace.overwrite_policy { + crate::crd::WorkspaceOverwritePolicy::IfMissing => "IfMissing", + crate::crd::WorkspaceOverwritePolicy::Always => "Always", + }; + Some(WorkspaceBootstrapPlan { + volume: json!({ + "name": "workspace-bootstrap", + "configMap": { + "name": config_map.name, + "defaultMode": 288 + } + }), + init_container: json!({ + "name": "workspace-bootstrap", + "image": image, + "imagePullPolicy": image_pull_policy, + "command": ["/usr/local/bin/workspace-bootstrap.sh"], + "env": [ + { + "name": "KARS_WORKSPACE_OVERWRITE_POLICY", + "value": overwrite_policy + }, + { + "name": "KARS_WORKSPACE_BOOTSTRAP_CONFIG_MAP_UID", + "value": config_map_uid + }, + { + "name": "KARS_WORKSPACE_BOOTSTRAP_RESOURCE_VERSION", + "value": resource_version + } + ], + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": [ + {"name": "sandbox-data", "mountPath": "/sandbox"}, + {"name": "workspace-bootstrap", "mountPath": "/etc/kars/workspace-bootstrap", "readOnly": true} + ], + "resources": { + "requests": {"cpu": "5m", "memory": "16Mi"}, + "limits": {"cpu": "100m", "memory": "64Mi"} + } + }), + }) +} + +fn validate_workspace_bootstrap_config_map(config_map: &ConfigMap) -> Result<(), String> { + if config_map + .binary_data + .as_ref() + .is_some_and(|data| !data.is_empty()) + { + return Err("workspace bootstrap ConfigMap must not contain binaryData".to_string()); + } + + const ALLOWED_FILES: &[&str] = &[ + "AGENTS.md", + "SOUL.md", + "HEARTBEAT.md", + "TOOLS.md", + "USER.md", + ]; + if let Some(data) = config_map.data.as_ref() { + let unsupported = data + .keys() + .filter(|key| !ALLOWED_FILES.contains(&key.as_str())) + .cloned() + .collect::>(); + if !unsupported.is_empty() { + return Err(format!( + "workspace bootstrap ConfigMap contains unsupported files: {}", + unsupported.join(", ") + )); + } + } + Ok(()) +} + +fn preserve_namespace_on_delete(storage: Option<&crate::crd::SandboxStorageSpec>) -> bool { + storage + .and_then(|storage| storage.workspace.as_ref()) + .is_some_and(|workspace| { + workspace.existing_claim.is_some() + || workspace.retain_policy == crate::crd::WorkspaceRetainPolicy::Retain + }) +} + +fn should_preserve_namespace_on_delete( + storage: Option<&crate::crd::SandboxStorageSpec>, + claims: &[PersistentVolumeClaim], + _sandbox_uid: Option<&str>, +) -> bool { + preserve_namespace_on_delete(storage) || !claims.is_empty() +} + +fn deletable_workspace_claims( + storage: Option<&crate::crd::SandboxStorageSpec>, + sandbox_name: &str, + claims: &[PersistentVolumeClaim], + sandbox_uid: Option<&str>, +) -> Vec { + let delete_authorized = storage + .and_then(|storage| storage.workspace.as_ref()) + .is_some_and(|workspace| { + workspace.existing_claim.is_none() + && workspace.retain_policy == crate::crd::WorkspaceRetainPolicy::Delete + }); + if !delete_authorized { + return Vec::new(); + } + let generated_claim_name = format!("{sandbox_name}-workspace"); + claims + .iter() + .filter(|claim| { + let annotations = claim.metadata.annotations.as_ref(); + let labels = claim.metadata.labels.as_ref(); + claim.metadata.name.as_deref() == Some(generated_claim_name.as_str()) + && labels + .and_then(|values| values.get("app.kubernetes.io/managed-by")) + .is_some_and(|value| value == "kars-controller") + && labels + .and_then(|values| values.get("kars.azure.com/storage-role")) + .is_some_and(|value| value == "workspace") + && annotations + .and_then(|values| values.get("kars.azure.com/sandbox-uid")) + .map(String::as_str) + == sandbox_uid + }) + .filter_map(|claim| claim.metadata.name.clone()) + .collect() +} + +fn validate_namespace_claim_reuse( + claims: &[PersistentVolumeClaim], + requested_existing_claim: Option<&str>, + current_claim: Option<&str>, + sandbox_uid: Option<&str>, +) -> Result<(), String> { + for claim in claims { + let recorded_uid = claim + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get("kars.azure.com/sandbox-uid")) + .map(String::as_str); + if recorded_uid == sandbox_uid + || requested_existing_claim == claim.metadata.name.as_deref() + || current_claim == claim.metadata.name.as_deref() + { + continue; + } + return Err(format!( + "namespace contains retained PVC `{}` from another sandbox instance; set spec.storage.workspace.existingClaim to recover it explicitly", + claim.name_any() + )); + } + Ok(()) +} + +fn workspace_deployment_strategy(persistent: bool) -> Option { + persistent.then(|| json!({"type": "Recreate"})) +} + +fn deployment_needs_recreate_cleanup(deployment: Option<&Deployment>, recreate: bool) -> bool { + recreate + && deployment + .and_then(|deployment| deployment.spec.as_ref()) + .and_then(|spec| spec.strategy.as_ref()) + .and_then(|strategy| strategy.rolling_update.as_ref()) + .is_some() +} + +fn workspace_desired_replicas( + suspended: bool, + existing_claim_requested: bool, + claim_phase: Option<&str>, +) -> i64 { + if suspended || (existing_claim_requested && claim_phase != Some("Bound")) { + 0 + } else { + 1 + } +} + +fn kube_api_access_mount() -> serde_json::Value { + json!({ + "name": "kube-api-access", + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "readOnly": true + }) +} + +fn kube_api_access_volume() -> serde_json::Value { + json!({ + "name": "kube-api-access", + "projected": { + "defaultMode": 420, + "sources": [ + {"serviceAccountToken": {"expirationSeconds": 3607, "path": "token"}}, + {"configMap": {"name": "kube-root-ca.crt", "items": [{"key": "ca.crt", "path": "ca.crt"}]}}, + {"downwardAPI": {"items": [{"path": "namespace", "fieldRef": {"apiVersion": "v1", "fieldPath": "metadata.namespace"}}]}} + ] + } + }) +} + +const WORKLOAD_IDENTITY_SKIP_CONTAINERS: &str = "egress-guard;workspace-bootstrap"; + +fn validate_workspace_volume_transition( + current_claim: Option<&str>, + desired_claim: Option<&str>, + suspended: bool, +) -> Result<(), String> { + if current_claim == desired_claim || (current_claim.is_none() && desired_claim.is_none()) { + return Ok(()); + } + if suspended { + return Ok(()); + } + Err("changing workspace volume mode or claim requires spec.suspended=true".to_string()) +} + +fn deployment_workspace_claim(deployment: &Deployment) -> Option<&str> { + deployment + .spec + .as_ref()? + .template + .spec + .as_ref()? + .volumes + .as_ref()? + .iter() + .find(|volume| volume.name == "sandbox-data")? + .persistent_volume_claim + .as_ref() + .map(|source| source.claim_name.as_str()) +} + +async fn cleanup_namespaced_sandbox_resources( + client: &Client, + namespace: &str, + sandbox_name: &str, +) -> Result<(), kube::Error> { + let delete = DeleteParams::default(); + let list = ListParams::default().labels(&format!("kars.azure.com/sandbox={sandbox_name}")); + + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + let secret_api = Api::::namespaced(client.clone(), namespace); + match secret_api + .delete(&format!("{sandbox_name}-credentials"), &delete) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(error), + } + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Ok(()) +} + +async fn stamp_degraded_with_condition( + client: &Client, + sandbox: &KarsSandbox, + name: &str, + condition_type: &'static str, + reason: &'static str, + message: &str, +) { + let mut patch = crate::status::build_degraded_status_patch(sandbox, reason, message); + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let condition = crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, condition_type), + condition_type, + crate::status::conditions::status::FALSE, + reason, + message, + sandbox.metadata.generation, + ); + if let Some(conditions) = patch["status"]["conditions"].as_array_mut() { + conditions.retain(|value| value["type"] != condition_type); + conditions.push(serde_json::to_value(condition).unwrap_or_default()); + } + let api = + Api::::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + if let Err(error) = api + .patch_status(name, &PatchParams::default(), &Patch::Merge(patch)) + .await + { + tracing::warn!(sandbox = %name, error = %error, "failed to patch dependency-specific degraded status"); + } +} + /// Build pod security context, conditionally including SELinux options and /// choosing between RuntimeDefault and Localhost seccomp profiles. /// For Kata (confidential), we use RuntimeDefault since the VM provides isolation. @@ -401,19 +1142,80 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = Api::all(client.clone()); - match ns_api.delete(&sandbox_ns, &DeleteParams::default()).await { - Ok(_) => tracing::info!("Namespace {sandbox_ns} deletion initiated"), - Err(kube::Error::Api(ae)) if ae.code == 404 => { - tracing::info!("Namespace {sandbox_ns} already gone"); - } - Err(e) => { - tracing::error!(error = %e, "Failed to delete namespace {sandbox_ns}"); + let claim_api: Api = Api::namespaced(client.clone(), &sandbox_ns); + let claims = match claim_api.list(&ListParams::default()).await { + Ok(claims) => claims.items, + Err(kube::Error::Api(error)) if error.code == 404 => Vec::new(), + Err(error) => { + tracing::error!(error = %error, "Failed to inspect PVCs in namespace {sandbox_ns}"); return Ok(Action::requeue(Duration::from_secs(10))); } + }; + let preserve_namespace = should_preserve_namespace_on_delete( + sandbox.spec.storage.as_ref(), + &claims, + sandbox.metadata.uid.as_deref(), + ); + if preserve_namespace { + tracing::info!( + sandbox = %name, + namespace = %sandbox_ns, + "KarsSandbox is being deleted — retaining workspace PVC and namespace" + ); + match cleanup_namespaced_sandbox_resources(client, &sandbox_ns, &name).await { + Ok(()) => tracing::info!( + namespace = %sandbox_ns, + "Kars-managed workload resources deleted; PVCs retained" + ), + Err(kube::Error::Api(error)) if error.code == 404 => { + tracing::info!("Namespace {sandbox_ns} already gone"); + } + Err(error) => { + tracing::error!(error = %error, "Failed to clean namespace {sandbox_ns}"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } + let claim_api: Api = + Api::namespaced(client.clone(), &sandbox_ns); + let deletable_claims = deletable_workspace_claims( + sandbox.spec.storage.as_ref(), + &name, + &claims, + sandbox.metadata.uid.as_deref(), + ); + for claim_name in &deletable_claims { + match claim_api.delete(claim_name, &DeleteParams::default()).await { + Ok(_) => { + tracing::info!(claim = %claim_name, "Delete-policy workspace PVC deletion initiated") + } + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => { + tracing::error!(claim = %claim_name, error = %error, "Failed to delete Delete-policy workspace PVC"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } + } + if !deletable_claims.is_empty() { + return Ok(Action::requeue(Duration::from_secs(2))); + } + } else { + tracing::info!( + "KarsSandbox {name} is being deleted — cleaning up namespace {sandbox_ns}" + ); + + // Ephemeral and Delete-policy workspaces retain the historical + // namespace-level cascade. + let ns_api: Api = Api::all(client.clone()); + match ns_api.delete(&sandbox_ns, &DeleteParams::default()).await { + Ok(_) => tracing::info!("Namespace {sandbox_ns} deletion initiated"), + Err(kube::Error::Api(ae)) if ae.code == 404 => { + tracing::info!("Namespace {sandbox_ns} already gone"); + } + Err(e) => { + tracing::error!(error = %e, "Failed to delete namespace {sandbox_ns}"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } } // Clean up the spawner ClusterRoleBinding @@ -493,9 +1295,13 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox_ns); + let namespace_claims = claim_api.list(&ListParams::default()).await?.items; + let requested_existing_claim = spec + .storage + .as_ref() + .and_then(|storage| storage.workspace.as_ref()) + .and_then(|workspace| workspace.existing_claim.as_deref()); + let current_workspace_claim = if spec.suspended.unwrap_or(false) { + Api::::namespaced(client.clone(), &sandbox_ns) + .get_opt(&name) + .await? + .as_ref() + .and_then(deployment_workspace_claim) + .map(str::to_string) + } else { + None + }; + if let Err(message) = validate_namespace_claim_reuse( + &namespace_claims, + requested_existing_claim, + current_workspace_claim.as_deref(), + sandbox.metadata.uid.as_deref(), + ) { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + + // ── Step 1b: Reconcile per-sandbox workspace storage ──────────────── + // Backward-compatible sandboxes keep the historical emptyDir. A dynamic + // workspace produces a PVC, while existingClaim only changes the pod + // volume and remains entirely operator-owned. + let workspace_storage_plan = build_workspace_storage_plan( + &name, + &sandbox_ns, + spec.storage.as_ref(), + sandbox.metadata.uid.as_deref(), + ); + let workspace_claim = if let Some(claim) = workspace_storage_plan.claim.as_ref() { + let claim: PersistentVolumeClaim = serde_json::from_value(claim.clone())?; + let claim_name = claim.name_any(); + if let Some(existing) = claim_api.get_opt(&claim_name).await? + && let Err(message) = + validate_dynamic_claim_provenance(&existing, sandbox.metadata.uid.as_deref()) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + let applied = match claim_api + .patch( + &claim_name, + &PatchParams::apply(crate::field_managers::CLAWSANDBOX).force(), + &Patch::Apply(&claim), + ) + .await + { + Ok(claim) => claim, + Err(kube::Error::Api(error)) if error.code == 422 => { + let message = format!( + "workspace PVC `{claim_name}` update was rejected: {}", + error.message + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::IMMUTABLE_FIELD_CHANGED, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + Err(error) => return Err(error.into()), + }; + Some(applied) + } else if let Some(existing_claim) = spec + .storage + .as_ref() + .and_then(|storage| storage.workspace.as_ref()) + .and_then(|workspace| workspace.existing_claim.as_deref()) + { + match claim_api.get(existing_claim).await { + Ok(claim) => Some(claim), + Err(kube::Error::Api(error)) if error.code == 404 => { + let message = format!( + "workspace PVC `{existing_claim}` not found in namespace `{sandbox_ns}`" + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_NOT_FOUND, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(30))); + } + Err(error) => return Err(error.into()), + } + } else { + None + }; + if let Some(claim) = workspace_claim.as_ref() + && let Err(message) = validate_workspace_claim(claim) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + let workspace_claim_phase = workspace_claim + .as_ref() + .and_then(|claim| claim.status.as_ref()) + .and_then(|status| status.phase.clone()); + + // Validate the OpenClaw bootstrap ConfigMap in the KarsSandbox's own + // namespace, then mirror it into the generated runtime namespace. The + // runtime namespace cannot mount a ConfigMap across namespace boundaries. + let mut workspace_bootstrap_provenance: Option<(String, String)> = None; + if let Some(bootstrap_ref) = runtime_spec + .openclaw + .as_ref() + .and_then(|config| config.workspace.as_ref()) + .and_then(|workspace| workspace.bootstrap_config_map_ref.as_ref()) + { + let source_api: Api = Api::namespaced(client.clone(), &sandbox_self_ns); + let source = match source_api.get(&bootstrap_ref.name).await { + Ok(config_map) => config_map, + Err(kube::Error::Api(error)) if error.code == 404 => { + let message = format!( + "workspace bootstrap ConfigMap `{}` not found in namespace `{}`", + bootstrap_ref.name, sandbox_self_ns + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::reason::BOOTSTRAP_CONFIG_NOT_FOUND, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + Err(error) => return Err(error.into()), + }; + if let Err(message) = validate_workspace_bootstrap_config_map(&source) { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::reason::BOOTSTRAP_INVALID, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + workspace_bootstrap_provenance = Some(( + source.metadata.uid.clone().unwrap_or_default(), + source.metadata.resource_version.clone().unwrap_or_default(), + )); + + if sandbox_self_ns != sandbox_ns { + let mirrored: ConfigMap = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": bootstrap_ref.name, + "namespace": sandbox_ns, + "labels": { + "app.kubernetes.io/managed-by": "kars-controller", + "kars.azure.com/sandbox": name, + "kars.azure.com/artifact": "workspace-bootstrap" + }, + "annotations": { + "kars.azure.com/source-namespace": sandbox_self_ns, + "kars.azure.com/source-resource-version": source.metadata.resource_version + } + }, + "data": source.data + }))?; + let target_api: Api = Api::namespaced(client.clone(), &sandbox_ns); + target_api + .patch( + &bootstrap_ref.name, + &PatchParams::apply(crate::field_managers::CLAWSANDBOX).force(), + &Patch::Apply(&mirrored), + ) + .await?; + } + } + // ── Step 2: Create ServiceAccount with Workload Identity ───────────── let sa_api: Api = Api::namespaced(client.clone(), &sandbox_ns); let sa: ServiceAccount = serde_json::from_value(json!({ @@ -1525,7 +2563,11 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox_ns); + let desired_workspace_claim = workspace_storage_plan + .volume + .pointer("/persistentVolumeClaim/claimName") + .and_then(serde_json::Value::as_str); + let current_deployment = deploy_api.get_opt(&name).await?; + if let Some(current_deployment) = current_deployment.as_ref() + && let Err(message) = validate_workspace_volume_transition( + deployment_workspace_claim(current_deployment), + desired_workspace_claim, + suspended_by_spec, + ) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::IMMUTABLE_FIELD_CHANGED, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } // Token budget values resolved from the InferencePolicy ref above // (hoisted to the top of `reconcile` after S13). 0 = unlimited. @@ -2091,6 +3175,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result::namespaced(client.clone(), &sandbox_ns) + .list(&ListParams::default().labels(&format!("kars.azure.com/sandbox={name}"))) + .await?; + workspace_bootstrap_state(&pods.items) + } else { + WorkspaceBootstrapState::NotConfigured + }; + // ── Step 5: Update status ──────────────────────────────────────────── // Idempotency guard: skip the patch when the desired status already // matches reality. Without this, every reconcile bumps @@ -3273,6 +4408,18 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = allowlist_resolution.conditions.clone(); + extras.extend(workspace_storage_status_conditions( + &sandbox, + if workspace_claim.is_some() { + Some(workspace_claim_phase.as_deref().unwrap_or("Pending")) + } else { + None + }, + )); + extras.extend(workspace_bootstrap_status_conditions( + &sandbox, + &workspace_bootstrap_state, + )); // Phase G P1 #4: stamp Suspended condition when spec.suspended // is true, or surface Suspended=False/Active when there is a @@ -3361,7 +4508,15 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result 0" message: "spec.memoryRef.name must not be empty" + storage: + type: object + description: "Optional per-sandbox persistent storage. Omission preserves the legacy emptyDir workspace." + properties: + workspace: + type: object + description: "Workspace storage mounted at /sandbox." + x-kubernetes-validations: + - rule: "!has(self.existingClaim) || (!has(self.size) && !has(self.storageClassName) && !has(self.accessModes) && !has(self.retainPolicy))" + message: "existingClaim is mutually exclusive with dynamic provisioning fields" + - rule: "!has(self.size) || quantity(self.size).isGreaterThan(quantity('0'))" + message: "workspace size must be a positive Kubernetes quantity" + properties: + existingClaim: + type: string + maxLength: 253 + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + description: "Existing same-namespace PVC managed outside Kars." + size: + type: string + storageClassName: + type: string + accessModes: + type: array + minItems: 1 + maxItems: 1 + items: + type: string + enum: ["ReadWriteOnce", "ReadWriteOncePod"] + retainPolicy: + type: string + enum: ["Retain", "Delete"] agent: type: object description: "Foundry Agent Service configuration — controller creates a prompt agent on reconcile" @@ -691,8 +739,10 @@ spec: namespace, NetworkPolicy, ServiceAccount, and all governance overlay objects are preserved byte-identical, so flipping this field back to - `false` (or removing it) restores the agent - in-place. Default `false`. + `false` (or removing it) restores the agent. Runtime + files survive only when `spec.storage.workspace` + uses a PVC; the legacy `emptyDir` is deleted with + the Pod. Default `false`. Distinct from the `Suspended=True / Reason=OverlayMode` transition, which is induced by diff --git a/deploy/helm/kars/templates/rbac.yaml b/deploy/helm/kars/templates/rbac.yaml index efbf5fb3c..dc7227d8f 100644 --- a/deploy/helm/kars/templates/rbac.yaml +++ b/deploy/helm/kars/templates/rbac.yaml @@ -59,10 +59,10 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list", "create", "update", "patch", "delete"] - # Manage pods, services, configmaps in sandbox namespaces + # Manage pods, services, configmaps, and workspace PVCs in sandbox namespaces - apiGroups: [""] - resources: ["pods", "services", "configmaps", "secrets", "serviceaccounts"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + resources: ["pods", "services", "configmaps", "secrets", "serviceaccounts", "persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Read pod logs (for offload result relay) - apiGroups: [""] resources: ["pods/log"] @@ -70,7 +70,7 @@ rules: # Manage deployments - apiGroups: ["apps"] resources: ["deployments"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Slice 3 of kars-sre — typed actions RolloutRestart targets # StatefulSet / DaemonSet as well. Read+patch is sufficient (we # only ever rollout-restart, never create/delete those kinds). @@ -84,11 +84,11 @@ rules: # KarsEval runs jobs and cronjobs to invoke the conformance runner - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Manage network policies - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Events — Kubernetes ships two Event APIs: the legacy core v1 # Events ("" apiGroup) and the modern events.k8s.io/v1 Events. # The controller writes to events.k8s.io (the kube-rs Recorder diff --git a/docs/api/conditions.md b/docs/api/conditions.md index 6b253c8e9..e666136bb 100644 --- a/docs/api/conditions.md +++ b/docs/api/conditions.md @@ -21,6 +21,8 @@ These types are reused across CRDs. | `Progressing` | Reconciler is making forward progress toward the spec. | | `Degraded` | Object is partially functional but a sub-resource is failing. | | `Suspended` | Operator has paused the object via `spec.suspended: true`. | +| `StorageReady` | Sandbox workspace storage is available to the runtime. | +| `BootstrapReady` | OpenClaw workspace bootstrap init container completed. | `status: True` means the type predicate holds. For `Degraded`, that means the object **is** degraded; for `Ready`, that it **is** ready. @@ -37,6 +39,15 @@ means the object **is** degraded; for `Ready`, that it **is** ready. | `TimedOut` | most kinds | A wait loop hit its budget. | | `SuspendedBySpec` | `KarsSandbox` | `spec.suspended: true`; Deployment scaled to 0. | | `Active` | `KarsSandbox` | Pairs with `Suspended=False` to clear a prior `SuspendedBySpec`. | +| `EmptyDir` | `KarsSandbox` | Workspace uses the backward-compatible ephemeral volume. | +| `ClaimBound` | `KarsSandbox` | Workspace PVC is bound and available. | +| `ClaimPending` | `KarsSandbox` | Workspace PVC has not bound yet; sandbox remains Creating. | +| `ClaimNotFound` | `KarsSandbox` | Referenced `existingClaim` does not exist. | +| `ClaimIncompatible` | `KarsSandbox` | PVC mode, phase, or provenance is incompatible with the sandbox. | +| `ImmutableFieldChanged` | `KarsSandbox` | Kubernetes rejected an unsafe PVC mutation such as changing StorageClass or access mode. | +| `BootstrapConfigNotFound` | `KarsSandbox` | Referenced workspace bootstrap ConfigMap does not exist. | +| `BootstrapInvalid` | `KarsSandbox` | Bootstrap ConfigMap contains unsupported files or binary data. | +| `BootstrapFailed` | `KarsSandbox` | Bootstrap init container exited unsuccessfully. | ## KarsSandbox @@ -50,6 +61,8 @@ end-to-end runtime. | `Degraded` | True/False | `AuthMisconfigured`, `MemoryStoreMissing`, `FailedClosed` | | `Suspended` | True/False | `SuspendedBySpec`, `Active` | | `RuntimeReady` | True/False | `AdapterMissing` (Falsey when the runtime adapter isn't wired) | +| `StorageReady` | True/False | `EmptyDir`, `ClaimBound`, `ClaimPending` | +| `BootstrapReady` | True/False | `Reconciled`, `Creating`, `BootstrapFailed` | | `AllowlistVerified` | True/False | `Verified`, `Unsigned`, `FailedClosed` | | `AllowlistAuthoritative` | True/False | `Inline`, `Verified`, `StaleLKG`, `FailedClosed`, `InlineDiffersFromArtifact` | | `AllowlistDrift` | True/False | `InlineDiffersFromArtifact`, `InlineCleared` | diff --git a/docs/api/crd-reference.md b/docs/api/crd-reference.md index 3f7c49029..840fe6a1b 100644 --- a/docs/api/crd-reference.md +++ b/docs/api/crd-reference.md @@ -183,6 +183,16 @@ spec: # LangGraph | Anthropic | PydanticAi | BYO openclaw: # block name matches `kind` (CEL-validated) image: karsacr.azurecr.io/kars-runtime-openclaw:latest + workspace: + bootstrapConfigMapRef: + name: my-agent-workspace # AGENTS/SOUL/HEARTBEAT/TOOLS/USER.md only + overwritePolicy: IfMissing # IfMissing (default) | Always + storage: + workspace: + size: 10Gi + storageClassName: managed-csi + accessModes: [ReadWriteOnce] + retainPolicy: Retain # Retain (default) | Delete inferenceRef: name: shared-inference # required: sibling InferencePolicy memoryRef: # optional: sibling KarsMemory @@ -225,6 +235,9 @@ status: | Field | Type | Purpose | |---|---|---| | `spec.memoryRef.name` | LocalObjectRef | Bind to a sibling `KarsMemory` (same namespace). | +| `spec.storage.workspace` | object | Optional PVC-backed `/sandbox`. Omit for legacy `emptyDir`; set `existingClaim` to attach an explicitly selected PVC, or set dynamic fields (`size`, `storageClassName`, `accessModes`, `retainPolicy`). | +| `spec.runtime.openclaw.workspace.bootstrapConfigMapRef.name` | LocalObjectRef | Same-namespace ConfigMap containing only `AGENTS.md`, `SOUL.md`, `HEARTBEAT.md`, `TOOLS.md`, or `USER.md`. The controller mirrors it into the runtime namespace. | +| `spec.runtime.openclaw.workspace.overwritePolicy` | enum | `IfMissing` (default) preserves files already on the PVC; `Always` reapplies declared bootstrap files on every Pod start. | | `spec.sandbox` | object | Isolation primitives — `isolation` (`standard` \| `enhanced`, default `enhanced`), `seccompProfile` (default `kars-strict`), `writablePaths` (default `[/sandbox, /tmp]`). | | `spec.networkPolicy` | object | Baseline egress allowlist. Defaults `defaultDeny: true`, `egressMode: Learn`. | | `spec.governance.enabled` | bool | **Defaults to `true`.** Turn on AGT governance — router guardrails are always on regardless; this gates AGT trust/audit + creates the per-sandbox Service on `:8443` (required for InferencePolicy enforcement + cross-agent mesh DNS). Set to `false` to opt out. | @@ -234,7 +247,7 @@ status: | `spec.governance.registryMode` | string | `local` (default) or `global`. Global enables cross-cluster mesh + handoff tools. | | `spec.governance.trustedPeers` | string | Pre-seeded `"name:AMID,..."` peers — used by sub-agents to auto-trust the spawning parent. | | `spec.a2a` | object | Inbound A2A 1.0.0 ingress configuration. Default off; see [A2A gateway](../architecture/a2a-gateway.md). | -| `spec.suspended` | bool | Operator-driven graceful pause. When `true`, the controller scales the Deployment to `replicas: 0` and stamps `Suspended=True / SuspendedBySpec` without tearing down namespace, NetworkPolicy, or federated credentials. | +| `spec.suspended` | bool | Operator-driven graceful pause. When `true`, the controller scales the Deployment to `replicas: 0`. Runtime files survive only when `spec.storage.workspace` uses a PVC; the legacy `emptyDir` is deleted with the Pod. | **Status** @@ -248,6 +261,15 @@ status: | `status.observedGeneration` | `metadata.generation` that produced this status. Compare against `metadata.generation` to detect stale observations. | | `status.conditions[]` | The full condition chain — every reason emitted by the controller is enumerated in [`docs/api/conditions.md`](conditions.md). | +`StorageReady=True/ClaimBound` means a persistent workspace is bound. While a +claim is pending, the Deployment exists so `WaitForFirstConsumer` storage can +bind, but the sandbox remains `phase=Creating`, `Ready=False`, and +`StorageReady=False/ClaimPending`. + +For a retained workspace, deleting the `KarsSandbox` removes Kars-managed +workloads but leaves the generated namespace and PVC. Reattach it explicitly +with `existingClaim`; a new same-name CR cannot silently adopt a retained claim. + ### `spec.runtime.hermes` (`HermesConfig`) {#hermesconfig} Runtime-kind config block used when `spec.runtime.kind: Hermes`. All fields are optional — defaults give a working smoke-test agent on first boot. diff --git a/docs/api/lifecycle.md b/docs/api/lifecycle.md index 8c4bfb148..4113a18d3 100644 --- a/docs/api/lifecycle.md +++ b/docs/api/lifecycle.md @@ -28,7 +28,7 @@ flowchart LR CLI["kars CLI
or GitOps / kubectl"] CRD[("CRD
(12 kinds)")] Ctrl["kars-controller
(kube-rs)"] - Art[("Cluster artifacts
Namespace · ServiceAccount · NetworkPolicy
Deployment · Service · ConfigMap · Secret
FederatedIdentityCredential")] + Art[("Cluster artifacts
Namespace · ServiceAccount · NetworkPolicy
Deployment · Service · ConfigMap · Secret · optional PVC
FederatedIdentityCredential")] Runtime["Runtime data plane
inference-router · A2A gateway · sandbox pod"] CLI -->|writes| CRD @@ -125,7 +125,7 @@ Every CLI command is a thin wrapper around `kubectl apply`. The CLI does no orch |---|---|---|---| | `kars up` | `KarsSandbox` (one or more) | Tenant namespace + everything inside it | The agent pod itself | | `kars add ` | `KarsSandbox` | Same as above | Same | -| `kars destroy ` | Deletes `KarsSandbox` | Cascades via finalizer to delete the namespace + federated credential | — | +| `kars destroy ` | Deletes `KarsSandbox` | Ephemeral/Delete storage: delete namespace. Retain/existing storage: delete labelled workloads but retain namespace + PVC. Always deletes federated credential. | — | | `kars inferencepolicy apply` | `InferencePolicy` | `ConfigMap` `inferencepolicy--profile` | Inference router (`/v1/chat`, `/v1/responses`) | | `kars toolpolicy apply` | `ToolPolicy` | `ConfigMap` `toolpolicy--profile` | Inference router (every tool dispatch) | | `kars mcp add` | `McpServer` | `Secret` `mcp--signing` (Ed25519 keypair)
`ConfigMap` `mcp--jwks` (when `productionMode=true`) | Inference router (`/mcp` proxy — multi-issuer OAuth verifier + namespaced `{server}.{tool}` dispatch) | @@ -368,7 +368,7 @@ Every reconciler installs a finalizer on first reconcile. This blocks Kubernetes | CRD | Finalizer | Cleanup work | |---|---|---| -| `KarsSandbox` | `kars.azure.com/namespace-cleanup` | Delete the tenant namespace (cascades to all resources), delete spawner ClusterRoleBinding, **delete federated credential**, release pairing slot, then remove finalizer. | +| `KarsSandbox` | `kars.azure.com/namespace-cleanup` | With ephemeral or `retainPolicy: Delete` storage, delete the tenant namespace. With `Retain` or `existingClaim`, delete labelled Kars workloads but preserve namespace and PVC (namespaced owner references cannot cross from the source CR namespace). Then delete spawner ClusterRoleBinding and federated credential, release pairing slot, and remove the finalizer. | | Compile-pattern CRDs | `kars.azure.com/-cleanup` | Delete the produced `ConfigMap` (and `Secret`, where present), then remove finalizer. | The federated-credential delete is the one cleanup that crosses the cluster boundary into Microsoft Graph. See `controller/src/fedcred_reaper.rs` for the orphan-collector that backstops force-delete and pre-finalizer CRs. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3908642f1..3fbda6c7f 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -413,6 +413,12 @@ kars add [options] | `--agent-instructions ` | — | System prompt for the Foundry agent | | `--agent-tools ` | — | Foundry tools: `file_search,web_search,code_interpreter` (comma-separated) | | `--image ` | — | Custom sandbox image (default: from Helm values) | +| `--workspace-storage ` | — | Create a persistent workspace PVC, for example `10Gi` | +| `--workspace-storage-class ` | cluster default | StorageClass for the generated workspace PVC | +| `--workspace-existing-claim ` | — | Attach an existing PVC in `kars-`; mutually exclusive with generated storage | +| `--workspace-retain-policy ` | `Retain` | Generated PVC deletion policy: `Retain` or `Delete` | +| `--workspace-bootstrap ` | — | OpenClaw-only same-namespace ConfigMap for declarative workspace files | +| `--workspace-overwrite ` | `IfMissing` | Bootstrap policy: `IfMissing` or `Always` | | `--governance` | `true` | Enable AGT governance (tool policy, trust, audit) | | `--no-governance` | — | Disable AGT governance | | `--trust-threshold ` | `500` | AGT trust threshold (0–1000) | @@ -444,6 +450,14 @@ kars add researcher --model gpt-4.1 --token-budget-daily 100000 # Add a Telegram-connected agent with enhanced isolation kars add support-bot --channels telegram --telegram-token $TOKEN --isolation enhanced +# Add an OpenClaw agent with a retained 20Gi workspace and bootstrap files +kars add teaching-agent --workspace-storage 20Gi \ + --workspace-storage-class managed-csi \ + --workspace-bootstrap teaching-agent-workspace + +# Explicitly recover a retained workspace +kars add teaching-agent --workspace-existing-claim teaching-agent-workspace + # Add a BYO-runtime agent kars add my-agent --runtime byo --byo-image myacr.azurecr.io/my-agent:latest diff --git a/docs/getting-started.md b/docs/getting-started.md index bc114b875..bb55d9a4b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -315,6 +315,19 @@ kars add another-agent --runtime LangGraph --model gpt-4.1 `kars add` reuses the existing AKS cluster and Foundry project — only the pod is new. See **[CLI reference](cli-reference.md)** for the full surface. +By default `/sandbox` is ephemeral. Enable a retained workspace when sessions +and files must survive Pod recreation or `spec.suspended` scale-to-zero: + +```bash +kars add teaching-agent --runtime openclaw --model gpt-4.1 \ + --workspace-storage 20Gi \ + --workspace-storage-class managed-csi +``` + +This creates one PVC in `kars-teaching-agent`, which incurs storage charges +while retained. A PVC is not a backup; configure snapshots and regional +placement through the selected StorageClass and your platform policy. + ### 2.5a Try a Hermes-runtime sandbox instead The same `kars add` works for [Hermes](https://github.com/NousResearch/hermes-agent), a channels-first agent harness with native MCP support — useful when you want a Telegram or Slack-driven agent without writing the integration: diff --git a/docs/plans/2026-08-07-persistent-workspace-design.md b/docs/plans/2026-08-07-persistent-workspace-design.md new file mode 100644 index 000000000..72fbe8054 --- /dev/null +++ b/docs/plans/2026-08-07-persistent-workspace-design.md @@ -0,0 +1,616 @@ +# Kars Persistent Workspace Design + +**Status:** Approved design + +**Date:** 2026-08-07 + +**Scope:** Per-sandbox persistent workspace storage and declarative OpenClaw workspace bootstrap files. +**Out of scope:** Feishu channel integration, message-triggered wake-up, cross-cluster volume migration, multi-replica RWX, online volume expansion. + +## 1. Problem + +Every `KarsSandbox` runtime currently mounts `/sandbox` from an `emptyDir` volume. OpenClaw stores its workspace, sessions, runtime configuration, pairing state, dynamic bindings, local memory and AgentMesh identity below this directory. The data survives a container restart inside the same Pod but is lost when the Pod is recreated, the Deployment is rolled out, or a suspended sandbox scales from zero back to one. + +The Controller also has no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source. OpenClaw's entrypoint currently rewrites Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. Although `runtime.openclaw.config` exists in the CRD schema, the OpenClaw deployment planner does not consume it. + +## 2. Goals + +1. Give each `KarsSandbox` an optional, dedicated persistent volume mounted at `/sandbox`. +2. Preserve sessions, workspace files, pairing state, dynamic bindings and runtime identity across Pod recreation and scale-to-zero. +3. Let an operator initialize selected OpenClaw workspace Markdown files from a same-namespace ConfigMap. +4. Preserve user changes by default after the initial bootstrap. +5. Retain dynamically provisioned storage by default when a `KarsSandbox` is deleted. +6. Keep sensitive credentials in Kubernetes Secrets and outside workspace ConfigMaps. +7. Maintain backward compatibility: sandboxes without the new storage block continue using `emptyDir`. +8. Surface storage readiness and bootstrap failures through explicit status conditions. + +## 3. Non-goals + +The first version does not: + +- add Feishu or other channel configuration; +- receive IM messages while a sandbox is scaled to zero; +- define a wake gateway or durable message queue; +- share one volume among multiple simultaneously running runtime replicas; +- migrate data between clusters, regions or storage classes; +- continuously reconcile file contents after bootstrap; +- use `runtime.openclaw.config` as an arbitrary pass-through to `openclaw.json`; +- provide backup, snapshot or disaster-recovery orchestration; +- persist `/tmp`; +- replace Foundry Memory Store or another external semantic-memory service. + +## 4. Design principles + +### 4.1 Separate desired configuration from mutable state + +- The CR selects storage behavior and references declarative inputs. +- A ConfigMap contains non-sensitive bootstrap files. +- A Secret contains credentials. +- The PVC contains runtime-mutated state. + +### 4.2 Fail closed on ambiguous storage + +The Controller must not start a sandbox against an unexpected or incompatible claim. Invalid combinations fail admission where possible; missing claims and incompatible claim modes produce a non-running Deployment and an explicit condition. + +### 4.3 Preserve data by default + +Dynamically created claims default to `Retain`. Deleting the `KarsSandbox` removes workload resources but leaves the PVC. Destructive deletion requires an explicit `Delete` policy. + +### 4.4 Bootstrap is initialization, not synchronization + +The default `IfMissing` policy copies each managed file only when the destination does not exist. A ConfigMap update does not overwrite files already modified on the PVC. + +## 5. Proposed API + +### 5.1 KarsSandbox storage + +Add the following optional block to `KarsSandboxSpec`: + +```yaml +apiVersion: kars.azure.com/v1alpha1 +kind: KarsSandbox +metadata: + name: teaching-agent + namespace: kars-teaching-agent +spec: + storage: + workspace: + size: 10Gi + storageClassName: managed-csi + accessModes: + - ReadWriteOnce + retainPolicy: Retain + runtime: + kind: OpenClaw + openclaw: + workspace: + bootstrapConfigMapRef: + name: teaching-agent-workspace + overwritePolicy: IfMissing + inferenceRef: + name: teaching-agent-inference +``` + +Rust schema: + +```rust +pub struct KarsSandboxSpec { + // existing fields... + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, +} + +pub struct SandboxStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, +} + +pub struct WorkspaceStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub existing_claim: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_class_name: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub access_modes: Option>, + + #[serde(default)] + pub retain_policy: WorkspaceRetainPolicy, +} + +pub enum PersistentVolumeAccessMode { + ReadWriteOnce, + ReadWriteOncePod, +} + +pub enum WorkspaceRetainPolicy { + Retain, + Delete, +} +``` + +Defaults for dynamically provisioned storage: + +| Field | Default | +|---|---| +| `size` | `10Gi` | +| `accessModes` | `[ReadWriteOnce]` | +| `retainPolicy` | `Retain` | +| `storageClassName` | unset; use the cluster default StorageClass | +| generated claim name | `-workspace` | + +When `spec.storage.workspace` is omitted, the Controller preserves the current `emptyDir` behavior. + +### 5.2 Existing claims + +Advanced users may provide a same-namespace claim: + +```yaml +spec: + storage: + workspace: + existingClaim: teaching-agent-imported-workspace +``` + +Rules: + +- `existingClaim` is mutually exclusive with `size`, `storageClassName`, `accessModes`, and `retainPolicy`. +- The claim must exist in the sandbox namespace. +- The Controller never creates, resizes, mutates, adopts or deletes an existing claim. +- The existing claim must advertise `ReadWriteOnce` or `ReadWriteOncePod`. +- A Bound claim is required before the Deployment scales above zero. + +### 5.3 OpenClaw workspace bootstrap + +Extend `OpenClawConfig` with a typed workspace block rather than adding more unstructured fields to `config`: + +```rust +pub struct OpenClawConfig { + pub version: Option, + pub image: Option, + pub config: Option, + pub workspace: Option, + pub extra_env: Option>, +} + +pub struct OpenClawWorkspaceSpec { + pub bootstrap_config_map_ref: Option, + #[serde(default)] + pub overwrite_policy: WorkspaceOverwritePolicy, +} + +pub enum WorkspaceOverwritePolicy { + IfMissing, + Always, +} +``` + +Default `overwritePolicy` is `IfMissing`. + +The reference is same-namespace only. Cross-namespace references are not allowed. + +### 5.4 Allowed bootstrap files + +The bootstrap ConfigMap may contain only these keys: + +- `AGENTS.md` +- `SOUL.md` +- `HEARTBEAT.md` +- `TOOLS.md` +- `USER.md` + +The following are explicitly forbidden: + +- `MEMORY.md` +- `openclaw.json` +- credentials or token files; +- session database files; +- pairing or binding state; +- AgentMesh identity or prekeys; +- arbitrary paths, nested keys, symlinks or executable files. + +`MEMORY.md` is excluded because OpenClaw and Kars mutate it during normal operation. Declaratively overwriting it risks erasing inbox state, Foundry discovery context and user memory. + +### 5.5 ConfigMap example + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: teaching-agent-workspace + namespace: kars-teaching-agent +data: + SOUL.md: | + # Soul + + You are a teaching assistant for the university's internal learning platform. + Protect student data and cite sources when making factual claims. + + HEARTBEAT.md: | + # Heartbeat + + - Check for unfinished tasks. + - Check the AgentMesh inbox. + - If there is no work, return HEARTBEAT_OK. + + AGENTS.md: | + # Agent Instructions + + Follow the institution's teaching and privacy policies. +``` + +## 6. Admission and validation + +The generated CRD and Helm CRD must enforce these constraints with schema and CEL where possible: + +1. `existingClaim` must not appear with dynamic provisioning fields. +2. `size` must parse as a positive Kubernetes quantity. +3. `accessModes` must contain one value and that value must be `ReadWriteOnce` or `ReadWriteOncePod` in v1. +4. `retainPolicy` must be `Retain` or `Delete`. +5. `runtime.openclaw.workspace` is valid only when `runtime.kind == OpenClaw`. +6. `bootstrapConfigMapRef.name` must be a valid Kubernetes object name. +7. `overwritePolicy` must be `IfMissing` or `Always`. +8. Unknown bootstrap file keys cause reconciliation failure; they are not silently ignored. + +Dynamic workspace PVCs do not support `ReadWriteMany` in v1 because each `KarsSandbox` has one active runtime replica and the security model assumes a private per-agent filesystem. + +## 7. Reconciliation + +### 7.1 Dynamic PVC creation + +For a dynamic workspace, the Controller creates a PVC named `-workspace` in the sandbox namespace. + +Required metadata: + +```yaml +metadata: + labels: + kars.azure.com/managed: "true" + kars.azure.com/sandbox: teaching-agent + kars.azure.com/storage-role: workspace + annotations: + kars.azure.com/retain-policy: Retain +``` + +The source `KarsSandbox` and generated PVC live in different namespaces, so a +namespaced owner reference would be invalid. Both policies use labels plus a +`kars.azure.com/sandbox-uid` provenance annotation. `Delete` relies on deletion +of the generated sandbox namespace; `Retain` preserves that namespace and its +PVC while removing labelled workload resources. + +### 7.2 Immutable fields and drift + +The Controller reconciles desired PVC shape without attempting invalid in-place mutations. + +- Increasing `size` may be applied only when the StorageClass allows expansion. +- Decreasing `size` is rejected. +- Changing `storageClassName` is rejected after creation. +- Changing `accessModes` is rejected after creation. +- Changing from a generated claim to `existingClaim`, or the reverse, is rejected while either claim contains active state. + +These errors set `StorageReady=False` and leave the last-known-good Deployment and claim untouched. + +### 7.3 Deployment volume + +When storage is enabled: + +```yaml +volumes: + - name: sandbox-data + persistentVolumeClaim: + claimName: teaching-agent-workspace +``` + +The runtime container continues mounting: + +```yaml +volumeMounts: + - name: sandbox-data + mountPath: /sandbox +``` + +`/tmp` remains a memory-backed `emptyDir`. + +The inference-router does not receive access to the workspace PVC unless a separately reviewed feature requires it. The least-privilege boundary remains unchanged. + +### 7.4 Bootstrap ConfigMap mount + +When a bootstrap ConfigMap is configured, mount it read-only at: + +```text +/etc/kars/workspace-bootstrap +``` + +Do not mount the ConfigMap directly over the writable OpenClaw workspace. + +### 7.5 Bootstrap init container + +Add an init container before the runtime starts. It mounts: + +- `sandbox-data` at `/sandbox`; +- the bootstrap ConfigMap at `/bootstrap`, read-only. + +Its responsibilities are: + +1. Create `/sandbox/.openclaw/workspace` with UID/GID `1000:1000`. +2. Validate that every source entry is an allowed regular file. +3. Reject symlinks and path traversal. +4. Copy via a temporary file in the destination directory. +5. Set file ownership to `1000:1000` and mode `0640`. +6. Atomically rename the temporary file to the destination. +7. Under `IfMissing`, skip existing destinations. +8. Under `Always`, replace allowed destination files. +9. Write a non-sensitive manifest to `/sandbox/.kars/bootstrap-state.json` containing ConfigMap UID, resourceVersion, policy, filenames and SHA-256 digests. + +The init container must not log file contents. + +### 7.6 Default Kars templates + +If no bootstrap ConfigMap is configured, the existing Kars default `AGENTS.md`, `SOUL.md`, and `TOOLS.md` remain available, but entrypoint behavior changes from unconditional overwrite to create-if-missing. + +This preserves current first-run behavior while preventing Pod restarts from replacing user edits on a PVC. + +`HEARTBEAT.md` has no Kars default and is not created unless supplied by the operator or OpenClaw itself. + +### 7.7 Suspended sandboxes + +When `spec.suspended: true`: + +- the Deployment remains at zero replicas; +- the PVC and bootstrap ConfigMap reference remain reconciled; +- no init container runs until the sandbox resumes; +- `StorageReady` can still become true based on PVC state; +- resuming mounts the same claim and restores runtime state. + +The CRD documentation must stop describing state preservation for sandboxes that still use `emptyDir`. The guarantee applies only when persistent workspace storage is configured. + +## 8. Deletion semantics + +### 8.1 Retain + +For `retainPolicy: Retain`: + +1. Delete the Deployment and normal sandbox-owned resources. +2. Leave the PVC and backing PV intact. +3. Add an event and final status message naming the retained claim before the CR disappears. +4. Do not remove the PVC protection finalizer. +5. Do not automatically expose the retained claim to another sandbox. + +A future sandbox may use the retained data only by explicitly setting `existingClaim`. + +### 8.2 Delete + +For `retainPolicy: Delete`: + +- the finalizer deletes the generated sandbox namespace; +- namespace cascading deletion removes the claim; +- backing-volume deletion follows the StorageClass/PV reclaim policy. + +The CLI must show a destructive warning before creating or updating a sandbox to `Delete`. + +### 8.3 Namespace deletion + +Kubernetes namespace deletion can delete both Retain and Delete claims. `retainPolicy: Retain` protects against deletion of the `KarsSandbox`, not deletion of its namespace. Documentation and CLI output must state this explicitly. + +## 9. Status and events + +Add a `StorageReady` condition to `KarsSandbox.status.conditions`. + +| Status | Reason | Meaning | +|---|---|---| +| `True` | `EmptyDir` | Persistence not requested; current ephemeral behavior is active. | +| `True` | `ClaimBound` | Workspace PVC exists and is Bound. | +| `False` | `ClaimPending` | PVC exists but is not Bound. | +| `False` | `ClaimNotFound` | Referenced existing claim does not exist. | +| `False` | `ClaimIncompatible` | Access mode or claim shape is unsupported. | +| `False` | `ImmutableFieldChanged` | Requested storage mutation cannot be applied safely. | +| `False` | `BootstrapConfigNotFound` | Referenced ConfigMap does not exist. | +| `False` | `BootstrapInvalid` | ConfigMap contains unsupported or unsafe entries. | +| `False` | `BootstrapFailed` | Init container failed to initialize the workspace. | + +`Ready=True` requires `StorageReady=True` when persistent storage or bootstrap is configured. + +The Controller emits Kubernetes Events for claim creation, retention, incompatible mutation, missing bootstrap ConfigMap and bootstrap failure. + +## 10. Security + +1. ConfigMap data is non-sensitive. Admission documentation prohibits secrets in workspace bootstrap files. +2. Credentials remain in `-credentials` and are injected through `envFrom` or mounted Secret files. +3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials, service account token or network access. +4. Bootstrap source and destination paths are fixed; user-controlled path fields are not supported. +5. Symlinks are rejected at both source and destination. +6. Atomic writes prevent partially initialized files. +7. The runtime remains UID 1000 with a read-only root filesystem. +8. PVCs are per sandbox and same namespace. Cross-namespace claim references are impossible. +9. A retained PVC is not automatically adopted based only on labels; adoption requires an explicit `existingClaim` name. +10. Backup encryption, StorageClass encryption and customer-managed keys remain operator responsibilities and must be documented. + +## 11. CLI behavior + +Add storage flags to `kars add`: + +```text +--workspace-storage Enable a generated workspace PVC, e.g. 10Gi +--workspace-storage-class Select a StorageClass +--workspace-existing-claim Use a pre-created same-namespace PVC +--workspace-retain-policy Retain|Delete; default Retain +--workspace-bootstrap Initialize OpenClaw workspace files +--workspace-overwrite IfMissing|Always; default IfMissing +``` + +Examples: + +```bash +kars add teaching-agent \ + --workspace-storage 20Gi \ + --workspace-storage-class managed-csi \ + --workspace-bootstrap teaching-agent-workspace +``` + +```bash +kars add teaching-agent \ + --workspace-existing-claim teaching-agent-workspace +``` + +CLI output must show: + +- claim name; +- storage class and requested size; +- retain policy; +- bootstrap ConfigMap and overwrite policy; +- a warning when persistence is omitted; +- a destructive warning for `retainPolicy=Delete`. + +## 12. Backward compatibility and migration + +### 12.1 Existing sandboxes + +Existing CRs remain valid. When `spec.storage.workspace` is absent, the Controller continues producing `emptyDir`. + +No automatic migration occurs because copying a live workspace requires coordination and can produce inconsistent sessions. + +### 12.2 Opt-in migration from emptyDir + +A safe migration procedure is: + +1. Suspend the sandbox. +2. Create the desired PVC. +3. Copy data from a backup or an explicitly captured workspace archive into the claim. +4. Patch the CR to use `existingClaim` or dynamic storage. +5. Resume the sandbox. +6. Verify sessions, workspace files and AgentMesh identity. + +Because `emptyDir` disappears when the Pod is scaled to zero, users must capture data before suspension. The CLI should eventually offer an export/import workflow, but it is outside this spec. + +### 12.3 Entrypoint migration + +Changing default files from always-overwrite to create-if-missing changes restart behavior intentionally. On the first persistent-storage rollout: + +- existing `emptyDir` sandboxes still receive defaults on every new Pod because the directory starts empty; +- persistent sandboxes retain prior edits; +- `systemPromptOverride` remains the authoritative Kars security/welcome instruction unless a later spec explicitly exposes it. + +## 13. Testing + +### 13.1 CRD and schema tests + +Verify: + +- valid dynamic workspace storage is accepted; +- valid `existingClaim` is accepted; +- mixed existing/dynamic fields are rejected; +- invalid access modes are rejected; +- invalid retain and overwrite policies are rejected; +- OpenClaw workspace config is rejected for non-OpenClaw runtimes; +- old CRs without storage remain valid. + +### 13.2 Controller unit tests + +Verify generated resources for: + +- omitted storage produces `emptyDir`; +- dynamic storage produces the expected PVC and claim mount; +- both policies omit cross-namespace owner references and record sandbox UID provenance; +- Delete removes the namespace; Retain preserves namespace + PVC; +- existing claims do not produce a PVC object; +- bootstrap produces ConfigMap volume, mount and init container; +- Router does not mount the workspace claim; +- suspended sandboxes retain PVC reconciliation with replicas zero; +- immutable changes set the expected condition. + +### 13.3 Bootstrap tests + +Verify: + +- `IfMissing` initializes absent files; +- `IfMissing` preserves modified files; +- `Always` replaces allowed files; +- forbidden filenames fail; +- symlink source and destination attacks fail; +- file contents never appear in logs; +- ownership and mode are correct; +- manifest digests match initialized files; +- an interrupted copy cannot leave a partial destination. + +### 13.4 End-to-end tests + +A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath-backed PVC, must prove: + +1. Create a sandbox with persistent workspace and bootstrap ConfigMap. +2. Wait for `StorageReady=True/ClaimBound` and `Ready=True`. +3. Verify `SOUL.md` and `HEARTBEAT.md` exist. +4. Modify `SOUL.md` and create representative session/workspace state through a legitimate runtime-facing flow. +5. Delete the Pod. +6. Verify the replacement Pod sees the modified file and state. +7. Set `spec.suspended=true`, wait for zero replicas, then resume. +8. Verify state remains. +9. Delete a Retain sandbox and verify the PVC remains. +10. Create a new sandbox with `existingClaim` and verify explicit recovery. +11. Delete a Delete-policy sandbox and verify its claim is removed. + +Do not use `kubectl exec` into the agent container in AKS E2E because the validating admission policy correctly blocks that path. Use an approved test runtime, init-container evidence, `kars connect`, or a purpose-built test probe. + +## 14. Observability + +Add metrics: + +```text +kars_workspace_storage_reconcile_total{result,mode} +kars_workspace_storage_ready{sandbox,mode} +kars_workspace_storage_requested_bytes{sandbox} +kars_workspace_bootstrap_total{result,policy} +kars_workspace_bootstrap_files_total{result,policy} +``` + +Do not label metrics with PVC UID, ConfigMap content, user identifiers or filenames if that creates unbounded cardinality. + +Log claim names, mode, condition reason and bootstrap resourceVersion. Never log workspace file contents or Secret values. + +## 15. Documentation updates + +Implementation must update: + +- `docs/api/crd-reference.md` with storage and OpenClaw workspace fields; +- `docs/api/lifecycle.md` with PVC creation, suspension and deletion behavior; +- `docs/runtimes/CONTRACT.md` to distinguish ephemeral and persistent `/sandbox`; +- `docs/security.md` with storage trust boundary and encryption responsibilities; +- `docs/cli-reference.md` with new flags; +- `docs/getting-started.md` with persistence opt-in and cost warning; +- existing CRD comments that currently claim suspension restores state without qualifying storage mode. + +## 16. Acceptance criteria + +The feature is complete when all of the following are true: + +1. An old `KarsSandbox` without storage still runs with `emptyDir`. +2. A sandbox with dynamic storage gets a unique Bound PVC mounted at `/sandbox`. +3. Pod recreation and scale-to-zero preserve OpenClaw workspace and runtime state. +4. A same-namespace existing claim can be explicitly attached without Controller adoption or deletion. +5. Retain is the default and deleting the CR leaves the claim intact. +6. Delete is explicit and removes the generated claim through namespace cascading deletion. +7. A bootstrap ConfigMap initializes only the allowed files. +8. `IfMissing` preserves runtime/user edits across restart. +9. `Always` deterministically reapplies operator content. +10. `MEMORY.md`, credentials and runtime databases cannot be supplied through bootstrap. +11. Missing/incompatible claims and invalid bootstrap data prevent false `Ready=True` and expose actionable conditions. +12. No Secret or workspace content is emitted to logs or status. +13. Controller, CRD, CLI and E2E tests cover the behaviors above. +14. Documentation no longer claims state survives suspension when `/sandbox` is ephemeral. + +## 17. Future extensions + +Potential follow-up specs may add: + +- Feishu and other per-sandbox channel configuration; +- a wake gateway and durable message queue for scale-from-zero; +- volume snapshots, backup and restore; +- controlled workspace export/import; +- storage quota metrics and alerts; +- RWX and active/passive failover; +- per-file bootstrap policy; +- OCI-based signed workspace bundles; +- explicit migration jobs between claims or storage classes. diff --git a/docs/plans/2026-08-07-persistent-workspace-implementation.md b/docs/plans/2026-08-07-persistent-workspace-implementation.md new file mode 100644 index 000000000..2919ee44f --- /dev/null +++ b/docs/plans/2026-08-07-persistent-workspace-implementation.md @@ -0,0 +1,130 @@ +# Persistent Workspace Implementation Plan + +**Status:** Implemented on `feature/persistent-workspace`. + +Implementation notes: + +- Storage/bootstrap pure helpers remain in `controller/src/reconciler/mod.rs` + with focused tests in `controller/src/reconciler/tests.rs`; a separate + `workspace_storage.rs` module was not introduced because the helpers share + the reconciler's status and finalizer vocabulary. +- The bootstrap implementation is the image-baked + `sandbox-images/openclaw/workspace-bootstrap.sh`, tested by executing it + against temporary filesystems (including file and directory symlink attacks). +- Cluster-facing validation uses controller tests, Helm/Rust schema parity, + live API-server CRD dry-run, and generated-manifest checks. A destructive + persistence lifecycle E2E was not added to `tests/e2e/run.sh`; that suite + requires a CSI provisioner and is tracked as follow-up coverage. + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add opt-in per-sandbox persistent `/sandbox` storage and declarative OpenClaw workspace bootstrap files while preserving the existing `emptyDir` default. + +**Architecture:** Typed CRD fields describe dynamic or existing workspace claims and an optional same-namespace bootstrap ConfigMap. The Controller builds a pure workspace plan, reconciles storage before the Deployment, mounts the resulting volume only into the runtime container, and initializes approved Markdown files through a least-privilege init container. Storage readiness is surfaced independently and gates overall readiness. + +**Tech Stack:** Rust 2024, kube-rs, k8s-openapi, schemars, Kubernetes PVC/ConfigMap/Deployment APIs, Helm CRD OpenAPI/CEL, shell entrypoint tests. + +--- + +### Task 1: CRD storage and workspace schema + +**Files:** +- Modify: `controller/src/crd.rs` +- Modify: `deploy/helm/kars/templates/crd.yaml` +- Test: `controller/src/crd.rs` + +1. Add failing serialization/default tests for dynamic storage, existing claims and OpenClaw bootstrap configuration. +2. Add typed Rust schema with `Retain`, `ReadWriteOnce`, and `IfMissing` defaults. +3. Add matching Helm OpenAPI fields and CEL mutual-exclusion validation. +4. Run focused controller CRD tests and schema diagnostics. + +### Task 2: Pure workspace resource planning + +**Files:** +- Create: `controller/src/reconciler/workspace_storage.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` + +1. Add failing tests for ephemeral, dynamically provisioned and existing-claim plans. +2. Implement a pure planner that returns the `sandbox-data` volume source and optional PVC object. +3. Verify Retain/Delete ownership metadata and immutable field validation. +4. Run focused planner tests. + +### Task 3: PVC reconciliation and Pod mount + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Modify: `controller/src/reconciler/tests.rs` + +1. Add failing tests for the generated Pod volume and runtime-only mount. +2. Apply dynamic PVCs after namespace creation and before Deployment reconciliation. +3. Resolve existing claims and block Deployment readiness on missing/incompatible claims. +4. Replace only `sandbox-data.emptyDir` with `persistentVolumeClaim` when configured. +5. Verify existing pod-shape tests and new storage tests. + +### Task 4: Bootstrap ConfigMap and init container + +**Files:** +- Modify: `controller/src/reconciler/workspace_storage.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` + +1. Add failing tests for allowed file validation, unsafe keys and init-container security shape. +2. Resolve the same-namespace ConfigMap before Deployment apply. +3. Add read-only bootstrap volume and a non-networked, non-privileged init container. +4. Implement `IfMissing` and `Always` atomic-copy behavior without logging contents. +5. Verify generated Pod JSON and security settings. + +### Task 5: StorageReady status gating + +**Files:** +- Modify: `controller/src/status/conditions.rs` +- Modify: `controller/src/status/mod.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/status/mod.rs` + +1. Add failing tests preventing `Ready=True` together with `StorageReady=False`. +2. Add StorageReady reasons and status construction. +3. Gate Deployment/Running status on claim and bootstrap readiness. +4. Verify status idempotency and suspended behavior. + +### Task 6: OpenClaw default workspace preservation + +**Files:** +- Modify: `sandbox-images/openclaw/entrypoint.sh` +- Create: `sandbox-images/openclaw/testM_workspace_defaults.sh` + +1. Add a failing shell regression proving existing `AGENTS.md`, `SOUL.md`, and `TOOLS.md` survive initialization. +2. Extract create-if-missing helpers and preserve current first-run templates. +3. Verify shell syntax and the regression script. + +### Task 7: Retain deletion semantics + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` +- Modify: `docs/plans/2026-08-07-persistent-workspace-design.md` only if the feasible namespace ownership model differs from the approved design. + +1. Add a failing test that demonstrates namespace deletion destroys an in-namespace retained claim. +2. Implement a feasible retention model before claiming `Retain` support. Do not rely solely on owner-reference omission. +3. Ensure `Delete` remains explicit and destructive. +4. Verify CR deletion, namespace deletion warning, and explicit recovery behavior. + +### Task 8: CLI, docs, and end-to-end validation + +**Files:** +- Modify: `cli/src/commands/add.ts` +- Modify: `cli/src/commands/add.test.ts` +- Modify: `docs/api/crd-reference.md` +- Modify: `docs/api/lifecycle.md` +- Modify: `docs/runtimes/CONTRACT.md` +- Modify: `docs/security.md` +- Modify: `docs/cli-reference.md` +- Modify: `docs/getting-started.md` +- Modify: `tests/e2e/run.sh` + +1. Add failing CLI tests for storage/bootstrap flags and destructive warnings. +2. Generate the new CR fields from CLI options. +3. Update public documentation and remove unconditional state-preservation claims. +4. Add lifecycle E2E coverage using a test PVC without bypassing the agent exec admission policy. +5. Run controller tests, CLI tests/typecheck/lint, shell checks and diff validation. diff --git a/docs/runtimes/CONTRACT.md b/docs/runtimes/CONTRACT.md index 2f6d47936..9f35f1149 100644 --- a/docs/runtimes/CONTRACT.md +++ b/docs/runtimes/CONTRACT.md @@ -130,7 +130,8 @@ The controller mounts these paths into the runtime container. | `/etc/kars/a2a-card/agent.json` (optional) | ConfigMap (A2AAgent compiled) | Router (mounts `/.well-known/agent.json` + `/a2a` routes when present) | ✅ router | | `/etc/kars/trustgraph/projection.json` (optional) | ConfigMap (TrustGraph projection per sandbox) | Runtime + router | ⚠️ TrustGraph reconciler exists; runtime-side consumption is planned | | `/sandbox/agent/` | OCI artifact or git via `spec.openclaw.config.agentCode` (and per-runtime equivalent) | Runtime entrypoint — user-supplied agent code lands here | ✅ all runtimes that support `agentCode` | -| `/sandbox/.openclaw/`, `/sandbox/.hermes/`, etc. | emptyDir | Runtime writable state (sessions, memory cache, prekeys) | ✅ all runtimes; per-runtime subdir | +| `/sandbox/.openclaw/`, `/sandbox/.hermes/`, etc. | `emptyDir` by default; optional per-sandbox PVC via `spec.storage.workspace` | Runtime writable state (sessions, memory cache, prekeys) | ✅ all runtimes; PVC preserves state across Pod recreation and suspension | +| `/etc/kars/workspace-bootstrap` | Same-namespace ConfigMap mirrored by the controller | OpenClaw bootstrap init container | ✅ OpenClaw; allowed files are `AGENTS.md`, `SOUL.md`, `HEARTBEAT.md`, `TOOLS.md`, `USER.md` | | `/tmp` (4 GiB tmpfs by default) | pod spec | Runtime scratch space | ✅ all runtimes | **Read-only root filesystem**: all of `/`, `/usr`, `/opt`, `/etc` (except mounted ConfigMaps/Secrets) is RO on AKS + local-k8s. Runtimes that need writable state under those paths must mirror to `/tmp` at entrypoint time (see `sandbox-images/openclaw/entrypoint.sh:42-52` for the OpenClaw mirror pattern). diff --git a/docs/security.md b/docs/security.md index 5d7927b68..bd616427b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -59,7 +59,15 @@ Applied to every sandbox pod: | User | Non-root (`runAsNonRoot: true`) — agent UID 1000, router UID 1001 | | Privilege escalation | Blocked (`allowPrivilegeEscalation: false`) | | Capabilities | All dropped (`drop: [ALL]`) | -| Writable paths | `/sandbox` and `/tmp` only (emptyDir) | +| Writable paths | `/sandbox` and `/tmp` only. `/sandbox` is `emptyDir` by default or an optional per-sandbox PVC; `/tmp` remains memory-backed `emptyDir`. | + +PVC persistence does not weaken UID, seccomp, read-only-rootfs, or egress +boundaries, but it extends the lifetime of data written by the runtime. The +operator remains responsible for StorageClass encryption, snapshots, backup, +regional placement, and customer-managed keys. `retainPolicy: Retain` protects +against deletion of the `KarsSandbox`; manually deleting the retained namespace +still deletes namespaced claims unless the backing PV policy independently +retains them. ### Layer 4 — Kernel confinement (seccomp) diff --git a/sandbox-images/openclaw/Dockerfile b/sandbox-images/openclaw/Dockerfile index 9a78b039f..962b9c7c6 100644 --- a/sandbox-images/openclaw/Dockerfile +++ b/sandbox-images/openclaw/Dockerfile @@ -175,7 +175,8 @@ RUN mkdir -p /etc/kars/policies /etc/kars/blocklist && \ # Copy entrypoint that auto-configures OpenClaw from mounted secrets COPY sandbox-images/openclaw/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY sandbox-images/openclaw/workspace-bootstrap.sh /usr/local/bin/workspace-bootstrap.sh +RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/workspace-bootstrap.sh # Labels LABEL org.opencontainers.image.title="kars OpenClaw Sandbox" \ diff --git a/sandbox-images/openclaw/entrypoint.sh b/sandbox-images/openclaw/entrypoint.sh index 6a90b173e..78c1088e4 100644 --- a/sandbox-images/openclaw/entrypoint.sh +++ b/sandbox-images/openclaw/entrypoint.sh @@ -383,7 +383,9 @@ else export OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 fi -# Always (re)generate config + workspace seed files on every container start. +# Always regenerate env-driven config on every container start. Workspace seed +# files are created only when missing so PVC-backed user/bootstrap content +# survives restarts. # # Previously this block was guarded by `[ ! -f "$OPENCLAW_CONFIG" ]` for "idempotency", # but on AKS `/sandbox` is a persistent volume and OpenClaw's runtime workspace @@ -1029,8 +1031,10 @@ RCEOF printf '\n# kars env (managed by entrypoint)\n[ -f /sandbox/.kars-env.sh ] && . /sandbox/.kars-env.sh\n' >> /sandbox/.bashrc fi - # Write minimal workspace files so OpenClaw doesn't need onboarding - cat > "$WORKSPACE_DIR/AGENTS.md" << AGENTSEOF + # Write defaults only when a bootstrap init container or prior runtime has + # not already supplied the file. + if [ ! -e "$WORKSPACE_DIR/AGENTS.md" ]; then + cat > "$WORKSPACE_DIR/AGENTS.md" << AGENTSEOF # kars Agent You are a helpful AI assistant running inside an **kars** sandbox — a secure, @@ -1143,9 +1147,11 @@ Network egress starts in **learn mode** — all domains are allowed and recorded The operator can graduate to enforcement with \`kars egress --enforce\`, which promotes learned domains to the allowlist. After that, new domains require approval. AGENTSEOF + fi # Write TOOLS.md describing available Foundry endpoints - cat > "$WORKSPACE_DIR/TOOLS.md" << 'TOOLSEOF' + if [ ! -e "$WORKSPACE_DIR/TOOLS.md" ]; then + cat > "$WORKSPACE_DIR/TOOLS.md" << 'TOOLSEOF' # kars Tools All tools are accessed via the inference router at http://localhost:8443. @@ -1210,8 +1216,10 @@ curl -s -X POST http://localhost:8443/egress/fetch \ **IMPORTANT:** Do NOT use `curl https://...` directly — it will time out. Always use `curl http://localhost:8443/egress/fetch` with the target URL in the body. TOOLSEOF + fi - cat > "$WORKSPACE_DIR/SOUL.md" << SOULEOF + if [ ! -e "$WORKSPACE_DIR/SOUL.md" ]; then + cat > "$WORKSPACE_DIR/SOUL.md" << SOULEOF # Soul You are **kars Agent** — a secure, sandboxed AI assistant powered by Azure AI Foundry. @@ -1239,6 +1247,7 @@ for clarification; interpret the task as given and deliver your best work. For memory: write important facts, preferences, and decisions to memory files so they persist across sessions. Use \`foundry_memory\` for cross-agent/cross-session recall. SOULEOF + fi echo "[kars] OpenClaw configured — model: ${MODEL}, endpoint: ${ENDPOINT}" else diff --git a/sandbox-images/openclaw/workspace-bootstrap.sh b/sandbox-images/openclaw/workspace-bootstrap.sh new file mode 100644 index 000000000..214e85672 --- /dev/null +++ b/sandbox-images/openclaw/workspace-bootstrap.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +set -euo pipefail + +source_dir="${KARS_WORKSPACE_BOOTSTRAP_SOURCE:-/etc/kars/workspace-bootstrap}" +destination_dir="${KARS_WORKSPACE_BOOTSTRAP_DESTINATION:-/sandbox/.openclaw/workspace}" +state_dir="${KARS_WORKSPACE_BOOTSTRAP_STATE_DIR:-/sandbox/.kars}" +policy="${KARS_WORKSPACE_OVERWRITE_POLICY:-IfMissing}" +config_map_uid="${KARS_WORKSPACE_BOOTSTRAP_CONFIG_MAP_UID:-}" +resource_version="${KARS_WORKSPACE_BOOTSTRAP_RESOURCE_VERSION:-}" + +case "$policy" in + IfMissing|Always) ;; + *) + echo "workspace bootstrap received invalid overwrite policy: $policy" >&2 + exit 1 + ;; +esac + +reject_symlink_components() { + local path="$1" + local current="" + local component + if [[ "$path" != /* ]]; then + echo "workspace bootstrap requires an absolute destination path: $path" >&2 + exit 1 + fi + IFS='/' read -r -a components <<< "$path" + for component in "${components[@]}"; do + [ -n "$component" ] || continue + current="$current/$component" + if [ -L "$current" ]; then + echo "workspace bootstrap refused symlink path component: $current" >&2 + exit 1 + fi + if [ -e "$current" ] && [ ! -d "$current" ]; then + echo "workspace bootstrap path component is not a directory: $current" >&2 + exit 1 + fi + done +} + +reject_symlink_components "$destination_dir" +reject_symlink_components "$state_dir" +mkdir -p "$destination_dir" "$state_dir" +umask 027 +manifest_tmp=$(mktemp "$state_dir/.bootstrap-state.XXXXXX") +copy_tmp="" +cleanup() { + [ -z "$copy_tmp" ] || rm -f -- "$copy_tmp" + rm -f -- "$manifest_tmp" +} +trap cleanup EXIT HUP INT TERM + +printf '{"configMapUid":"%s","resourceVersion":"%s","policy":"%s","files":{' \ + "$config_map_uid" "$resource_version" "$policy" > "$manifest_tmp" +first_file=true +for filename in AGENTS.md SOUL.md HEARTBEAT.md TOOLS.md USER.md; do + source_file="$source_dir/$filename" + destination_file="$destination_dir/$filename" + [ -f "$source_file" ] || continue + + if [ -L "$destination_file" ]; then + echo "workspace bootstrap refused symlink destination: $filename" >&2 + exit 1 + fi + if [ "$policy" = "Always" ] || [ ! -e "$destination_file" ]; then + copy_tmp=$(mktemp "$destination_dir/.${filename}.kars-bootstrap.XXXXXX") + cat -- "$source_file" > "$copy_tmp" + chmod 0640 "$copy_tmp" + mv -f -- "$copy_tmp" "$destination_file" + copy_tmp="" + fi + + digest=$(sha256sum "$destination_file" | awk '{print $1}') + if [ "$first_file" = true ]; then + first_file=false + else + printf ',' >> "$manifest_tmp" + fi + printf '"%s":"%s"' "$filename" "$digest" >> "$manifest_tmp" +done +printf '}}\n' >> "$manifest_tmp" +chmod 0640 "$manifest_tmp" +mv -f -- "$manifest_tmp" "$state_dir/bootstrap-state.json" +trap - EXIT HUP INT TERM From d3d45c8aea0b4bd833f5c581649e6aa0321f16b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 9 Aug 2026 04:02:36 +0000 Subject: [PATCH 2/3] docs: align workspace spec with implementation --- .../2026-08-07-persistent-workspace-design.md | 125 ++++++++++++------ 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/docs/plans/2026-08-07-persistent-workspace-design.md b/docs/plans/2026-08-07-persistent-workspace-design.md index 72fbe8054..83fbae9b0 100644 --- a/docs/plans/2026-08-07-persistent-workspace-design.md +++ b/docs/plans/2026-08-07-persistent-workspace-design.md @@ -1,22 +1,24 @@ # Kars Persistent Workspace Design -**Status:** Approved design +**Status:** Implemented in [PR #494](https://github.com/Azure/kars/pull/494); follow-up validation and observability are listed in Sections 13, 14, and 17. **Date:** 2026-08-07 -**Scope:** Per-sandbox persistent workspace storage and declarative OpenClaw workspace bootstrap files. +**Scope:** Runtime-agnostic per-sandbox persistent `/sandbox` storage, plus declarative OpenClaw workspace bootstrap files. **Out of scope:** Feishu channel integration, message-triggered wake-up, cross-cluster volume migration, multi-replica RWX, online volume expansion. ## 1. Problem -Every `KarsSandbox` runtime currently mounts `/sandbox` from an `emptyDir` volume. OpenClaw stores its workspace, sessions, runtime configuration, pairing state, dynamic bindings, local memory and AgentMesh identity below this directory. The data survives a container restart inside the same Pod but is lost when the Pod is recreated, the Deployment is rolled out, or a suspended sandbox scales from zero back to one. +Before this change, every `KarsSandbox` runtime mounted `/sandbox` from an `emptyDir` volume. Runtime state survived a container restart inside the same Pod but was lost when the Pod was recreated, the Deployment rolled out, or a suspended sandbox scaled from zero back to one. -The Controller also has no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source. OpenClaw's entrypoint currently rewrites Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. Although `runtime.openclaw.config` exists in the CRD schema, the OpenClaw deployment planner does not consume it. +The persistent volume is a platform-level capability shared by OpenClaw, Hermes, OpenAI Agents, Microsoft Agent Framework, LangGraph, Anthropic, PydanticAI, and BYO runtimes. It preserves only state the runtime writes below `/sandbox`; the controller cannot make framework state persistent when an adapter stores that state elsewhere or only in memory. + +Before this change, the Controller had no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source, and OpenClaw's entrypoint rewrote Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. The implementation now uses a typed workspace block and create-if-missing defaults; the unrelated unstructured `runtime.openclaw.config` field remains outside this feature. ## 2. Goals -1. Give each `KarsSandbox` an optional, dedicated persistent volume mounted at `/sandbox`. -2. Preserve sessions, workspace files, pairing state, dynamic bindings and runtime identity across Pod recreation and scale-to-zero. +1. Give every supported `KarsSandbox` runtime an optional, dedicated persistent volume mounted at `/sandbox`. +2. Preserve runtime state written below `/sandbox` across Pod recreation and scale-to-zero. 3. Let an operator initialize selected OpenClaw workspace Markdown files from a same-namespace ConfigMap. 4. Preserve user changes by default after the initial bootstrap. 5. Retain dynamically provisioned storage by default when a `KarsSandbox` is deleted. @@ -60,11 +62,13 @@ Dynamically created claims default to `Retain`. Deleting the `KarsSandbox` remov The default `IfMissing` policy copies each managed file only when the destination does not exist. A ConfigMap update does not overwrite files already modified on the PVC. -## 5. Proposed API +## 5. API + +The storage block is runtime-agnostic. The nested OpenClaw workspace block is intentionally runtime-specific because `SOUL.md`, `HEARTBEAT.md`, and related files are OpenClaw contracts, not portable Kars runtime contracts. ### 5.1 KarsSandbox storage -Add the following optional block to `KarsSandboxSpec`: +`KarsSandboxSpec` exposes the following optional block: ```yaml apiVersion: kars.azure.com/v1alpha1 @@ -116,7 +120,7 @@ pub struct WorkspaceStorageSpec { pub storage_class_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub access_modes: Option>, + pub access_modes: Vec, #[serde(default)] pub retain_policy: WorkspaceRetainPolicy, @@ -222,7 +226,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: teaching-agent-workspace - namespace: kars-teaching-agent + namespace: kars-system # same namespace as the KarsSandbox CR data: SOUL.md: | # Soul @@ -245,7 +249,7 @@ data: ## 6. Admission and validation -The generated CRD and Helm CRD must enforce these constraints with schema and CEL where possible: +The generated CRD and Helm CRD enforce these constraints with schema and CEL where possible: 1. `existingClaim` must not appear with dynamic provisioning fields. 2. `size` must parse as a positive Kubernetes quantity. @@ -269,18 +273,24 @@ Required metadata: ```yaml metadata: labels: - kars.azure.com/managed: "true" + app.kubernetes.io/managed-by: kars-controller kars.azure.com/sandbox: teaching-agent kars.azure.com/storage-role: workspace annotations: kars.azure.com/retain-policy: Retain + kars.azure.com/sandbox-uid: ``` The source `KarsSandbox` and generated PVC live in different namespaces, so a -namespaced owner reference would be invalid. Both policies use labels plus a -`kars.azure.com/sandbox-uid` provenance annotation. `Delete` relies on deletion -of the generated sandbox namespace; `Retain` preserves that namespace and its -PVC while removing labelled workload resources. +namespaced owner reference would be invalid. Both policies use strict management +labels plus a `kars.azure.com/sandbox-uid` provenance annotation. + +Any PVC in the generated namespace prevents direct namespace cascading. On +deletion, the controller only deletes a PVC when the current CR still declares +a dynamically generated `retainPolicy: Delete` workspace and the claim name, +management labels, storage-role label, and sandbox UID all match. It then +requeues until the claim is gone. `Retain`, `existingClaim`, and omitted storage +never authorize PVC deletion, even if a claim carries stale annotations. ### 7.2 Immutable fields and drift @@ -290,7 +300,7 @@ The Controller reconciles desired PVC shape without attempting invalid in-place - Decreasing `size` is rejected. - Changing `storageClassName` is rejected after creation. - Changing `accessModes` is rejected after creation. -- Changing from a generated claim to `existingClaim`, or the reverse, is rejected while either claim contains active state. +- Changing from a generated claim to `existingClaim`, the reverse, or changing existing claim names requires `spec.suspended: true`; active transitions fail closed with `StorageReady=False/ImmutableFieldChanged`. These errors set `StorageReady=False` and leave the last-known-good Deployment and claim untouched. @@ -315,6 +325,10 @@ volumeMounts: `/tmp` remains a memory-backed `emptyDir`. +PVC-backed or bootstrap-enabled Deployments use the `Recreate` strategy. This +prevents two runtime Pods from concurrently writing a ReadWriteOnce workspace +and avoids rolling-update multi-attach deadlocks. + The inference-router does not receive access to the workspace PVC unless a separately reviewed feature requires it. The least-privilege boundary remains unchanged. ### 7.4 Bootstrap ConfigMap mount @@ -329,10 +343,10 @@ Do not mount the ConfigMap directly over the writable OpenClaw workspace. ### 7.5 Bootstrap init container -Add an init container before the runtime starts. It mounts: +The controller adds an init container before the runtime starts. It mounts: - `sandbox-data` at `/sandbox`; -- the bootstrap ConfigMap at `/bootstrap`, read-only. +- the bootstrap ConfigMap at `/etc/kars/workspace-bootstrap`, read-only. Its responsibilities are: @@ -348,6 +362,11 @@ Its responsibilities are: The init container must not log file contents. +The Pod disables automatic service-account-token mounting. A projected +Kubernetes token is mounted explicitly into the runtime and router containers, +while `egress-guard` and `workspace-bootstrap` receive neither that token nor an +Azure Workload Identity token. + ### 7.6 Default Kars templates If no bootstrap ConfigMap is configured, the existing Kars default `AGENTS.md`, `SOUL.md`, and `TOOLS.md` remain available, but entrypoint behavior changes from unconditional overwrite to create-if-missing. @@ -376,18 +395,21 @@ For `retainPolicy: Retain`: 1. Delete the Deployment and normal sandbox-owned resources. 2. Leave the PVC and backing PV intact. -3. Add an event and final status message naming the retained claim before the CR disappears. -4. Do not remove the PVC protection finalizer. -5. Do not automatically expose the retained claim to another sandbox. +3. Preserve the generated namespace because PVCs are namespaced resources. +4. Do not automatically expose the retained claim to another sandbox instance. -A future sandbox may use the retained data only by explicitly setting `existingClaim`. +A future sandbox may use the retained data only by explicitly setting +`existingClaim`. Because the claim remains in `kars-`, recovery +normally recreates the same sandbox name. ### 8.2 Delete For `retainPolicy: Delete`: -- the finalizer deletes the generated sandbox namespace; -- namespace cascading deletion removes the claim; +- the current CR spec must still explicitly request a dynamic Delete workspace; +- the controller verifies the generated claim's name, management labels, + storage-role label, and sandbox UID before deleting it; +- the finalizer waits for the claim to disappear, then deletes the namespace; - backing-volume deletion follows the StorageClass/PV reclaim policy. The CLI must show a destructive warning before creating or updating a sandbox to `Delete`. @@ -396,9 +418,9 @@ The CLI must show a destructive warning before creating or updating a sandbox to Kubernetes namespace deletion can delete both Retain and Delete claims. `retainPolicy: Retain` protects against deletion of the `KarsSandbox`, not deletion of its namespace. Documentation and CLI output must state this explicitly. -## 9. Status and events +## 9. Status conditions -Add a `StorageReady` condition to `KarsSandbox.status.conditions`. +The controller adds a `StorageReady` condition to `KarsSandbox.status.conditions`. | Status | Reason | Meaning | |---|---|---| @@ -408,21 +430,28 @@ Add a `StorageReady` condition to `KarsSandbox.status.conditions`. | `False` | `ClaimNotFound` | Referenced existing claim does not exist. | | `False` | `ClaimIncompatible` | Access mode or claim shape is unsupported. | | `False` | `ImmutableFieldChanged` | Requested storage mutation cannot be applied safely. | + +`BootstrapReady` is a separate condition: + +| Status | Reason | Meaning | +|---|---|---| +| `True` | `Reconciled` | The bootstrap init container completed successfully. | +| `False` | `Creating` | The bootstrap Pod/init container is pending. | | `False` | `BootstrapConfigNotFound` | Referenced ConfigMap does not exist. | | `False` | `BootstrapInvalid` | ConfigMap contains unsupported or unsafe entries. | -| `False` | `BootstrapFailed` | Init container failed to initialize the workspace. | +| `False` | `BootstrapFailed` | The init container terminated or crash-looped. | -`Ready=True` requires `StorageReady=True` when persistent storage or bootstrap is configured. +`Ready=True` requires both storage and configured bootstrap initialization to be ready. The controller aggregates all active Pods with `Failed > Pending > Ready` precedence so an old successful Pod cannot hide a new failing rollout. -The Controller emits Kubernetes Events for claim creation, retention, incompatible mutation, missing bootstrap ConfigMap and bootstrap failure. +Kubernetes Events for storage/bootstrap transitions are not part of PR #494 and remain follow-up work. ## 10. Security 1. ConfigMap data is non-sensitive. Admission documentation prohibits secrets in workspace bootstrap files. 2. Credentials remain in `-credentials` and are injected through `envFrom` or mounted Secret files. -3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials, service account token or network access. +3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials or service-account token. Existing Pod-level NetworkPolicy still governs the Pod network; the bootstrap script itself performs no network operations. 4. Bootstrap source and destination paths are fixed; user-controlled path fields are not supported. -5. Symlinks are rejected at both source and destination. +5. ConfigMap projection symlinks are accepted as read-only sources; destination files and every writable destination/state directory component reject symlinks. 6. Atomic writes prevent partially initialized files. 7. The runtime remains UID 1000 with a read-only root filesystem. 8. PVCs are per sandbox and same namespace. Cross-namespace claim references are impossible. @@ -431,7 +460,7 @@ The Controller emits Kubernetes Events for claim creation, retention, incompatib ## 11. CLI behavior -Add storage flags to `kars add`: +`kars add` exposes these storage flags: ```text --workspace-storage Enable a generated workspace PVC, e.g. 10Gi @@ -515,10 +544,15 @@ Verify generated resources for: - omitted storage produces `emptyDir`; - dynamic storage produces the expected PVC and claim mount; - both policies omit cross-namespace owner references and record sandbox UID provenance; -- Delete removes the namespace; Retain preserves namespace + PVC; +- Delete removes only a strictly matched generated claim, then its namespace; +- Retain, existingClaim, and omitted storage preserve any namespace containing PVCs; - existing claims do not produce a PVC object; - bootstrap produces ConfigMap volume, mount and init container; - Router does not mount the workspace claim; +- bootstrap init containers do not receive Kubernetes or Azure identity tokens; +- PVC/bootstrap Deployments use `Recreate`; +- valid pending existing claims remain `ClaimPending` with zero replicas; +- valid pending dynamic claims retain one consumer Pod for `WaitForFirstConsumer`; - suspended sandboxes retain PVC reconciliation with replicas zero; - immutable changes set the expected condition. @@ -538,6 +572,10 @@ Verify: ### 13.4 End-to-end tests +**Follow-up, not delivered by PR #494.** The PR includes controller/CLI unit +tests, Rust/Helm schema parity, API-server CRD dry-run, and executable bootstrap +filesystem tests. A CSI-backed lifecycle suite should additionally prove: + A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath-backed PVC, must prove: 1. Create a sandbox with persistent workspace and bootstrap ConfigMap. @@ -551,12 +589,14 @@ A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath 9. Delete a Retain sandbox and verify the PVC remains. 10. Create a new sandbox with `existingClaim` and verify explicit recovery. 11. Delete a Delete-policy sandbox and verify its claim is removed. +12. Repeat the write/recreate/suspend sentinel flow for Hermes, LangGraph, and + BYO to verify each adapter places recoverable state below `/sandbox`. Do not use `kubectl exec` into the agent container in AKS E2E because the validating admission policy correctly blocks that path. Use an approved test runtime, init-container evidence, `kars connect`, or a purpose-built test probe. ## 14. Observability -Add metrics: +**Follow-up, not delivered by PR #494.** Proposed metrics: ```text kars_workspace_storage_reconcile_total{result,mode} @@ -572,7 +612,7 @@ Log claim names, mode, condition reason and bootstrap resourceVersion. Never log ## 15. Documentation updates -Implementation must update: +The implementation updates: - `docs/api/crd-reference.md` with storage and OpenClaw workspace fields; - `docs/api/lifecycle.md` with PVC creation, suspension and deletion behavior; @@ -584,21 +624,21 @@ Implementation must update: ## 16. Acceptance criteria -The feature is complete when all of the following are true: +The implemented acceptance criteria are: 1. An old `KarsSandbox` without storage still runs with `emptyDir`. 2. A sandbox with dynamic storage gets a unique Bound PVC mounted at `/sandbox`. -3. Pod recreation and scale-to-zero preserve OpenClaw workspace and runtime state. +3. Pod recreation and scale-to-zero preserve state that any runtime writes below `/sandbox`. 4. A same-namespace existing claim can be explicitly attached without Controller adoption or deletion. 5. Retain is the default and deleting the CR leaves the claim intact. -6. Delete is explicit and removes the generated claim through namespace cascading deletion. +6. Delete is explicit, requires strict current-spec and PVC provenance checks, and removes the generated claim before namespace deletion. 7. A bootstrap ConfigMap initializes only the allowed files. 8. `IfMissing` preserves runtime/user edits across restart. 9. `Always` deterministically reapplies operator content. 10. `MEMORY.md`, credentials and runtime databases cannot be supplied through bootstrap. 11. Missing/incompatible claims and invalid bootstrap data prevent false `Ready=True` and expose actionable conditions. 12. No Secret or workspace content is emitted to logs or status. -13. Controller, CRD, CLI and E2E tests cover the behaviors above. +13. Controller, CRD, CLI, and executable bootstrap tests cover the implemented behaviors; CSI-backed multi-runtime E2E remains tracked follow-up work. 14. Documentation no longer claims state survives suspension when `/sandbox` is ephemeral. ## 17. Future extensions @@ -614,3 +654,8 @@ Potential follow-up specs may add: - per-file bootstrap policy; - OCI-based signed workspace bundles; - explicit migration jobs between claims or storage classes. +- runtime-specific bootstrap contracts for Hermes or other harnesses (rather + than reusing OpenClaw Markdown semantics); +- a CSI-backed persistence matrix for Hermes, LangGraph, BYO, and the remaining + shipping adapters; +- workspace lifecycle Events and Prometheus metrics listed in Section 14. From 92bb3be95af19c796f7bb280b9ae5d09507f7cf2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 9 Aug 2026 04:27:53 +0000 Subject: [PATCH 3/3] feat(runtime): generalize persistent workspace support --- cli/src/commands/add.test.ts | 90 ++++++++++++++---- cli/src/commands/add.ts | 92 ++++++++++++------- cli/src/commands/up.ts | 22 +++++ cli/src/commands/up/sandbox_bringup.ts | 19 ++++ cli/src/lib/testM_workspace_storage.test.ts | 49 ++++++++++ cli/src/lib/workspace-storage.ts | 30 ++++++ controller/src/reconciler/mod.rs | 19 ++++ controller/src/reconciler/tests.rs | 20 ++++ docs/cli-reference.md | 22 +++++ docs/getting-started.md | 5 + .../2026-08-07-persistent-workspace-design.md | 11 +++ docs/runtimes/CONTRACT.md | 25 +++++ 12 files changed, 349 insertions(+), 55 deletions(-) create mode 100644 cli/src/lib/testM_workspace_storage.test.ts create mode 100644 cli/src/lib/workspace-storage.ts diff --git a/cli/src/commands/add.test.ts b/cli/src/commands/add.test.ts index d5f61c879..86ed16600 100644 --- a/cli/src/commands/add.test.ts +++ b/cli/src/commands/add.test.ts @@ -2,6 +2,9 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; +import { validateRuntimeSpecificAddFlags } from "./add.js"; +import { buildRuntimeBlock, type RuntimeKind } from "../runtime.js"; +import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; /** * Tests for the `add` command's sandbox manifest generation logic. @@ -42,6 +45,7 @@ interface AddOptions { workspaceRetainPolicy?: "Retain" | "Delete"; workspaceBootstrap?: string; workspaceOverwrite?: "IfMissing" | "Always"; + runtimeKind?: RuntimeKind; } function defaultOptions(overrides: Partial = {}): AddOptions { @@ -60,19 +64,21 @@ function defaultOptions(overrides: Partial = {}): AddOptions { /** Build the KarsSandbox manifest object (mirrors add.ts action logic). */ function buildSandboxManifest(name: string, options: AddOptions) { + const runtimeKind = options.runtimeKind ?? "OpenClaw"; const sandbox: Record = { apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", metadata: { name, namespace: "kars-system" }, spec: { - runtime: { - kind: "OpenClaw", - openclaw: { - version: "2026.3.13", - ...(options.image ? { image: options.image } : {}), - config: { agent: { model: `azure/${options.model}` } }, - }, - }, + runtime: buildRuntimeBlock({ + kind: runtimeKind, + openclawVersion: "2026.3.13", + model: options.model, + image: options.image, + byoImage: runtimeKind === "BYO" ? "example.invalid/byo:latest" : undefined, + byoContractVersion: "v1", + mafLanguage: "python", + }), sandbox: { isolation: options.isolation, seccompProfile: options.isolation === "standard" ? "RuntimeDefault" : "kars-strict", @@ -119,18 +125,9 @@ function buildSandboxManifest(name: string, options: AddOptions) { np.egressMode = "Learn"; } - if (options.workspaceStorage || options.workspaceExistingClaim) { - const workspace = options.workspaceExistingClaim - ? { existingClaim: options.workspaceExistingClaim } - : { - size: options.workspaceStorage, - ...(options.workspaceStorageClass - ? { storageClassName: options.workspaceStorageClass } - : {}), - accessModes: ["ReadWriteOnce"], - retainPolicy: options.workspaceRetainPolicy ?? "Retain", - }; - (sandbox.spec as Record).storage = { workspace }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandbox.spec as Record).storage = storage; } if (options.workspaceBootstrap) { @@ -377,6 +374,59 @@ describe("KarsSandbox manifest generation", () => { expect(spec.sandbox.allowPrivilegeEscalation).toBe(false); expect(spec.sandbox.writablePaths).toEqual(["/sandbox", "/tmp"]); }); + + it.each([ + "OpenClaw", + "Hermes", + "OpenAIAgents", + "MicrosoftAgentFramework", + "LangGraph", + "Anthropic", + "PydanticAi", + "BYO", + ] satisfies RuntimeKind[])("configures workspace storage for %s", (runtimeKind) => { + const manifest = buildSandboxManifest( + "persistent-agent", + defaultOptions({ runtimeKind, workspaceStorage: "10Gi" }), + ); + const spec = manifest.spec as any; + expect(spec.runtime.kind).toBe(runtimeKind); + expect(spec.storage.workspace).toEqual({ + size: "10Gi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }); + }); +}); + +describe("runtime-specific add flag validation", () => { + it("allows channels for Hermes", () => { + expect( + validateRuntimeSpecificAddFlags("Hermes", { + channels: "telegram", + telegramToken: "test-token", + }), + ).toEqual([]); + }); + + it("rejects channels for runtimes without channel adapters", () => { + expect( + validateRuntimeSpecificAddFlags("LangGraph", { channels: "telegram" }), + ).toEqual([ + "--channels is only valid with --runtime openclaw or --runtime hermes.", + ]); + }); + + it("keeps skills, plugins, and image overrides OpenClaw-only", () => { + expect( + validateRuntimeSpecificAddFlags("Hermes", { + skills: "browser", + image: "example.invalid/custom:latest", + }), + ).toEqual([ + "--skills, --image are only valid with --runtime openclaw.", + ]); + }); }); describe("channel and plugin secret generation", () => { diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index a7f0d5beb..d02bbe083 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -12,6 +12,56 @@ import { inferenceRefName, toolPolicyRefName, } from "../refs.js"; +import type { RuntimeKind } from "../runtime.js"; +import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; + +export function validateRuntimeSpecificAddFlags( + runtimeKind: RuntimeKind, + options: Record, +): string[] { + const errors: string[] = []; + const channelFlags: Array<[string, unknown]> = [ + ["--channels", options.channels], + ["--telegram-token", options.telegramToken], + ["--telegram-allow-from", options.telegramAllowFrom], + ["--slack-token", options.slackToken], + ["--discord-token", options.discordToken], + ]; + if (runtimeKind !== "OpenClaw" && runtimeKind !== "Hermes") { + const used = channelFlags + .filter(([, value]) => value !== undefined && value !== "" && value !== false) + .map(([flag]) => flag); + if (used.length > 0) { + errors.push( + `${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` + + `--runtime openclaw or --runtime hermes.`, + ); + } + } + + const openClawOnlyFlags: Array<[string, unknown]> = [ + ["--skills", options.skills], + ["--brave-api-key", options.braveApiKey], + ["--tavily-api-key", options.tavilyApiKey], + ["--exa-api-key", options.exaApiKey], + ["--firecrawl-api-key", options.firecrawlApiKey], + ["--perplexity-api-key", options.perplexityApiKey], + ["--openai-api-key", options.openaiApiKey], + ["--image", options.image], + ]; + if (runtimeKind !== "OpenClaw") { + const used = openClawOnlyFlags + .filter(([, value]) => value !== undefined && value !== "" && value !== false) + .map(([flag]) => flag); + if (used.length > 0) { + errors.push( + `${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` + + `--runtime openclaw.`, + ); + } + } + return errors; +} export function addCommand(): Command { const cmd = new Command("add"); @@ -102,29 +152,10 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. // Validate runtime-specific flag combinations before doing any work. // Reject incompatible flags up-front with a clear, actionable error // — better than silently ignoring a user's intent. - const openClawOnlyFlags: Array<[string, unknown]> = [ - ["--channels", options.channels], - ["--telegram-token", options.telegramToken], - ["--telegram-allow-from", options.telegramAllowFrom], - ["--slack-token", options.slackToken], - ["--discord-token", options.discordToken], - ["--skills", options.skills], - ["--brave-api-key", options.braveApiKey], - ["--tavily-api-key", options.tavilyApiKey], - ["--exa-api-key", options.exaApiKey], - ["--firecrawl-api-key", options.firecrawlApiKey], - ["--perplexity-api-key", options.perplexityApiKey], - ["--openai-api-key", options.openaiApiKey], - ["--image", options.image], - ]; - if (runtimeKind !== "OpenClaw") { - const used = openClawOnlyFlags.filter(([, v]) => v !== undefined && v !== "" && v !== false).map(([f]) => f); - if (used.length > 0) { - console.error(chalk.red(`\n Error: ${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with --runtime openclaw.`)); - console.error(chalk.dim(` Channels, skills, and plugin API keys are OpenClaw-specific entrypoint features.`)); - console.error(chalk.dim(` For ${options.runtime}, configure equivalents inside the agent's own code.\n`)); - process.exit(1); - } + const runtimeFlagErrors = validateRuntimeSpecificAddFlags(runtimeKind, options); + if (runtimeFlagErrors.length > 0) { + console.error(chalk.red(`\n Error: ${runtimeFlagErrors.join(" ")}\n`)); + process.exit(1); } if (runtimeKind !== "BYO" && (options.byoImage || (options.byoContractVersion && options.byoContractVersion !== "v1"))) { console.error(chalk.red(`\n Error: --byo-image / --byo-contract-version are only valid with --runtime byo.\n`)); @@ -224,18 +255,9 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. }, }; - if (options.workspaceStorage || options.workspaceExistingClaim) { - const workspace = options.workspaceExistingClaim - ? { existingClaim: options.workspaceExistingClaim } - : { - size: options.workspaceStorage, - ...(options.workspaceStorageClass - ? { storageClassName: options.workspaceStorageClass } - : {}), - accessModes: ["ReadWriteOnce"], - retainPolicy: options.workspaceRetainPolicy, - }; - (sandbox.spec as Record).storage = { workspace }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandbox.spec as Record).storage = storage; } // Add Foundry agent config if provided diff --git a/cli/src/commands/up.ts b/cli/src/commands/up.ts index 8e1a71340..a852f399d 100644 --- a/cli/src/commands/up.ts +++ b/cli/src/commands/up.ts @@ -51,6 +51,11 @@ export function upCommand(): Command { .option("--release [version]", "Import the PUBLIC, signed GHCR release images (ghcr.io/azure/*) into your ACR instead of building from source. Bare --release uses the latest release; pass a tag (e.g. v0.1.4) to pin. No Rust/Docker build needed.") .option("--build", "Build images locally and push to ACR (developer mode)", false) .option("--skip-runtime-images", "Skip building/importing the 7 multi-runtime adapter images (faster first deploy; only OpenClaw + BYO will be runnable)", false) + // ── Initial sandbox workspace ─────────────────────────────────────── + .option("--workspace-storage ", "Create a persistent workspace PVC for the initial sandbox, e.g. 10Gi") + .option("--workspace-storage-class ", "StorageClass for the initial sandbox workspace PVC") + .option("--workspace-existing-claim ", "Use an existing PVC in the initial sandbox namespace") + .option("--workspace-retain-policy ", "Initial workspace deletion policy: Retain | Delete", "Retain") // ── Foundry / Azure OpenAI ──────────────────────────────────────── .option("--foundry-endpoint ", "Existing Azure AI Foundry project endpoint (services.ai.azure.com)") .option("--openai-endpoint ", "Existing Azure OpenAI endpoint (openai.azure.com, derived from Foundry if omitted)") @@ -84,6 +89,7 @@ Flag groups: Cluster / region: --region, --cluster-name, --isolation, --resource-group Infrastructure: --skip-infra, --force-infra, --skip-preflight Images: --source-acr, --build, --skip-runtime-images + Workspace: --workspace-storage, --workspace-existing-claim, --workspace-retain-policy Foundry: --foundry-endpoint, --openai-endpoint Mesh federation: --mesh-peer / --no-mesh-peer, --global-registry, --expose-registry, --mesh-trust=anonymous|entra Output / lifecycle: --dry-run, --upgrade, --from-scratch @@ -115,6 +121,22 @@ Auto-resume: console.error(chalk.red(`\n Error: --policy must be one of: ${policyPresets.join(" | ")} (got "${options.policy}").\n`)); process.exit(1); } + if (options.workspaceStorage && options.workspaceExistingClaim) { + console.error(chalk.red("\n Error: --workspace-storage and --workspace-existing-claim are mutually exclusive.\n")); + process.exit(1); + } + if (options.workspaceStorageClass && !options.workspaceStorage) { + console.error(chalk.red("\n Error: --workspace-storage-class requires --workspace-storage .\n")); + process.exit(1); + } + if (!["Retain", "Delete"].includes(options.workspaceRetainPolicy)) { + console.error(chalk.red("\n Error: --workspace-retain-policy must be Retain or Delete.\n")); + process.exit(1); + } + if (options.workspaceExistingClaim && options.workspaceRetainPolicy !== "Retain") { + console.error(chalk.red("\n Error: --workspace-retain-policy applies only to generated PVCs.\n")); + process.exit(1); + } const { execa } = await import("execa"); diff --git a/cli/src/commands/up/sandbox_bringup.ts b/cli/src/commands/up/sandbox_bringup.ts index 78aee6294..462201b5f 100644 --- a/cli/src/commands/up/sandbox_bringup.ts +++ b/cli/src/commands/up/sandbox_bringup.ts @@ -23,6 +23,7 @@ import { inferenceRefName, toolPolicyRefName, } from "../../refs.js"; +import { buildWorkspaceStorageSpec } from "../../lib/workspace-storage.js"; export interface SandboxBringUpContext { options: { @@ -30,6 +31,10 @@ export interface SandboxBringUpContext { model: string; region: string; isolation: string; + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; [key: string]: unknown; }; baseName: string; @@ -516,6 +521,10 @@ export async function bringUpSandbox(ctx: SandboxBringUpContext): Promise }, }, }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandboxManifest.spec as Record).storage = storage; + } // KarsMemory binding — only meaningful with a Foundry project endpoint // (Memory Store is a Foundry feature). Gives the sandbox the same // controller-managed binding `kars dev` creates, instead of relying purely @@ -638,6 +647,16 @@ export async function bringUpSandbox(ctx: SandboxBringUpContext): Promise kvLine("Sandbox", options.name); kvLine("Model", `${options.model} (Azure OpenAI, Entra ID auth)`); kvLine("Isolation", isolationDesc[options.isolation] || options.isolation); + if (options.workspaceExistingClaim) { + kvLine("Workspace", `existing PVC ${String(options.workspaceExistingClaim)}`); + } else if (options.workspaceStorage) { + kvLine( + "Workspace", + `${String(options.workspaceStorage)} (${String(options.workspaceStorageClass || "default StorageClass")}, ${String(options.workspaceRetainPolicy || "Retain")})`, + ); + } else { + kvLine("Workspace", "ephemeral emptyDir"); + } kvLine("Region", options.region); kvLine("Cluster", `${baseName}-aks`); kvLine("ACR", acrLoginServer); diff --git a/cli/src/lib/testM_workspace_storage.test.ts b/cli/src/lib/testM_workspace_storage.test.ts new file mode 100644 index 000000000..da17b9f20 --- /dev/null +++ b/cli/src/lib/testM_workspace_storage.test.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { buildWorkspaceStorageSpec } from "./workspace-storage.js"; + +describe("buildWorkspaceStorageSpec", () => { + it("omits storage when persistence is not requested", () => { + expect(buildWorkspaceStorageSpec({})).toBeUndefined(); + }); + + it("builds a retained dynamic workspace", () => { + expect( + buildWorkspaceStorageSpec({ + workspaceStorage: "20Gi", + workspaceStorageClass: "managed-csi", + workspaceRetainPolicy: "Retain", + }), + ).toEqual({ + workspace: { + size: "20Gi", + storageClassName: "managed-csi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }, + }); + }); + + it("builds a destructive dynamic workspace only when requested", () => { + expect( + buildWorkspaceStorageSpec({ + workspaceStorage: "1.5Gi", + workspaceRetainPolicy: "Delete", + }), + ).toEqual({ + workspace: { + size: "1.5Gi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Delete", + }, + }); + }); + + it("builds a pure existing-claim reference", () => { + expect( + buildWorkspaceStorageSpec({ workspaceExistingClaim: "restored-workspace" }), + ).toEqual({ workspace: { existingClaim: "restored-workspace" } }); + }); +}); \ No newline at end of file diff --git a/cli/src/lib/workspace-storage.ts b/cli/src/lib/workspace-storage.ts new file mode 100644 index 000000000..33c411f82 --- /dev/null +++ b/cli/src/lib/workspace-storage.ts @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export interface WorkspaceStorageOptions { + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; +} + +export function buildWorkspaceStorageSpec( + options: WorkspaceStorageOptions, +): Record | undefined { + if (options.workspaceExistingClaim) { + return { + workspace: { existingClaim: options.workspaceExistingClaim }, + }; + } + if (!options.workspaceStorage) return undefined; + return { + workspace: { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy ?? "Retain", + }, + }; +} \ No newline at end of file diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 0268d7819..675e4819d 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -648,6 +648,20 @@ fn workspace_desired_replicas( } } +fn runtime_state_dir(kind: &crate::crd::RuntimeKind) -> &'static str { + match kind { + crate::crd::RuntimeKind::OpenClaw => "/sandbox/.openclaw", + crate::crd::RuntimeKind::Hermes => "/sandbox/.hermes", + crate::crd::RuntimeKind::OpenAIAgents => "/sandbox/.openai-agents", + crate::crd::RuntimeKind::MicrosoftAgentFramework => "/sandbox/.maf", + crate::crd::RuntimeKind::LangGraph => "/sandbox/.langgraph", + crate::crd::RuntimeKind::Anthropic => "/sandbox/.anthropic", + crate::crd::RuntimeKind::PydanticAi => "/sandbox/.pydantic-ai", + crate::crd::RuntimeKind::BYO => "/sandbox/.byo", + crate::crd::RuntimeKind::SemanticKernel => "/sandbox/.semantic-kernel", + } +} + fn kube_api_access_mount() -> serde_json::Value { json!({ "name": "kube-api-access", @@ -2676,6 +2690,11 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result` | controller runtime-kind mapping | Recommended private state directory for sessions, checkpoints, identities, and runtime metadata. Adapters may expose a native alias such as `HERMES_HOME`. | ✅ all Kubernetes runtimes | ### Conditionally present @@ -136,6 +138,29 @@ The controller mounts these paths into the runtime container. **Read-only root filesystem**: all of `/`, `/usr`, `/opt`, `/etc` (except mounted ConfigMaps/Secrets) is RO on AKS + local-k8s. Runtimes that need writable state under those paths must mirror to `/tmp` at entrypoint time (see `sandbox-images/openclaw/entrypoint.sh:42-52` for the OpenClaw mirror pattern). +### Persistent workspace contract + +`spec.storage.workspace` is a platform-level contract shared by every wired +runtime. The controller guarantees only that the same filesystem is mounted at +`/sandbox` after Pod recreation or suspension. Each runtime adapter or user +application remains responsible for placing recoverable state there. + +| Runtime | Known state below `/sandbox` | Persistence boundary | +|---|---|---| +| OpenClaw | `/sandbox/.openclaw` | Sessions, workspace, memory cache, pairing/bindings, and mesh identity use the PVC. OpenClaw-only ConfigMap bootstrap is supported. | +| Hermes | `/sandbox/.hermes` and `HOME=/sandbox` | Hermes config, sessions, memory, channel-local state, plugins, and AGT identity use the PVC. No declarative Hermes bootstrap yet. | +| OpenAI Agents | `/sandbox/agent` | User code and files persist. Conversation/session state persists only if the application configures its store/checkpoint under `/sandbox`. | +| Microsoft Agent Framework Python | `/sandbox/agent` | User code and files persist. Thread/conversation persistence remains application-defined. | +| LangGraph Python/TypeScript | `/sandbox/agent` | User code and files persist. Agents must configure a durable checkpointer (for example SQLite under `/sandbox` or an external database); in-memory checkpointers do not become durable merely because a PVC is mounted. | +| Anthropic | `/sandbox/agent` | User code and files persist. SDK session state remains application-defined. | +| PydanticAI | `/sandbox/agent` | User code and files persist. Application message history/state must be written under `/sandbox` or to an external store. | +| BYO | `/sandbox` by contract | The image must write every state item it expects to recover below `/sandbox`; writes elsewhere are not covered. | + +The PVC capability is implemented on Kubernetes targets (AKS and local-k8s). +The Docker `kars dev` target has no `KarsSandbox` CRD or PVC and continues to +use its Docker bind/volume behavior. Do not interpret `--workspace-storage` as +a portable Docker-dev flag. + --- ## HTTP contract — runtime ↔ router