From 90d208fa5e1221c68f069119444ff978adf20ed9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 8 Aug 2026 05:27:23 +0000 Subject: [PATCH 1/5] feat(controller): add persistent sandbox workspaces --- cli/src/commands/add.test.ts | 69 + cli/src/commands/add.ts | 76 ++ controller/src/crd.rs | 175 ++- controller/src/crd_validations.rs | 54 +- controller/src/helm_drift.rs | 55 +- controller/src/reconciler/mod.rs | 1199 ++++++++++++++++- controller/src/reconciler/tests.rs | 779 ++++++++++- controller/src/status/conditions.rs | 18 + controller/src/status/mod.rs | 155 ++- deploy/helm/kars/templates/crd.yaml | 54 +- deploy/helm/kars/templates/rbac.yaml | 12 +- docs/api/conditions.md | 13 + docs/api/crd-reference.md | 24 +- docs/api/lifecycle.md | 6 +- docs/cli-reference.md | 14 + docs/getting-started.md | 13 + .../2026-08-07-persistent-workspace-design.md | 616 +++++++++ ...-07-persistent-workspace-implementation.md | 130 ++ docs/runtimes/CONTRACT.md | 3 +- docs/security.md | 10 +- sandbox-images/openclaw/Dockerfile | 3 +- sandbox-images/openclaw/entrypoint.sh | 19 +- .../openclaw/workspace-bootstrap.sh | 87 ++ 23 files changed, 3523 insertions(+), 61 deletions(-) create mode 100644 docs/plans/2026-08-07-persistent-workspace-design.md create mode 100644 docs/plans/2026-08-07-persistent-workspace-implementation.md create mode 100644 sandbox-images/openclaw/workspace-bootstrap.sh diff --git a/cli/src/commands/add.test.ts b/cli/src/commands/add.test.ts index 417e48867..d5f61c879 100644 --- a/cli/src/commands/add.test.ts +++ b/cli/src/commands/add.test.ts @@ -36,6 +36,12 @@ interface AddOptions { openaiApiKey?: string; learnEgress: boolean; skills?: string; + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; + workspaceBootstrap?: string; + workspaceOverwrite?: "IfMissing" | "Always"; } function defaultOptions(overrides: Partial = {}): AddOptions { @@ -113,6 +119,28 @@ function buildSandboxManifest(name: string, options: AddOptions) { np.egressMode = "Learn"; } + if (options.workspaceStorage || options.workspaceExistingClaim) { + const workspace = options.workspaceExistingClaim + ? { existingClaim: options.workspaceExistingClaim } + : { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy ?? "Retain", + }; + (sandbox.spec as Record).storage = { workspace }; + } + + if (options.workspaceBootstrap) { + const runtime = (sandbox.spec as any).runtime; + runtime.openclaw.workspace = { + bootstrapConfigMapRef: { name: options.workspaceBootstrap }, + overwritePolicy: options.workspaceOverwrite ?? "IfMissing", + }; + } + return sandbox; } @@ -180,6 +208,47 @@ describe("KarsSandbox manifest generation", () => { }); }); + it("configures a dynamically provisioned workspace PVC", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ + workspaceStorage: "20Gi", + workspaceStorageClass: "managed-csi", + workspaceRetainPolicy: "Retain", + }), + ); + expect((manifest.spec as any).storage.workspace).toEqual({ + size: "20Gi", + storageClassName: "managed-csi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }); + }); + + it("references an existing workspace claim without dynamic fields", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ workspaceExistingClaim: "restored-workspace" }), + ); + expect((manifest.spec as any).storage.workspace).toEqual({ + existingClaim: "restored-workspace", + }); + }); + + it("configures OpenClaw workspace bootstrap", () => { + const manifest = buildSandboxManifest( + "my-agent", + defaultOptions({ + workspaceBootstrap: "my-agent-workspace", + workspaceOverwrite: "Always", + }), + ); + expect((manifest.spec as any).runtime.openclaw.workspace).toEqual({ + bootstrapConfigMapRef: { name: "my-agent-workspace" }, + overwritePolicy: "Always", + }); + }); + it("uses default model gpt-4.1 with azure/ prefix", () => { const manifest = buildSandboxManifest("a", defaultOptions()); const spec = manifest.spec as any; diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index b46b459a5..a7f0d5beb 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -26,6 +26,14 @@ export function addCommand(): Command { .option("--isolation ", "Isolation level: standard | enhanced | confidential", "enhanced") .option("--image ", "Custom sandbox image (default: from Helm values; OpenClaw runtime only)") + // ── Workspace storage (all runtimes; bootstrap is OpenClaw-only) ─── + .option("--workspace-storage ", "Create a persistent workspace PVC, e.g. 10Gi") + .option("--workspace-storage-class ", "StorageClass for a generated workspace PVC") + .option("--workspace-existing-claim ", "Use an existing PVC in the generated sandbox namespace") + .option("--workspace-retain-policy ", "Generated PVC deletion policy: Retain | Delete", "Retain") + .option("--workspace-bootstrap ", "[OpenClaw only] ConfigMap containing workspace bootstrap files") + .option("--workspace-overwrite ", "[OpenClaw only] Bootstrap policy: IfMissing | Always", "IfMissing") + // ── Inference budget (all runtimes) ──────────────────────────────── .option("--token-budget-daily ", "Daily token budget (0 = unlimited)", "0") .option("--token-budget-per-request ", "Per-request token limit (0 = unlimited)", "0") @@ -72,6 +80,7 @@ export function addCommand(): Command { .addHelpText("after", ` Flag groups (see --help for details): Core: --runtime, --model, --isolation, --image + Workspace: --workspace-storage, --workspace-existing-claim, --workspace-bootstrap Inference budget: --token-budget-* Governance / net: --governance, --trust-threshold, --policy-profile, --learn-egress Foundry agent: --agent-instructions, --agent-tools @@ -129,6 +138,40 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. console.error(chalk.red(`\n Error: --maf-language is only valid with --runtime microsoft-agent-framework.\n`)); process.exit(1); } + if (options.workspaceStorage && options.workspaceExistingClaim) { + console.error(chalk.red(`\n Error: --workspace-storage and --workspace-existing-claim are mutually exclusive.\n`)); + process.exit(1); + } + if (options.workspaceStorageClass && !options.workspaceStorage) { + console.error(chalk.red(`\n Error: --workspace-storage-class requires --workspace-storage .\n`)); + process.exit(1); + } + if (options.workspaceExistingClaim && options.workspaceRetainPolicy !== "Retain") { + console.error(chalk.red(`\n Error: --workspace-retain-policy applies only to generated PVCs; existing claims are always externally managed.\n`)); + process.exit(1); + } + if (!(["Retain", "Delete"] as const).includes(options.workspaceRetainPolicy)) { + console.error(chalk.red(`\n Error: --workspace-retain-policy must be Retain or Delete.\n`)); + process.exit(1); + } + if (runtimeKind !== "OpenClaw" && options.workspaceBootstrap) { + console.error(chalk.red(`\n Error: --workspace-bootstrap is only valid with --runtime openclaw.\n`)); + process.exit(1); + } + if (!(["IfMissing", "Always"] as const).includes(options.workspaceOverwrite)) { + console.error(chalk.red(`\n Error: --workspace-overwrite must be IfMissing or Always.\n`)); + process.exit(1); + } + if (!options.workspaceBootstrap && options.workspaceOverwrite !== "IfMissing") { + console.error(chalk.red(`\n Error: --workspace-overwrite requires --workspace-bootstrap .\n`)); + process.exit(1); + } + if (options.workspaceStorage && options.workspaceRetainPolicy === "Delete") { + console.log(chalk.yellow(" ⚠ Workspace retain policy is Delete: deleting the KarsSandbox will delete its PVC and data.")); + } + if (!options.workspaceStorage && !options.workspaceExistingClaim) { + console.log(chalk.yellow(" ⚠ Workspace is ephemeral (emptyDir): Pod recreation or suspension deletes sessions and files.")); + } const runtimeBlock = buildRuntimeBlock({ kind: runtimeKind, @@ -139,6 +182,13 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. byoContractVersion: options.byoContractVersion, mafLanguage: options.mafLanguage as "python" | "dotnet", }); + if (options.workspaceBootstrap) { + const openclaw = runtimeBlock.openclaw as Record; + openclaw.workspace = { + bootstrapConfigMapRef: { name: options.workspaceBootstrap }, + overwritePolicy: options.workspaceOverwrite, + }; + } const sandbox: Record = { apiVersion: "kars.azure.com/v1alpha1", @@ -174,6 +224,20 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. }, }; + if (options.workspaceStorage || options.workspaceExistingClaim) { + const workspace = options.workspaceExistingClaim + ? { existingClaim: options.workspaceExistingClaim } + : { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy, + }; + (sandbox.spec as Record).storage = { workspace }; + } + // Add Foundry agent config if provided if (options.agentInstructions || options.agentTools) { const agentSpec: Record = {}; @@ -591,6 +655,18 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. console.log(chalk.dim(` Namespace: ${namespace}`)); console.log(chalk.dim(` Model: ${options.model}`)); console.log(chalk.dim(` Isolation: ${options.isolation}`)); + if (options.workspaceExistingClaim) { + console.log(chalk.dim(` Workspace: existing PVC ${options.workspaceExistingClaim} (externally managed)`)); + } else if (options.workspaceStorage) { + console.log(chalk.dim( + ` Workspace: ${name}-workspace (${options.workspaceStorage}, ${options.workspaceStorageClass || "default StorageClass"}, ${options.workspaceRetainPolicy})`, + )); + } else { + console.log(chalk.yellow(" Workspace: ephemeral emptyDir")); + } + if (options.workspaceBootstrap) { + console.log(chalk.dim(` Bootstrap: ${options.workspaceBootstrap} (${options.workspaceOverwrite})`)); + } if (options.channels) { console.log(chalk.dim(` Channels: ${options.channels}`)); } diff --git a/controller/src/crd.rs b/controller/src/crd.rs index 8eb458c17..d3c677b44 100644 --- a/controller/src/crd.rs +++ b/controller/src/crd.rs @@ -72,6 +72,11 @@ pub struct KarsSandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_ref: Option, + /// Optional per-sandbox storage configuration. When omitted, the runtime + /// workspace remains an ephemeral `emptyDir` for backward compatibility. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + /// Network policy pub network_policy: Option, @@ -128,9 +133,9 @@ pub struct KarsSandboxSpec { /// to `replicas: 0` and stamps the K8s `Suspended=True` Condition /// with reason `SuspendedBySpec`. The namespace, NetworkPolicy, /// ServiceAccount, governance ConfigMaps, and any Azure - /// federated-identity binding are preserved byte-identical, so - /// flipping back to `Some(false)` (or unsetting) restores the - /// agent in-place without losing state. + /// federated-identity binding are preserved byte-identical. Runtime files + /// survive the transition only when `spec.storage.workspace` uses a PVC; + /// the backward-compatible `emptyDir` workspace is deleted with the Pod. /// /// Distinct from `Suspended=True / Reason=OverlayMode` (induced by /// `spec.upstreamCompatibility.sigsAgentSandbox=overlay`), which @@ -155,6 +160,94 @@ pub struct KarsSandboxSpec { pub mesh_auth: Option, } +/// Per-sandbox storage resources. +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct SandboxStorageSpec { + /// Runtime workspace mounted at `/sandbox`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, +} + +/// Persistent workspace configuration. +/// +/// An existing claim is a pure reference. Otherwise omitted dynamic fields +/// resolve to 10Gi, ReadWriteOnce, and Retain. +#[derive(Debug, Serialize, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct WorkspaceStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub existing_claim: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_class_name: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub access_modes: Vec, + #[serde(default)] + pub retain_policy: WorkspaceRetainPolicy, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct RawWorkspaceStorageSpec { + existing_claim: Option, + size: Option, + storage_class_name: Option, + access_modes: Option>, + retain_policy: Option, +} + +impl<'de> Deserialize<'de> for WorkspaceStorageSpec { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = RawWorkspaceStorageSpec::deserialize(deserializer)?; + let references_existing_claim = raw.existing_claim.is_some(); + Ok(Self { + existing_claim: raw.existing_claim, + size: raw + .size + .or_else(|| (!references_existing_claim).then(|| "10Gi".to_string())), + storage_class_name: raw.storage_class_name, + access_modes: raw.access_modes.unwrap_or_else(|| { + if references_existing_claim { + Vec::new() + } else { + vec![PersistentVolumeAccessMode::ReadWriteOnce] + } + }), + retain_policy: raw.retain_policy.unwrap_or_default(), + }) + } +} + +impl Default for WorkspaceStorageSpec { + fn default() -> Self { + Self { + existing_claim: None, + size: Some("10Gi".to_string()), + storage_class_name: None, + access_modes: vec![PersistentVolumeAccessMode::ReadWriteOnce], + retain_policy: WorkspaceRetainPolicy::Retain, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum PersistentVolumeAccessMode { + ReadWriteOnce, + ReadWriteOncePod, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum WorkspaceRetainPolicy { + #[default] + Retain, + Delete, +} + /// Per-sandbox mesh authentication mode. /// /// Two terminal modes are supported: @@ -849,6 +942,9 @@ pub struct OpenClawConfig { pub version: Option, pub image: Option, pub config: Option, + /// Declarative initialization for selected OpenClaw workspace files. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, /// Extra environment variables injected into the openclaw container as `key: value` /// pairs. Used by the controller to propagate offload parameters /// (`OFFLOAD_REQUEST_ID`, `OFFLOAD_PARENT_AMID`, `OFFLOAD_TASK`, @@ -857,6 +953,22 @@ pub struct OpenClawConfig { pub extra_env: Option>, } +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct OpenClawWorkspaceSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bootstrap_config_map_ref: Option, + #[serde(default)] + pub overwrite_policy: WorkspaceOverwritePolicy, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum WorkspaceOverwritePolicy { + #[default] + IfMissing, + Always, +} + #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SandboxConfig { @@ -1393,12 +1505,69 @@ mod tests { // empty-name on apply. assert!(spec.inference_ref.name.is_empty()); assert!(spec.network_policy.is_none()); + assert!(spec.storage.is_none()); assert!(spec.agent.is_none()); assert!(spec.governance.is_none()); assert!(spec.azure_services.is_none()); assert!(spec.resources.is_none()); } + #[test] + fn dynamic_workspace_storage_defaults_and_uses_camel_case() { + let workspace: WorkspaceStorageSpec = serde_json::from_value(serde_json::json!({})) + .expect("empty workspace storage uses safe defaults"); + assert_eq!(workspace.size.as_deref(), Some("10Gi")); + assert_eq!( + workspace.access_modes, + vec![PersistentVolumeAccessMode::ReadWriteOnce] + ); + assert_eq!(workspace.retain_policy, WorkspaceRetainPolicy::Retain); + assert!(workspace.existing_claim.is_none()); + + let value = serde_json::to_value(SandboxStorageSpec { + workspace: Some(workspace), + }) + .unwrap(); + let workspace = &value["workspace"]; + assert_eq!(workspace["size"], "10Gi"); + assert_eq!( + workspace["accessModes"], + serde_json::json!(["ReadWriteOnce"]) + ); + assert_eq!(workspace["retainPolicy"], "Retain"); + assert!(workspace.get("existingClaim").is_none()); + } + + #[test] + fn existing_claim_and_openclaw_bootstrap_round_trip() { + let storage: SandboxStorageSpec = serde_json::from_value(serde_json::json!({ + "workspace": { "existingClaim": "restored-workspace" } + })) + .unwrap(); + let workspace = storage.workspace.expect("workspace storage"); + assert_eq!( + workspace.existing_claim.as_deref(), + Some("restored-workspace") + ); + + let cfg: OpenClawConfig = serde_json::from_value(serde_json::json!({ + "workspace": { + "bootstrapConfigMapRef": { "name": "teaching-agent-workspace" }, + "overwritePolicy": "IfMissing" + } + })) + .unwrap(); + let workspace = cfg.workspace.expect("OpenClaw workspace config"); + assert_eq!( + workspace.bootstrap_config_map_ref.unwrap().name, + "teaching-agent-workspace" + ); + assert_eq!( + workspace.overwrite_policy, + WorkspaceOverwritePolicy::IfMissing + ); + } + #[test] fn sandbox_config_security_defaults() { let cfg = SandboxConfig::default(); diff --git a/controller/src/crd_validations.rs b/controller/src/crd_validations.rs index bf121b5cd..d626a5da9 100644 --- a/controller/src/crd_validations.rs +++ b/controller/src/crd_validations.rs @@ -752,18 +752,52 @@ pub fn kars_sre_action_crd() -> CustomResourceDefinition { .expect("kube-rs derive must produce a spec property on KarsSREAction") } -/// `KarsSandbox` CRD as produced by the kube-rs derive. -/// -/// Currently no `kars_sandbox_validations()` helper exists — `KarsSandbox` -/// has historically relied on its hand-written -/// `deploy/helm/kars/templates/crd.yaml` (with rich CEL rules baked -/// in there) rather than rule-injection in code. This helper is exposed -/// so future drift tests / dumpers can compare the kube-rs-derived -/// schema to the hand-written one without each call site reimplementing -/// the `KarsSandbox::crd()` invocation. +fn inject_kars_sandbox_workspace_validations( + mut crd: CustomResourceDefinition, +) -> Option { + let root = crd + .spec + .versions + .first_mut()? + .schema + .as_mut()? + .open_api_v3_schema + .as_mut()?; + let workspace = root + .properties + .as_mut()? + .get_mut("spec")? + .properties + .as_mut()? + .get_mut("storage")? + .properties + .as_mut()? + .get_mut("workspace")?; + workspace.x_kubernetes_validations = Some(vec![ + ValidationRule { + rule: "!has(self.existingClaim) || (!has(self.size) && !has(self.storageClassName) && !has(self.accessModes) && !has(self.retainPolicy))".into(), + message: Some( + "existingClaim is mutually exclusive with dynamic provisioning fields".into(), + ), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ValidationRule { + rule: "!has(self.size) || quantity(self.size).isGreaterThan(quantity('0'))".into(), + message: Some("workspace size must be a positive Kubernetes quantity".into()), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ]); + Some(crd) +} + +/// `KarsSandbox` CRD with workspace storage CEL injected into the generated +/// schema. The hand-written Helm CRD carries the same rule. #[must_use] pub fn kars_sandbox_crd() -> CustomResourceDefinition { - crate::crd::KarsSandbox::crd() + inject_kars_sandbox_workspace_validations(crate::crd::KarsSandbox::crd()) + .expect("kube-rs derive must produce spec.storage.workspace") } #[cfg(test)] diff --git a/controller/src/helm_drift.rs b/controller/src/helm_drift.rs index 7d37ab7b4..e65e407bc 100644 --- a/controller/src/helm_drift.rs +++ b/controller/src/helm_drift.rs @@ -33,9 +33,14 @@ #[cfg(test)] use crate::crd_validations::{ a2a_agent_crd, egress_approval_crd, inference_policy_crd, kars_eval_crd, kars_memory_crd, - kars_sre_action_crd, mcp_server_crd, tool_policy_crd, trust_graph_crd, + kars_sandbox_crd, kars_sre_action_crd, mcp_server_crd, tool_policy_crd, trust_graph_crd, }; +const KARSSANDBOX_HELM_CRD_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../deploy/helm/kars/templates/crd.yaml" +); + const MCP_HELM_CRD_PATH: &str = concat!( env!("CARGO_MANIFEST_DIR"), "/../deploy/helm/kars/templates/crd-mcpserver.yaml" @@ -106,6 +111,7 @@ fn canonical_form(value: &serde_json::Value) -> serde_json::Value { #[cfg(test)] mod tests { use super::*; + use serde::Deserialize; /// One-shot dumper. Run via: /// @@ -181,6 +187,53 @@ mod tests { assert_helm_matches_rust(MCP_HELM_CRD_PATH, rust_crd_value, "mcpserver"); } + #[test] + fn helm_and_rust_expose_workspace_storage_and_bootstrap() { + let rust = serde_json::to_value(kars_sandbox_crd()).expect("rust CRD serializes"); + let helm_text = std::fs::read_to_string(KARSSANDBOX_HELM_CRD_PATH) + .expect("KarsSandbox Helm CRD exists"); + let helm: serde_json::Value = serde_yaml::Deserializer::from_str(&helm_text) + .next() + .map(serde_json::Value::deserialize) + .expect("KarsSandbox document exists") + .expect("KarsSandbox Helm CRD parses"); + + const SPEC: &str = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties"; + for (label, crd) in [("Rust", &rust), ("Helm", &helm)] { + assert!( + crd.pointer(&format!("{SPEC}/storage/properties/workspace")) + .is_some(), + "{label} schema must expose spec.storage.workspace" + ); + assert!( + crd.pointer(&format!( + "{SPEC}/runtime/properties/openclaw/properties/workspace" + )) + .is_some(), + "{label} schema must expose runtime.openclaw.workspace" + ); + } + + for (label, crd) in [("Rust", &rust), ("Helm", &helm)] { + let validations = crd + .pointer(&format!( + "{SPEC}/storage/properties/workspace/x-kubernetes-validations" + )) + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("{label} workspace CEL validations")); + assert!(validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("existingClaim") && rule.contains("size")) + })); + assert!(validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("quantity(self.size)")) + })); + } + } + #[test] fn helm_toolpolicy_crd_matches_rust_schema() { let rust_crd_value = diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index ce919983d..0268d7819 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -14,7 +14,8 @@ use anyhow::Result; use futures::StreamExt; use k8s_openapi::api::{ apps::v1::Deployment, - core::v1::{ConfigMap, Namespace, Secret, Service, ServiceAccount}, + batch::v1::{CronJob, Job}, + core::v1::{ConfigMap, Namespace, PersistentVolumeClaim, Pod, Secret, Service, ServiceAccount}, networking::v1::NetworkPolicy, rbac::v1::ClusterRoleBinding, }; @@ -41,6 +42,746 @@ pub(crate) mod trustgraph_mount; use mcp_egress::mcp_egress_rule; +#[derive(Debug, Clone, PartialEq)] +struct WorkspaceStoragePlan { + volume: serde_json::Value, + claim: Option, +} + +fn build_workspace_storage_plan( + sandbox_name: &str, + sandbox_namespace: &str, + storage: Option<&crate::crd::SandboxStorageSpec>, + sandbox_uid: Option<&str>, +) -> WorkspaceStoragePlan { + let Some(workspace) = storage.and_then(|storage| storage.workspace.as_ref()) else { + return WorkspaceStoragePlan { + volume: json!({"name": "sandbox-data", "emptyDir": {}}), + claim: None, + }; + }; + + if let Some(existing_claim) = workspace.existing_claim.as_deref() { + return WorkspaceStoragePlan { + volume: json!({ + "name": "sandbox-data", + "persistentVolumeClaim": {"claimName": existing_claim} + }), + claim: None, + }; + } + + let claim_name = format!("{sandbox_name}-workspace"); + let retain_policy = match workspace.retain_policy { + crate::crd::WorkspaceRetainPolicy::Retain => "Retain", + crate::crd::WorkspaceRetainPolicy::Delete => "Delete", + }; + let mut metadata = json!({ + "name": claim_name, + "namespace": sandbox_namespace, + "labels": { + "app.kubernetes.io/managed-by": "kars-controller", + "kars.azure.com/sandbox": sandbox_name, + "kars.azure.com/storage-role": "workspace" + }, + "annotations": { + "kars.azure.com/retain-policy": retain_policy + } + }); + if let Some(uid) = sandbox_uid { + metadata["annotations"]["kars.azure.com/sandbox-uid"] = json!(uid); + } + + let access_modes = workspace + .access_modes + .iter() + .map(|mode| match mode { + crate::crd::PersistentVolumeAccessMode::ReadWriteOnce => "ReadWriteOnce", + crate::crd::PersistentVolumeAccessMode::ReadWriteOncePod => "ReadWriteOncePod", + }) + .collect::>(); + let mut spec = json!({ + "accessModes": access_modes, + "resources": { + "requests": { + "storage": workspace.size.as_deref().unwrap_or("10Gi") + } + } + }); + if let Some(storage_class_name) = workspace.storage_class_name.as_deref() { + spec["storageClassName"] = json!(storage_class_name); + } + + WorkspaceStoragePlan { + volume: json!({ + "name": "sandbox-data", + "persistentVolumeClaim": {"claimName": claim_name} + }), + claim: Some(json!({ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": metadata, + "spec": spec + })), + } +} + +fn validate_workspace_storage_spec( + workspace: &crate::crd::WorkspaceStorageSpec, +) -> Result<(), String> { + let mut errors = Vec::new(); + if workspace.existing_claim.is_some() { + if workspace.size.is_some() + || workspace.storage_class_name.is_some() + || !workspace.access_modes.is_empty() + || workspace.retain_policy != crate::crd::WorkspaceRetainPolicy::Retain + { + errors.push( + "existingClaim is mutually exclusive with size, storageClassName, accessModes, and retainPolicy" + .to_string(), + ); + } + } else { + if workspace.size.as_deref().is_none_or(str::is_empty) { + errors.push("dynamic workspace size must not be empty".to_string()); + } + if workspace.access_modes.len() != 1 { + errors.push("dynamic workspace accessModes must contain exactly one mode".to_string()); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + } +} + +fn validate_workspace_claim(claim: &PersistentVolumeClaim) -> Result<(), String> { + let access_modes = claim + .spec + .as_ref() + .and_then(|spec| spec.access_modes.as_ref()) + .cloned() + .unwrap_or_default(); + if access_modes.len() != 1 + || !matches!( + access_modes.first().map(String::as_str), + Some("ReadWriteOnce" | "ReadWriteOncePod") + ) + { + return Err(format!( + "workspace PVC `{}` must use exactly one of ReadWriteOnce or ReadWriteOncePod", + claim.name_any() + )); + } + if claim + .status + .as_ref() + .and_then(|status| status.phase.as_deref()) + == Some("Lost") + { + return Err(format!("workspace PVC `{}` is Lost", claim.name_any())); + } + if claim + .spec + .as_ref() + .and_then(|spec| spec.volume_mode.as_deref()) + .is_some_and(|mode| mode != "Filesystem") + { + return Err(format!( + "workspace PVC `{}` must use Filesystem volumeMode", + claim.name_any() + )); + } + Ok(()) +} + +fn validate_dynamic_claim_provenance( + claim: &PersistentVolumeClaim, + sandbox_uid: Option<&str>, +) -> Result<(), String> { + let recorded_uid = claim + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get("kars.azure.com/sandbox-uid")) + .map(String::as_str); + if sandbox_uid.is_some() && recorded_uid == sandbox_uid { + return Ok(()); + } + Err(format!( + "workspace PVC `{}` belongs to a different sandbox instance; use spec.storage.workspace.existingClaim for explicit recovery", + claim.name_any() + )) +} + +fn workspace_storage_status_conditions( + sandbox: &KarsSandbox, + claim_phase: Option<&str>, +) -> Vec { + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let generation = sandbox.metadata.generation; + let storage = match claim_phase { + None => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::EMPTY_DIR, + "workspace uses ephemeral emptyDir storage", + generation, + ), + Some("Bound") => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CLAIM_BOUND, + "workspace PVC is Bound", + generation, + ), + Some(_) => crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_STORAGE_READY), + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CLAIM_PENDING, + "workspace PVC is waiting to bind", + generation, + ), + }; + + if storage.status == crate::status::conditions::status::TRUE { + vec![storage] + } else { + vec![ + storage, + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_READY), + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace storage", + generation, + ), + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_PROGRESSING), + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CREATING, + "waiting for workspace storage", + generation, + ), + ] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum WorkspaceBootstrapState { + NotConfigured, + Pending, + Ready, + Failed(String), +} + +fn workspace_bootstrap_state(pods: &[Pod]) -> WorkspaceBootstrapState { + let mut saw_ready = false; + let mut saw_pending = false; + for pod in pods { + if pod.metadata.deletion_timestamp.is_some() { + continue; + } + let Some(statuses) = pod + .status + .as_ref() + .and_then(|status| status.init_container_statuses.as_ref()) + else { + saw_pending = true; + continue; + }; + let Some(status) = statuses + .iter() + .find(|status| status.name == "workspace-bootstrap") + else { + saw_pending = true; + continue; + }; + if let Some(terminated) = status + .state + .as_ref() + .and_then(|state| state.terminated.as_ref()) + { + if terminated.exit_code == 0 { + saw_ready = true; + continue; + } + return WorkspaceBootstrapState::Failed( + terminated + .message + .clone() + .or_else(|| terminated.reason.clone()) + .unwrap_or_else(|| format!("exit code {}", terminated.exit_code)), + ); + } + if let Some(terminated) = status + .last_state + .as_ref() + .and_then(|state| state.terminated.as_ref()) + && terminated.exit_code != 0 + && status.restart_count > 0 + { + return WorkspaceBootstrapState::Failed( + terminated + .message + .clone() + .or_else(|| terminated.reason.clone()) + .unwrap_or_else(|| format!("exit code {}", terminated.exit_code)), + ); + } + if let Some(waiting) = status + .state + .as_ref() + .and_then(|state| state.waiting.as_ref()) + && matches!( + waiting.reason.as_deref(), + Some("CreateContainerError" | "RunContainerError") + ) + { + return WorkspaceBootstrapState::Failed( + waiting + .message + .clone() + .or_else(|| waiting.reason.clone()) + .unwrap_or_else(|| "bootstrap container could not start".to_string()), + ); + } + saw_pending = true; + } + if saw_ready && !saw_pending { + WorkspaceBootstrapState::Ready + } else { + WorkspaceBootstrapState::Pending + } +} + +fn workspace_bootstrap_status_conditions( + sandbox: &KarsSandbox, + state: &WorkspaceBootstrapState, +) -> Vec { + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let generation = sandbox.metadata.generation; + let condition = |condition_type: &str, status: &str, reason: &str, message: &str| { + crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, condition_type), + condition_type, + status, + reason, + message, + generation, + ) + }; + + match state { + WorkspaceBootstrapState::NotConfigured => Vec::new(), + WorkspaceBootstrapState::Ready => vec![condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::RECONCILED, + "workspace bootstrap completed", + )], + WorkspaceBootstrapState::Pending => vec![ + condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap init container", + ), + condition( + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap", + ), + condition( + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CREATING, + "waiting for workspace bootstrap", + ), + ], + WorkspaceBootstrapState::Failed(message) => vec![ + condition( + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + message, + ), + condition( + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + "workspace bootstrap failed", + ), + condition( + crate::status::conditions::TYPE_PROGRESSING, + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::FAILED, + "workspace bootstrap cannot make progress", + ), + condition( + crate::status::conditions::TYPE_DEGRADED, + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::BOOTSTRAP_FAILED, + message, + ), + ], + } +} + +#[derive(Debug, Clone, PartialEq)] +struct WorkspaceBootstrapPlan { + volume: serde_json::Value, + init_container: serde_json::Value, +} + +fn build_workspace_bootstrap_plan( + config: &crate::crd::OpenClawConfig, + image: &str, + image_pull_policy: &str, + config_map_uid: &str, + resource_version: &str, +) -> Option { + let workspace = config.workspace.as_ref()?; + let config_map = workspace.bootstrap_config_map_ref.as_ref()?; + let overwrite_policy = match workspace.overwrite_policy { + crate::crd::WorkspaceOverwritePolicy::IfMissing => "IfMissing", + crate::crd::WorkspaceOverwritePolicy::Always => "Always", + }; + Some(WorkspaceBootstrapPlan { + volume: json!({ + "name": "workspace-bootstrap", + "configMap": { + "name": config_map.name, + "defaultMode": 288 + } + }), + init_container: json!({ + "name": "workspace-bootstrap", + "image": image, + "imagePullPolicy": image_pull_policy, + "command": ["/usr/local/bin/workspace-bootstrap.sh"], + "env": [ + { + "name": "KARS_WORKSPACE_OVERWRITE_POLICY", + "value": overwrite_policy + }, + { + "name": "KARS_WORKSPACE_BOOTSTRAP_CONFIG_MAP_UID", + "value": config_map_uid + }, + { + "name": "KARS_WORKSPACE_BOOTSTRAP_RESOURCE_VERSION", + "value": resource_version + } + ], + "securityContext": { + "runAsUser": 1000, + "runAsGroup": 1000, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": [ + {"name": "sandbox-data", "mountPath": "/sandbox"}, + {"name": "workspace-bootstrap", "mountPath": "/etc/kars/workspace-bootstrap", "readOnly": true} + ], + "resources": { + "requests": {"cpu": "5m", "memory": "16Mi"}, + "limits": {"cpu": "100m", "memory": "64Mi"} + } + }), + }) +} + +fn validate_workspace_bootstrap_config_map(config_map: &ConfigMap) -> Result<(), String> { + if config_map + .binary_data + .as_ref() + .is_some_and(|data| !data.is_empty()) + { + return Err("workspace bootstrap ConfigMap must not contain binaryData".to_string()); + } + + const ALLOWED_FILES: &[&str] = &[ + "AGENTS.md", + "SOUL.md", + "HEARTBEAT.md", + "TOOLS.md", + "USER.md", + ]; + if let Some(data) = config_map.data.as_ref() { + let unsupported = data + .keys() + .filter(|key| !ALLOWED_FILES.contains(&key.as_str())) + .cloned() + .collect::>(); + if !unsupported.is_empty() { + return Err(format!( + "workspace bootstrap ConfigMap contains unsupported files: {}", + unsupported.join(", ") + )); + } + } + Ok(()) +} + +fn preserve_namespace_on_delete(storage: Option<&crate::crd::SandboxStorageSpec>) -> bool { + storage + .and_then(|storage| storage.workspace.as_ref()) + .is_some_and(|workspace| { + workspace.existing_claim.is_some() + || workspace.retain_policy == crate::crd::WorkspaceRetainPolicy::Retain + }) +} + +fn should_preserve_namespace_on_delete( + storage: Option<&crate::crd::SandboxStorageSpec>, + claims: &[PersistentVolumeClaim], + _sandbox_uid: Option<&str>, +) -> bool { + preserve_namespace_on_delete(storage) || !claims.is_empty() +} + +fn deletable_workspace_claims( + storage: Option<&crate::crd::SandboxStorageSpec>, + sandbox_name: &str, + claims: &[PersistentVolumeClaim], + sandbox_uid: Option<&str>, +) -> Vec { + let delete_authorized = storage + .and_then(|storage| storage.workspace.as_ref()) + .is_some_and(|workspace| { + workspace.existing_claim.is_none() + && workspace.retain_policy == crate::crd::WorkspaceRetainPolicy::Delete + }); + if !delete_authorized { + return Vec::new(); + } + let generated_claim_name = format!("{sandbox_name}-workspace"); + claims + .iter() + .filter(|claim| { + let annotations = claim.metadata.annotations.as_ref(); + let labels = claim.metadata.labels.as_ref(); + claim.metadata.name.as_deref() == Some(generated_claim_name.as_str()) + && labels + .and_then(|values| values.get("app.kubernetes.io/managed-by")) + .is_some_and(|value| value == "kars-controller") + && labels + .and_then(|values| values.get("kars.azure.com/storage-role")) + .is_some_and(|value| value == "workspace") + && annotations + .and_then(|values| values.get("kars.azure.com/sandbox-uid")) + .map(String::as_str) + == sandbox_uid + }) + .filter_map(|claim| claim.metadata.name.clone()) + .collect() +} + +fn validate_namespace_claim_reuse( + claims: &[PersistentVolumeClaim], + requested_existing_claim: Option<&str>, + current_claim: Option<&str>, + sandbox_uid: Option<&str>, +) -> Result<(), String> { + for claim in claims { + let recorded_uid = claim + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get("kars.azure.com/sandbox-uid")) + .map(String::as_str); + if recorded_uid == sandbox_uid + || requested_existing_claim == claim.metadata.name.as_deref() + || current_claim == claim.metadata.name.as_deref() + { + continue; + } + return Err(format!( + "namespace contains retained PVC `{}` from another sandbox instance; set spec.storage.workspace.existingClaim to recover it explicitly", + claim.name_any() + )); + } + Ok(()) +} + +fn workspace_deployment_strategy(persistent: bool) -> Option { + persistent.then(|| json!({"type": "Recreate"})) +} + +fn deployment_needs_recreate_cleanup(deployment: Option<&Deployment>, recreate: bool) -> bool { + recreate + && deployment + .and_then(|deployment| deployment.spec.as_ref()) + .and_then(|spec| spec.strategy.as_ref()) + .and_then(|strategy| strategy.rolling_update.as_ref()) + .is_some() +} + +fn workspace_desired_replicas( + suspended: bool, + existing_claim_requested: bool, + claim_phase: Option<&str>, +) -> i64 { + if suspended || (existing_claim_requested && claim_phase != Some("Bound")) { + 0 + } else { + 1 + } +} + +fn kube_api_access_mount() -> serde_json::Value { + json!({ + "name": "kube-api-access", + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "readOnly": true + }) +} + +fn kube_api_access_volume() -> serde_json::Value { + json!({ + "name": "kube-api-access", + "projected": { + "defaultMode": 420, + "sources": [ + {"serviceAccountToken": {"expirationSeconds": 3607, "path": "token"}}, + {"configMap": {"name": "kube-root-ca.crt", "items": [{"key": "ca.crt", "path": "ca.crt"}]}}, + {"downwardAPI": {"items": [{"path": "namespace", "fieldRef": {"apiVersion": "v1", "fieldPath": "metadata.namespace"}}]}} + ] + } + }) +} + +const WORKLOAD_IDENTITY_SKIP_CONTAINERS: &str = "egress-guard;workspace-bootstrap"; + +fn validate_workspace_volume_transition( + current_claim: Option<&str>, + desired_claim: Option<&str>, + suspended: bool, +) -> Result<(), String> { + if current_claim == desired_claim || (current_claim.is_none() && desired_claim.is_none()) { + return Ok(()); + } + if suspended { + return Ok(()); + } + Err("changing workspace volume mode or claim requires spec.suspended=true".to_string()) +} + +fn deployment_workspace_claim(deployment: &Deployment) -> Option<&str> { + deployment + .spec + .as_ref()? + .template + .spec + .as_ref()? + .volumes + .as_ref()? + .iter() + .find(|volume| volume.name == "sandbox-data")? + .persistent_volume_claim + .as_ref() + .map(|source| source.claim_name.as_str()) +} + +async fn cleanup_namespaced_sandbox_resources( + client: &Client, + namespace: &str, + sandbox_name: &str, +) -> Result<(), kube::Error> { + let delete = DeleteParams::default(); + let list = ListParams::default().labels(&format!("kars.azure.com/sandbox={sandbox_name}")); + + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + let secret_api = Api::::namespaced(client.clone(), namespace); + match secret_api + .delete(&format!("{sandbox_name}-credentials"), &delete) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(error), + } + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Api::::namespaced(client.clone(), namespace) + .delete_collection(&delete, &list) + .await?; + Ok(()) +} + +async fn stamp_degraded_with_condition( + client: &Client, + sandbox: &KarsSandbox, + name: &str, + condition_type: &'static str, + reason: &'static str, + message: &str, +) { + let mut patch = crate::status::build_degraded_status_patch(sandbox, reason, message); + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let condition = crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, condition_type), + condition_type, + crate::status::conditions::status::FALSE, + reason, + message, + sandbox.metadata.generation, + ); + if let Some(conditions) = patch["status"]["conditions"].as_array_mut() { + conditions.retain(|value| value["type"] != condition_type); + conditions.push(serde_json::to_value(condition).unwrap_or_default()); + } + let api = + Api::::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); + if let Err(error) = api + .patch_status(name, &PatchParams::default(), &Patch::Merge(patch)) + .await + { + tracing::warn!(sandbox = %name, error = %error, "failed to patch dependency-specific degraded status"); + } +} + /// Build pod security context, conditionally including SELinux options and /// choosing between RuntimeDefault and Localhost seccomp profiles. /// For Kata (confidential), we use RuntimeDefault since the VM provides isolation. @@ -401,19 +1142,80 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = Api::all(client.clone()); - match ns_api.delete(&sandbox_ns, &DeleteParams::default()).await { - Ok(_) => tracing::info!("Namespace {sandbox_ns} deletion initiated"), - Err(kube::Error::Api(ae)) if ae.code == 404 => { - tracing::info!("Namespace {sandbox_ns} already gone"); - } - Err(e) => { - tracing::error!(error = %e, "Failed to delete namespace {sandbox_ns}"); + let claim_api: Api = Api::namespaced(client.clone(), &sandbox_ns); + let claims = match claim_api.list(&ListParams::default()).await { + Ok(claims) => claims.items, + Err(kube::Error::Api(error)) if error.code == 404 => Vec::new(), + Err(error) => { + tracing::error!(error = %error, "Failed to inspect PVCs in namespace {sandbox_ns}"); return Ok(Action::requeue(Duration::from_secs(10))); } + }; + let preserve_namespace = should_preserve_namespace_on_delete( + sandbox.spec.storage.as_ref(), + &claims, + sandbox.metadata.uid.as_deref(), + ); + if preserve_namespace { + tracing::info!( + sandbox = %name, + namespace = %sandbox_ns, + "KarsSandbox is being deleted — retaining workspace PVC and namespace" + ); + match cleanup_namespaced_sandbox_resources(client, &sandbox_ns, &name).await { + Ok(()) => tracing::info!( + namespace = %sandbox_ns, + "Kars-managed workload resources deleted; PVCs retained" + ), + Err(kube::Error::Api(error)) if error.code == 404 => { + tracing::info!("Namespace {sandbox_ns} already gone"); + } + Err(error) => { + tracing::error!(error = %error, "Failed to clean namespace {sandbox_ns}"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } + let claim_api: Api = + Api::namespaced(client.clone(), &sandbox_ns); + let deletable_claims = deletable_workspace_claims( + sandbox.spec.storage.as_ref(), + &name, + &claims, + sandbox.metadata.uid.as_deref(), + ); + for claim_name in &deletable_claims { + match claim_api.delete(claim_name, &DeleteParams::default()).await { + Ok(_) => { + tracing::info!(claim = %claim_name, "Delete-policy workspace PVC deletion initiated") + } + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => { + tracing::error!(claim = %claim_name, error = %error, "Failed to delete Delete-policy workspace PVC"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } + } + if !deletable_claims.is_empty() { + return Ok(Action::requeue(Duration::from_secs(2))); + } + } else { + tracing::info!( + "KarsSandbox {name} is being deleted — cleaning up namespace {sandbox_ns}" + ); + + // Ephemeral and Delete-policy workspaces retain the historical + // namespace-level cascade. + let ns_api: Api = Api::all(client.clone()); + match ns_api.delete(&sandbox_ns, &DeleteParams::default()).await { + Ok(_) => tracing::info!("Namespace {sandbox_ns} deletion initiated"), + Err(kube::Error::Api(ae)) if ae.code == 404 => { + tracing::info!("Namespace {sandbox_ns} already gone"); + } + Err(e) => { + tracing::error!(error = %e, "Failed to delete namespace {sandbox_ns}"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } } // Clean up the spawner ClusterRoleBinding @@ -493,9 +1295,13 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox_ns); + let namespace_claims = claim_api.list(&ListParams::default()).await?.items; + let requested_existing_claim = spec + .storage + .as_ref() + .and_then(|storage| storage.workspace.as_ref()) + .and_then(|workspace| workspace.existing_claim.as_deref()); + let current_workspace_claim = if spec.suspended.unwrap_or(false) { + Api::::namespaced(client.clone(), &sandbox_ns) + .get_opt(&name) + .await? + .as_ref() + .and_then(deployment_workspace_claim) + .map(str::to_string) + } else { + None + }; + if let Err(message) = validate_namespace_claim_reuse( + &namespace_claims, + requested_existing_claim, + current_workspace_claim.as_deref(), + sandbox.metadata.uid.as_deref(), + ) { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + + // ── Step 1b: Reconcile per-sandbox workspace storage ──────────────── + // Backward-compatible sandboxes keep the historical emptyDir. A dynamic + // workspace produces a PVC, while existingClaim only changes the pod + // volume and remains entirely operator-owned. + let workspace_storage_plan = build_workspace_storage_plan( + &name, + &sandbox_ns, + spec.storage.as_ref(), + sandbox.metadata.uid.as_deref(), + ); + let workspace_claim = if let Some(claim) = workspace_storage_plan.claim.as_ref() { + let claim: PersistentVolumeClaim = serde_json::from_value(claim.clone())?; + let claim_name = claim.name_any(); + if let Some(existing) = claim_api.get_opt(&claim_name).await? + && let Err(message) = + validate_dynamic_claim_provenance(&existing, sandbox.metadata.uid.as_deref()) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + let applied = match claim_api + .patch( + &claim_name, + &PatchParams::apply(crate::field_managers::CLAWSANDBOX).force(), + &Patch::Apply(&claim), + ) + .await + { + Ok(claim) => claim, + Err(kube::Error::Api(error)) if error.code == 422 => { + let message = format!( + "workspace PVC `{claim_name}` update was rejected: {}", + error.message + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::IMMUTABLE_FIELD_CHANGED, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + Err(error) => return Err(error.into()), + }; + Some(applied) + } else if let Some(existing_claim) = spec + .storage + .as_ref() + .and_then(|storage| storage.workspace.as_ref()) + .and_then(|workspace| workspace.existing_claim.as_deref()) + { + match claim_api.get(existing_claim).await { + Ok(claim) => Some(claim), + Err(kube::Error::Api(error)) if error.code == 404 => { + let message = format!( + "workspace PVC `{existing_claim}` not found in namespace `{sandbox_ns}`" + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_NOT_FOUND, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(30))); + } + Err(error) => return Err(error.into()), + } + } else { + None + }; + if let Some(claim) = workspace_claim.as_ref() + && let Err(message) = validate_workspace_claim(claim) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::CLAIM_INCOMPATIBLE, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + let workspace_claim_phase = workspace_claim + .as_ref() + .and_then(|claim| claim.status.as_ref()) + .and_then(|status| status.phase.clone()); + + // Validate the OpenClaw bootstrap ConfigMap in the KarsSandbox's own + // namespace, then mirror it into the generated runtime namespace. The + // runtime namespace cannot mount a ConfigMap across namespace boundaries. + let mut workspace_bootstrap_provenance: Option<(String, String)> = None; + if let Some(bootstrap_ref) = runtime_spec + .openclaw + .as_ref() + .and_then(|config| config.workspace.as_ref()) + .and_then(|workspace| workspace.bootstrap_config_map_ref.as_ref()) + { + let source_api: Api = Api::namespaced(client.clone(), &sandbox_self_ns); + let source = match source_api.get(&bootstrap_ref.name).await { + Ok(config_map) => config_map, + Err(kube::Error::Api(error)) if error.code == 404 => { + let message = format!( + "workspace bootstrap ConfigMap `{}` not found in namespace `{}`", + bootstrap_ref.name, sandbox_self_ns + ); + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::reason::BOOTSTRAP_CONFIG_NOT_FOUND, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + Err(error) => return Err(error.into()), + }; + if let Err(message) = validate_workspace_bootstrap_config_map(&source) { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_BOOTSTRAP_READY, + crate::status::conditions::reason::BOOTSTRAP_INVALID, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } + workspace_bootstrap_provenance = Some(( + source.metadata.uid.clone().unwrap_or_default(), + source.metadata.resource_version.clone().unwrap_or_default(), + )); + + if sandbox_self_ns != sandbox_ns { + let mirrored: ConfigMap = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": bootstrap_ref.name, + "namespace": sandbox_ns, + "labels": { + "app.kubernetes.io/managed-by": "kars-controller", + "kars.azure.com/sandbox": name, + "kars.azure.com/artifact": "workspace-bootstrap" + }, + "annotations": { + "kars.azure.com/source-namespace": sandbox_self_ns, + "kars.azure.com/source-resource-version": source.metadata.resource_version + } + }, + "data": source.data + }))?; + let target_api: Api = Api::namespaced(client.clone(), &sandbox_ns); + target_api + .patch( + &bootstrap_ref.name, + &PatchParams::apply(crate::field_managers::CLAWSANDBOX).force(), + &Patch::Apply(&mirrored), + ) + .await?; + } + } + // ── Step 2: Create ServiceAccount with Workload Identity ───────────── let sa_api: Api = Api::namespaced(client.clone(), &sandbox_ns); let sa: ServiceAccount = serde_json::from_value(json!({ @@ -1525,7 +2563,11 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result = Api::namespaced(client.clone(), &sandbox_ns); + let desired_workspace_claim = workspace_storage_plan + .volume + .pointer("/persistentVolumeClaim/claimName") + .and_then(serde_json::Value::as_str); + let current_deployment = deploy_api.get_opt(&name).await?; + if let Some(current_deployment) = current_deployment.as_ref() + && let Err(message) = validate_workspace_volume_transition( + deployment_workspace_claim(current_deployment), + desired_workspace_claim, + suspended_by_spec, + ) + { + stamp_degraded_with_condition( + client, + &sandbox, + &name, + crate::status::conditions::TYPE_STORAGE_READY, + crate::status::conditions::reason::IMMUTABLE_FIELD_CHANGED, + &message, + ) + .await; + return Ok(Action::requeue(Duration::from_secs(300))); + } // Token budget values resolved from the InferencePolicy ref above // (hoisted to the top of `reconcile` after S13). 0 = unlimited. @@ -2091,6 +3175,7 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result::namespaced(client.clone(), &sandbox_ns) + .list(&ListParams::default().labels(&format!("kars.azure.com/sandbox={name}"))) + .await?; + workspace_bootstrap_state(&pods.items) + } else { + WorkspaceBootstrapState::NotConfigured + }; + // ── Step 5: Update status ──────────────────────────────────────────── // Idempotency guard: skip the patch when the desired status already // matches reality. Without this, every reconcile bumps @@ -3273,6 +4408,18 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = allowlist_resolution.conditions.clone(); + extras.extend(workspace_storage_status_conditions( + &sandbox, + if workspace_claim.is_some() { + Some(workspace_claim_phase.as_deref().unwrap_or("Pending")) + } else { + None + }, + )); + extras.extend(workspace_bootstrap_status_conditions( + &sandbox, + &workspace_bootstrap_state, + )); // Phase G P1 #4: stamp Suspended condition when spec.suspended // is true, or surface Suspended=False/Active when there is a @@ -3361,7 +4508,15 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result 0" message: "spec.memoryRef.name must not be empty" + storage: + type: object + description: "Optional per-sandbox persistent storage. Omission preserves the legacy emptyDir workspace." + properties: + workspace: + type: object + description: "Workspace storage mounted at /sandbox." + x-kubernetes-validations: + - rule: "!has(self.existingClaim) || (!has(self.size) && !has(self.storageClassName) && !has(self.accessModes) && !has(self.retainPolicy))" + message: "existingClaim is mutually exclusive with dynamic provisioning fields" + - rule: "!has(self.size) || quantity(self.size).isGreaterThan(quantity('0'))" + message: "workspace size must be a positive Kubernetes quantity" + properties: + existingClaim: + type: string + maxLength: 253 + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + description: "Existing same-namespace PVC managed outside Kars." + size: + type: string + storageClassName: + type: string + accessModes: + type: array + minItems: 1 + maxItems: 1 + items: + type: string + enum: ["ReadWriteOnce", "ReadWriteOncePod"] + retainPolicy: + type: string + enum: ["Retain", "Delete"] agent: type: object description: "Foundry Agent Service configuration — controller creates a prompt agent on reconcile" @@ -691,8 +739,10 @@ spec: namespace, NetworkPolicy, ServiceAccount, and all governance overlay objects are preserved byte-identical, so flipping this field back to - `false` (or removing it) restores the agent - in-place. Default `false`. + `false` (or removing it) restores the agent. Runtime + files survive only when `spec.storage.workspace` + uses a PVC; the legacy `emptyDir` is deleted with + the Pod. Default `false`. Distinct from the `Suspended=True / Reason=OverlayMode` transition, which is induced by diff --git a/deploy/helm/kars/templates/rbac.yaml b/deploy/helm/kars/templates/rbac.yaml index efbf5fb3c..dc7227d8f 100644 --- a/deploy/helm/kars/templates/rbac.yaml +++ b/deploy/helm/kars/templates/rbac.yaml @@ -59,10 +59,10 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list", "create", "update", "patch", "delete"] - # Manage pods, services, configmaps in sandbox namespaces + # Manage pods, services, configmaps, and workspace PVCs in sandbox namespaces - apiGroups: [""] - resources: ["pods", "services", "configmaps", "secrets", "serviceaccounts"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + resources: ["pods", "services", "configmaps", "secrets", "serviceaccounts", "persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Read pod logs (for offload result relay) - apiGroups: [""] resources: ["pods/log"] @@ -70,7 +70,7 @@ rules: # Manage deployments - apiGroups: ["apps"] resources: ["deployments"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Slice 3 of kars-sre — typed actions RolloutRestart targets # StatefulSet / DaemonSet as well. Read+patch is sufficient (we # only ever rollout-restart, never create/delete those kinds). @@ -84,11 +84,11 @@ rules: # KarsEval runs jobs and cronjobs to invoke the conformance runner - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Manage network policies - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] # Events — Kubernetes ships two Event APIs: the legacy core v1 # Events ("" apiGroup) and the modern events.k8s.io/v1 Events. # The controller writes to events.k8s.io (the kube-rs Recorder diff --git a/docs/api/conditions.md b/docs/api/conditions.md index 6b253c8e9..e666136bb 100644 --- a/docs/api/conditions.md +++ b/docs/api/conditions.md @@ -21,6 +21,8 @@ These types are reused across CRDs. | `Progressing` | Reconciler is making forward progress toward the spec. | | `Degraded` | Object is partially functional but a sub-resource is failing. | | `Suspended` | Operator has paused the object via `spec.suspended: true`. | +| `StorageReady` | Sandbox workspace storage is available to the runtime. | +| `BootstrapReady` | OpenClaw workspace bootstrap init container completed. | `status: True` means the type predicate holds. For `Degraded`, that means the object **is** degraded; for `Ready`, that it **is** ready. @@ -37,6 +39,15 @@ means the object **is** degraded; for `Ready`, that it **is** ready. | `TimedOut` | most kinds | A wait loop hit its budget. | | `SuspendedBySpec` | `KarsSandbox` | `spec.suspended: true`; Deployment scaled to 0. | | `Active` | `KarsSandbox` | Pairs with `Suspended=False` to clear a prior `SuspendedBySpec`. | +| `EmptyDir` | `KarsSandbox` | Workspace uses the backward-compatible ephemeral volume. | +| `ClaimBound` | `KarsSandbox` | Workspace PVC is bound and available. | +| `ClaimPending` | `KarsSandbox` | Workspace PVC has not bound yet; sandbox remains Creating. | +| `ClaimNotFound` | `KarsSandbox` | Referenced `existingClaim` does not exist. | +| `ClaimIncompatible` | `KarsSandbox` | PVC mode, phase, or provenance is incompatible with the sandbox. | +| `ImmutableFieldChanged` | `KarsSandbox` | Kubernetes rejected an unsafe PVC mutation such as changing StorageClass or access mode. | +| `BootstrapConfigNotFound` | `KarsSandbox` | Referenced workspace bootstrap ConfigMap does not exist. | +| `BootstrapInvalid` | `KarsSandbox` | Bootstrap ConfigMap contains unsupported files or binary data. | +| `BootstrapFailed` | `KarsSandbox` | Bootstrap init container exited unsuccessfully. | ## KarsSandbox @@ -50,6 +61,8 @@ end-to-end runtime. | `Degraded` | True/False | `AuthMisconfigured`, `MemoryStoreMissing`, `FailedClosed` | | `Suspended` | True/False | `SuspendedBySpec`, `Active` | | `RuntimeReady` | True/False | `AdapterMissing` (Falsey when the runtime adapter isn't wired) | +| `StorageReady` | True/False | `EmptyDir`, `ClaimBound`, `ClaimPending` | +| `BootstrapReady` | True/False | `Reconciled`, `Creating`, `BootstrapFailed` | | `AllowlistVerified` | True/False | `Verified`, `Unsigned`, `FailedClosed` | | `AllowlistAuthoritative` | True/False | `Inline`, `Verified`, `StaleLKG`, `FailedClosed`, `InlineDiffersFromArtifact` | | `AllowlistDrift` | True/False | `InlineDiffersFromArtifact`, `InlineCleared` | diff --git a/docs/api/crd-reference.md b/docs/api/crd-reference.md index 3f7c49029..840fe6a1b 100644 --- a/docs/api/crd-reference.md +++ b/docs/api/crd-reference.md @@ -183,6 +183,16 @@ spec: # LangGraph | Anthropic | PydanticAi | BYO openclaw: # block name matches `kind` (CEL-validated) image: karsacr.azurecr.io/kars-runtime-openclaw:latest + workspace: + bootstrapConfigMapRef: + name: my-agent-workspace # AGENTS/SOUL/HEARTBEAT/TOOLS/USER.md only + overwritePolicy: IfMissing # IfMissing (default) | Always + storage: + workspace: + size: 10Gi + storageClassName: managed-csi + accessModes: [ReadWriteOnce] + retainPolicy: Retain # Retain (default) | Delete inferenceRef: name: shared-inference # required: sibling InferencePolicy memoryRef: # optional: sibling KarsMemory @@ -225,6 +235,9 @@ status: | Field | Type | Purpose | |---|---|---| | `spec.memoryRef.name` | LocalObjectRef | Bind to a sibling `KarsMemory` (same namespace). | +| `spec.storage.workspace` | object | Optional PVC-backed `/sandbox`. Omit for legacy `emptyDir`; set `existingClaim` to attach an explicitly selected PVC, or set dynamic fields (`size`, `storageClassName`, `accessModes`, `retainPolicy`). | +| `spec.runtime.openclaw.workspace.bootstrapConfigMapRef.name` | LocalObjectRef | Same-namespace ConfigMap containing only `AGENTS.md`, `SOUL.md`, `HEARTBEAT.md`, `TOOLS.md`, or `USER.md`. The controller mirrors it into the runtime namespace. | +| `spec.runtime.openclaw.workspace.overwritePolicy` | enum | `IfMissing` (default) preserves files already on the PVC; `Always` reapplies declared bootstrap files on every Pod start. | | `spec.sandbox` | object | Isolation primitives — `isolation` (`standard` \| `enhanced`, default `enhanced`), `seccompProfile` (default `kars-strict`), `writablePaths` (default `[/sandbox, /tmp]`). | | `spec.networkPolicy` | object | Baseline egress allowlist. Defaults `defaultDeny: true`, `egressMode: Learn`. | | `spec.governance.enabled` | bool | **Defaults to `true`.** Turn on AGT governance — router guardrails are always on regardless; this gates AGT trust/audit + creates the per-sandbox Service on `:8443` (required for InferencePolicy enforcement + cross-agent mesh DNS). Set to `false` to opt out. | @@ -234,7 +247,7 @@ status: | `spec.governance.registryMode` | string | `local` (default) or `global`. Global enables cross-cluster mesh + handoff tools. | | `spec.governance.trustedPeers` | string | Pre-seeded `"name:AMID,..."` peers — used by sub-agents to auto-trust the spawning parent. | | `spec.a2a` | object | Inbound A2A 1.0.0 ingress configuration. Default off; see [A2A gateway](../architecture/a2a-gateway.md). | -| `spec.suspended` | bool | Operator-driven graceful pause. When `true`, the controller scales the Deployment to `replicas: 0` and stamps `Suspended=True / SuspendedBySpec` without tearing down namespace, NetworkPolicy, or federated credentials. | +| `spec.suspended` | bool | Operator-driven graceful pause. When `true`, the controller scales the Deployment to `replicas: 0`. Runtime files survive only when `spec.storage.workspace` uses a PVC; the legacy `emptyDir` is deleted with the Pod. | **Status** @@ -248,6 +261,15 @@ status: | `status.observedGeneration` | `metadata.generation` that produced this status. Compare against `metadata.generation` to detect stale observations. | | `status.conditions[]` | The full condition chain — every reason emitted by the controller is enumerated in [`docs/api/conditions.md`](conditions.md). | +`StorageReady=True/ClaimBound` means a persistent workspace is bound. While a +claim is pending, the Deployment exists so `WaitForFirstConsumer` storage can +bind, but the sandbox remains `phase=Creating`, `Ready=False`, and +`StorageReady=False/ClaimPending`. + +For a retained workspace, deleting the `KarsSandbox` removes Kars-managed +workloads but leaves the generated namespace and PVC. Reattach it explicitly +with `existingClaim`; a new same-name CR cannot silently adopt a retained claim. + ### `spec.runtime.hermes` (`HermesConfig`) {#hermesconfig} Runtime-kind config block used when `spec.runtime.kind: Hermes`. All fields are optional — defaults give a working smoke-test agent on first boot. diff --git a/docs/api/lifecycle.md b/docs/api/lifecycle.md index 8c4bfb148..4113a18d3 100644 --- a/docs/api/lifecycle.md +++ b/docs/api/lifecycle.md @@ -28,7 +28,7 @@ flowchart LR CLI["kars CLI
or GitOps / kubectl"] CRD[("CRD
(12 kinds)")] Ctrl["kars-controller
(kube-rs)"] - Art[("Cluster artifacts
Namespace · ServiceAccount · NetworkPolicy
Deployment · Service · ConfigMap · Secret
FederatedIdentityCredential")] + Art[("Cluster artifacts
Namespace · ServiceAccount · NetworkPolicy
Deployment · Service · ConfigMap · Secret · optional PVC
FederatedIdentityCredential")] Runtime["Runtime data plane
inference-router · A2A gateway · sandbox pod"] CLI -->|writes| CRD @@ -125,7 +125,7 @@ Every CLI command is a thin wrapper around `kubectl apply`. The CLI does no orch |---|---|---|---| | `kars up` | `KarsSandbox` (one or more) | Tenant namespace + everything inside it | The agent pod itself | | `kars add ` | `KarsSandbox` | Same as above | Same | -| `kars destroy ` | Deletes `KarsSandbox` | Cascades via finalizer to delete the namespace + federated credential | — | +| `kars destroy ` | Deletes `KarsSandbox` | Ephemeral/Delete storage: delete namespace. Retain/existing storage: delete labelled workloads but retain namespace + PVC. Always deletes federated credential. | — | | `kars inferencepolicy apply` | `InferencePolicy` | `ConfigMap` `inferencepolicy--profile` | Inference router (`/v1/chat`, `/v1/responses`) | | `kars toolpolicy apply` | `ToolPolicy` | `ConfigMap` `toolpolicy--profile` | Inference router (every tool dispatch) | | `kars mcp add` | `McpServer` | `Secret` `mcp--signing` (Ed25519 keypair)
`ConfigMap` `mcp--jwks` (when `productionMode=true`) | Inference router (`/mcp` proxy — multi-issuer OAuth verifier + namespaced `{server}.{tool}` dispatch) | @@ -368,7 +368,7 @@ Every reconciler installs a finalizer on first reconcile. This blocks Kubernetes | CRD | Finalizer | Cleanup work | |---|---|---| -| `KarsSandbox` | `kars.azure.com/namespace-cleanup` | Delete the tenant namespace (cascades to all resources), delete spawner ClusterRoleBinding, **delete federated credential**, release pairing slot, then remove finalizer. | +| `KarsSandbox` | `kars.azure.com/namespace-cleanup` | With ephemeral or `retainPolicy: Delete` storage, delete the tenant namespace. With `Retain` or `existingClaim`, delete labelled Kars workloads but preserve namespace and PVC (namespaced owner references cannot cross from the source CR namespace). Then delete spawner ClusterRoleBinding and federated credential, release pairing slot, and remove the finalizer. | | Compile-pattern CRDs | `kars.azure.com/-cleanup` | Delete the produced `ConfigMap` (and `Secret`, where present), then remove finalizer. | The federated-credential delete is the one cleanup that crosses the cluster boundary into Microsoft Graph. See `controller/src/fedcred_reaper.rs` for the orphan-collector that backstops force-delete and pre-finalizer CRs. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3908642f1..3fbda6c7f 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -413,6 +413,12 @@ kars add [options] | `--agent-instructions ` | — | System prompt for the Foundry agent | | `--agent-tools ` | — | Foundry tools: `file_search,web_search,code_interpreter` (comma-separated) | | `--image ` | — | Custom sandbox image (default: from Helm values) | +| `--workspace-storage ` | — | Create a persistent workspace PVC, for example `10Gi` | +| `--workspace-storage-class ` | cluster default | StorageClass for the generated workspace PVC | +| `--workspace-existing-claim ` | — | Attach an existing PVC in `kars-`; mutually exclusive with generated storage | +| `--workspace-retain-policy ` | `Retain` | Generated PVC deletion policy: `Retain` or `Delete` | +| `--workspace-bootstrap ` | — | OpenClaw-only same-namespace ConfigMap for declarative workspace files | +| `--workspace-overwrite ` | `IfMissing` | Bootstrap policy: `IfMissing` or `Always` | | `--governance` | `true` | Enable AGT governance (tool policy, trust, audit) | | `--no-governance` | — | Disable AGT governance | | `--trust-threshold ` | `500` | AGT trust threshold (0–1000) | @@ -444,6 +450,14 @@ kars add researcher --model gpt-4.1 --token-budget-daily 100000 # Add a Telegram-connected agent with enhanced isolation kars add support-bot --channels telegram --telegram-token $TOKEN --isolation enhanced +# Add an OpenClaw agent with a retained 20Gi workspace and bootstrap files +kars add teaching-agent --workspace-storage 20Gi \ + --workspace-storage-class managed-csi \ + --workspace-bootstrap teaching-agent-workspace + +# Explicitly recover a retained workspace +kars add teaching-agent --workspace-existing-claim teaching-agent-workspace + # Add a BYO-runtime agent kars add my-agent --runtime byo --byo-image myacr.azurecr.io/my-agent:latest diff --git a/docs/getting-started.md b/docs/getting-started.md index bc114b875..bb55d9a4b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -315,6 +315,19 @@ kars add another-agent --runtime LangGraph --model gpt-4.1 `kars add` reuses the existing AKS cluster and Foundry project — only the pod is new. See **[CLI reference](cli-reference.md)** for the full surface. +By default `/sandbox` is ephemeral. Enable a retained workspace when sessions +and files must survive Pod recreation or `spec.suspended` scale-to-zero: + +```bash +kars add teaching-agent --runtime openclaw --model gpt-4.1 \ + --workspace-storage 20Gi \ + --workspace-storage-class managed-csi +``` + +This creates one PVC in `kars-teaching-agent`, which incurs storage charges +while retained. A PVC is not a backup; configure snapshots and regional +placement through the selected StorageClass and your platform policy. + ### 2.5a Try a Hermes-runtime sandbox instead The same `kars add` works for [Hermes](https://github.com/NousResearch/hermes-agent), a channels-first agent harness with native MCP support — useful when you want a Telegram or Slack-driven agent without writing the integration: diff --git a/docs/plans/2026-08-07-persistent-workspace-design.md b/docs/plans/2026-08-07-persistent-workspace-design.md new file mode 100644 index 000000000..72fbe8054 --- /dev/null +++ b/docs/plans/2026-08-07-persistent-workspace-design.md @@ -0,0 +1,616 @@ +# Kars Persistent Workspace Design + +**Status:** Approved design + +**Date:** 2026-08-07 + +**Scope:** Per-sandbox persistent workspace storage and declarative OpenClaw workspace bootstrap files. +**Out of scope:** Feishu channel integration, message-triggered wake-up, cross-cluster volume migration, multi-replica RWX, online volume expansion. + +## 1. Problem + +Every `KarsSandbox` runtime currently mounts `/sandbox` from an `emptyDir` volume. OpenClaw stores its workspace, sessions, runtime configuration, pairing state, dynamic bindings, local memory and AgentMesh identity below this directory. The data survives a container restart inside the same Pod but is lost when the Pod is recreated, the Deployment is rolled out, or a suspended sandbox scales from zero back to one. + +The Controller also has no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source. OpenClaw's entrypoint currently rewrites Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. Although `runtime.openclaw.config` exists in the CRD schema, the OpenClaw deployment planner does not consume it. + +## 2. Goals + +1. Give each `KarsSandbox` an optional, dedicated persistent volume mounted at `/sandbox`. +2. Preserve sessions, workspace files, pairing state, dynamic bindings and runtime identity across Pod recreation and scale-to-zero. +3. Let an operator initialize selected OpenClaw workspace Markdown files from a same-namespace ConfigMap. +4. Preserve user changes by default after the initial bootstrap. +5. Retain dynamically provisioned storage by default when a `KarsSandbox` is deleted. +6. Keep sensitive credentials in Kubernetes Secrets and outside workspace ConfigMaps. +7. Maintain backward compatibility: sandboxes without the new storage block continue using `emptyDir`. +8. Surface storage readiness and bootstrap failures through explicit status conditions. + +## 3. Non-goals + +The first version does not: + +- add Feishu or other channel configuration; +- receive IM messages while a sandbox is scaled to zero; +- define a wake gateway or durable message queue; +- share one volume among multiple simultaneously running runtime replicas; +- migrate data between clusters, regions or storage classes; +- continuously reconcile file contents after bootstrap; +- use `runtime.openclaw.config` as an arbitrary pass-through to `openclaw.json`; +- provide backup, snapshot or disaster-recovery orchestration; +- persist `/tmp`; +- replace Foundry Memory Store or another external semantic-memory service. + +## 4. Design principles + +### 4.1 Separate desired configuration from mutable state + +- The CR selects storage behavior and references declarative inputs. +- A ConfigMap contains non-sensitive bootstrap files. +- A Secret contains credentials. +- The PVC contains runtime-mutated state. + +### 4.2 Fail closed on ambiguous storage + +The Controller must not start a sandbox against an unexpected or incompatible claim. Invalid combinations fail admission where possible; missing claims and incompatible claim modes produce a non-running Deployment and an explicit condition. + +### 4.3 Preserve data by default + +Dynamically created claims default to `Retain`. Deleting the `KarsSandbox` removes workload resources but leaves the PVC. Destructive deletion requires an explicit `Delete` policy. + +### 4.4 Bootstrap is initialization, not synchronization + +The default `IfMissing` policy copies each managed file only when the destination does not exist. A ConfigMap update does not overwrite files already modified on the PVC. + +## 5. Proposed API + +### 5.1 KarsSandbox storage + +Add the following optional block to `KarsSandboxSpec`: + +```yaml +apiVersion: kars.azure.com/v1alpha1 +kind: KarsSandbox +metadata: + name: teaching-agent + namespace: kars-teaching-agent +spec: + storage: + workspace: + size: 10Gi + storageClassName: managed-csi + accessModes: + - ReadWriteOnce + retainPolicy: Retain + runtime: + kind: OpenClaw + openclaw: + workspace: + bootstrapConfigMapRef: + name: teaching-agent-workspace + overwritePolicy: IfMissing + inferenceRef: + name: teaching-agent-inference +``` + +Rust schema: + +```rust +pub struct KarsSandboxSpec { + // existing fields... + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, +} + +pub struct SandboxStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace: Option, +} + +pub struct WorkspaceStorageSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub existing_claim: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_class_name: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub access_modes: Option>, + + #[serde(default)] + pub retain_policy: WorkspaceRetainPolicy, +} + +pub enum PersistentVolumeAccessMode { + ReadWriteOnce, + ReadWriteOncePod, +} + +pub enum WorkspaceRetainPolicy { + Retain, + Delete, +} +``` + +Defaults for dynamically provisioned storage: + +| Field | Default | +|---|---| +| `size` | `10Gi` | +| `accessModes` | `[ReadWriteOnce]` | +| `retainPolicy` | `Retain` | +| `storageClassName` | unset; use the cluster default StorageClass | +| generated claim name | `-workspace` | + +When `spec.storage.workspace` is omitted, the Controller preserves the current `emptyDir` behavior. + +### 5.2 Existing claims + +Advanced users may provide a same-namespace claim: + +```yaml +spec: + storage: + workspace: + existingClaim: teaching-agent-imported-workspace +``` + +Rules: + +- `existingClaim` is mutually exclusive with `size`, `storageClassName`, `accessModes`, and `retainPolicy`. +- The claim must exist in the sandbox namespace. +- The Controller never creates, resizes, mutates, adopts or deletes an existing claim. +- The existing claim must advertise `ReadWriteOnce` or `ReadWriteOncePod`. +- A Bound claim is required before the Deployment scales above zero. + +### 5.3 OpenClaw workspace bootstrap + +Extend `OpenClawConfig` with a typed workspace block rather than adding more unstructured fields to `config`: + +```rust +pub struct OpenClawConfig { + pub version: Option, + pub image: Option, + pub config: Option, + pub workspace: Option, + pub extra_env: Option>, +} + +pub struct OpenClawWorkspaceSpec { + pub bootstrap_config_map_ref: Option, + #[serde(default)] + pub overwrite_policy: WorkspaceOverwritePolicy, +} + +pub enum WorkspaceOverwritePolicy { + IfMissing, + Always, +} +``` + +Default `overwritePolicy` is `IfMissing`. + +The reference is same-namespace only. Cross-namespace references are not allowed. + +### 5.4 Allowed bootstrap files + +The bootstrap ConfigMap may contain only these keys: + +- `AGENTS.md` +- `SOUL.md` +- `HEARTBEAT.md` +- `TOOLS.md` +- `USER.md` + +The following are explicitly forbidden: + +- `MEMORY.md` +- `openclaw.json` +- credentials or token files; +- session database files; +- pairing or binding state; +- AgentMesh identity or prekeys; +- arbitrary paths, nested keys, symlinks or executable files. + +`MEMORY.md` is excluded because OpenClaw and Kars mutate it during normal operation. Declaratively overwriting it risks erasing inbox state, Foundry discovery context and user memory. + +### 5.5 ConfigMap example + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: teaching-agent-workspace + namespace: kars-teaching-agent +data: + SOUL.md: | + # Soul + + You are a teaching assistant for the university's internal learning platform. + Protect student data and cite sources when making factual claims. + + HEARTBEAT.md: | + # Heartbeat + + - Check for unfinished tasks. + - Check the AgentMesh inbox. + - If there is no work, return HEARTBEAT_OK. + + AGENTS.md: | + # Agent Instructions + + Follow the institution's teaching and privacy policies. +``` + +## 6. Admission and validation + +The generated CRD and Helm CRD must enforce these constraints with schema and CEL where possible: + +1. `existingClaim` must not appear with dynamic provisioning fields. +2. `size` must parse as a positive Kubernetes quantity. +3. `accessModes` must contain one value and that value must be `ReadWriteOnce` or `ReadWriteOncePod` in v1. +4. `retainPolicy` must be `Retain` or `Delete`. +5. `runtime.openclaw.workspace` is valid only when `runtime.kind == OpenClaw`. +6. `bootstrapConfigMapRef.name` must be a valid Kubernetes object name. +7. `overwritePolicy` must be `IfMissing` or `Always`. +8. Unknown bootstrap file keys cause reconciliation failure; they are not silently ignored. + +Dynamic workspace PVCs do not support `ReadWriteMany` in v1 because each `KarsSandbox` has one active runtime replica and the security model assumes a private per-agent filesystem. + +## 7. Reconciliation + +### 7.1 Dynamic PVC creation + +For a dynamic workspace, the Controller creates a PVC named `-workspace` in the sandbox namespace. + +Required metadata: + +```yaml +metadata: + labels: + kars.azure.com/managed: "true" + kars.azure.com/sandbox: teaching-agent + kars.azure.com/storage-role: workspace + annotations: + kars.azure.com/retain-policy: Retain +``` + +The source `KarsSandbox` and generated PVC live in different namespaces, so a +namespaced owner reference would be invalid. Both policies use labels plus a +`kars.azure.com/sandbox-uid` provenance annotation. `Delete` relies on deletion +of the generated sandbox namespace; `Retain` preserves that namespace and its +PVC while removing labelled workload resources. + +### 7.2 Immutable fields and drift + +The Controller reconciles desired PVC shape without attempting invalid in-place mutations. + +- Increasing `size` may be applied only when the StorageClass allows expansion. +- Decreasing `size` is rejected. +- Changing `storageClassName` is rejected after creation. +- Changing `accessModes` is rejected after creation. +- Changing from a generated claim to `existingClaim`, or the reverse, is rejected while either claim contains active state. + +These errors set `StorageReady=False` and leave the last-known-good Deployment and claim untouched. + +### 7.3 Deployment volume + +When storage is enabled: + +```yaml +volumes: + - name: sandbox-data + persistentVolumeClaim: + claimName: teaching-agent-workspace +``` + +The runtime container continues mounting: + +```yaml +volumeMounts: + - name: sandbox-data + mountPath: /sandbox +``` + +`/tmp` remains a memory-backed `emptyDir`. + +The inference-router does not receive access to the workspace PVC unless a separately reviewed feature requires it. The least-privilege boundary remains unchanged. + +### 7.4 Bootstrap ConfigMap mount + +When a bootstrap ConfigMap is configured, mount it read-only at: + +```text +/etc/kars/workspace-bootstrap +``` + +Do not mount the ConfigMap directly over the writable OpenClaw workspace. + +### 7.5 Bootstrap init container + +Add an init container before the runtime starts. It mounts: + +- `sandbox-data` at `/sandbox`; +- the bootstrap ConfigMap at `/bootstrap`, read-only. + +Its responsibilities are: + +1. Create `/sandbox/.openclaw/workspace` with UID/GID `1000:1000`. +2. Validate that every source entry is an allowed regular file. +3. Reject symlinks and path traversal. +4. Copy via a temporary file in the destination directory. +5. Set file ownership to `1000:1000` and mode `0640`. +6. Atomically rename the temporary file to the destination. +7. Under `IfMissing`, skip existing destinations. +8. Under `Always`, replace allowed destination files. +9. Write a non-sensitive manifest to `/sandbox/.kars/bootstrap-state.json` containing ConfigMap UID, resourceVersion, policy, filenames and SHA-256 digests. + +The init container must not log file contents. + +### 7.6 Default Kars templates + +If no bootstrap ConfigMap is configured, the existing Kars default `AGENTS.md`, `SOUL.md`, and `TOOLS.md` remain available, but entrypoint behavior changes from unconditional overwrite to create-if-missing. + +This preserves current first-run behavior while preventing Pod restarts from replacing user edits on a PVC. + +`HEARTBEAT.md` has no Kars default and is not created unless supplied by the operator or OpenClaw itself. + +### 7.7 Suspended sandboxes + +When `spec.suspended: true`: + +- the Deployment remains at zero replicas; +- the PVC and bootstrap ConfigMap reference remain reconciled; +- no init container runs until the sandbox resumes; +- `StorageReady` can still become true based on PVC state; +- resuming mounts the same claim and restores runtime state. + +The CRD documentation must stop describing state preservation for sandboxes that still use `emptyDir`. The guarantee applies only when persistent workspace storage is configured. + +## 8. Deletion semantics + +### 8.1 Retain + +For `retainPolicy: Retain`: + +1. Delete the Deployment and normal sandbox-owned resources. +2. Leave the PVC and backing PV intact. +3. Add an event and final status message naming the retained claim before the CR disappears. +4. Do not remove the PVC protection finalizer. +5. Do not automatically expose the retained claim to another sandbox. + +A future sandbox may use the retained data only by explicitly setting `existingClaim`. + +### 8.2 Delete + +For `retainPolicy: Delete`: + +- the finalizer deletes the generated sandbox namespace; +- namespace cascading deletion removes the claim; +- backing-volume deletion follows the StorageClass/PV reclaim policy. + +The CLI must show a destructive warning before creating or updating a sandbox to `Delete`. + +### 8.3 Namespace deletion + +Kubernetes namespace deletion can delete both Retain and Delete claims. `retainPolicy: Retain` protects against deletion of the `KarsSandbox`, not deletion of its namespace. Documentation and CLI output must state this explicitly. + +## 9. Status and events + +Add a `StorageReady` condition to `KarsSandbox.status.conditions`. + +| Status | Reason | Meaning | +|---|---|---| +| `True` | `EmptyDir` | Persistence not requested; current ephemeral behavior is active. | +| `True` | `ClaimBound` | Workspace PVC exists and is Bound. | +| `False` | `ClaimPending` | PVC exists but is not Bound. | +| `False` | `ClaimNotFound` | Referenced existing claim does not exist. | +| `False` | `ClaimIncompatible` | Access mode or claim shape is unsupported. | +| `False` | `ImmutableFieldChanged` | Requested storage mutation cannot be applied safely. | +| `False` | `BootstrapConfigNotFound` | Referenced ConfigMap does not exist. | +| `False` | `BootstrapInvalid` | ConfigMap contains unsupported or unsafe entries. | +| `False` | `BootstrapFailed` | Init container failed to initialize the workspace. | + +`Ready=True` requires `StorageReady=True` when persistent storage or bootstrap is configured. + +The Controller emits Kubernetes Events for claim creation, retention, incompatible mutation, missing bootstrap ConfigMap and bootstrap failure. + +## 10. Security + +1. ConfigMap data is non-sensitive. Admission documentation prohibits secrets in workspace bootstrap files. +2. Credentials remain in `-credentials` and are injected through `envFrom` or mounted Secret files. +3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials, service account token or network access. +4. Bootstrap source and destination paths are fixed; user-controlled path fields are not supported. +5. Symlinks are rejected at both source and destination. +6. Atomic writes prevent partially initialized files. +7. The runtime remains UID 1000 with a read-only root filesystem. +8. PVCs are per sandbox and same namespace. Cross-namespace claim references are impossible. +9. A retained PVC is not automatically adopted based only on labels; adoption requires an explicit `existingClaim` name. +10. Backup encryption, StorageClass encryption and customer-managed keys remain operator responsibilities and must be documented. + +## 11. CLI behavior + +Add storage flags to `kars add`: + +```text +--workspace-storage Enable a generated workspace PVC, e.g. 10Gi +--workspace-storage-class Select a StorageClass +--workspace-existing-claim Use a pre-created same-namespace PVC +--workspace-retain-policy Retain|Delete; default Retain +--workspace-bootstrap Initialize OpenClaw workspace files +--workspace-overwrite IfMissing|Always; default IfMissing +``` + +Examples: + +```bash +kars add teaching-agent \ + --workspace-storage 20Gi \ + --workspace-storage-class managed-csi \ + --workspace-bootstrap teaching-agent-workspace +``` + +```bash +kars add teaching-agent \ + --workspace-existing-claim teaching-agent-workspace +``` + +CLI output must show: + +- claim name; +- storage class and requested size; +- retain policy; +- bootstrap ConfigMap and overwrite policy; +- a warning when persistence is omitted; +- a destructive warning for `retainPolicy=Delete`. + +## 12. Backward compatibility and migration + +### 12.1 Existing sandboxes + +Existing CRs remain valid. When `spec.storage.workspace` is absent, the Controller continues producing `emptyDir`. + +No automatic migration occurs because copying a live workspace requires coordination and can produce inconsistent sessions. + +### 12.2 Opt-in migration from emptyDir + +A safe migration procedure is: + +1. Suspend the sandbox. +2. Create the desired PVC. +3. Copy data from a backup or an explicitly captured workspace archive into the claim. +4. Patch the CR to use `existingClaim` or dynamic storage. +5. Resume the sandbox. +6. Verify sessions, workspace files and AgentMesh identity. + +Because `emptyDir` disappears when the Pod is scaled to zero, users must capture data before suspension. The CLI should eventually offer an export/import workflow, but it is outside this spec. + +### 12.3 Entrypoint migration + +Changing default files from always-overwrite to create-if-missing changes restart behavior intentionally. On the first persistent-storage rollout: + +- existing `emptyDir` sandboxes still receive defaults on every new Pod because the directory starts empty; +- persistent sandboxes retain prior edits; +- `systemPromptOverride` remains the authoritative Kars security/welcome instruction unless a later spec explicitly exposes it. + +## 13. Testing + +### 13.1 CRD and schema tests + +Verify: + +- valid dynamic workspace storage is accepted; +- valid `existingClaim` is accepted; +- mixed existing/dynamic fields are rejected; +- invalid access modes are rejected; +- invalid retain and overwrite policies are rejected; +- OpenClaw workspace config is rejected for non-OpenClaw runtimes; +- old CRs without storage remain valid. + +### 13.2 Controller unit tests + +Verify generated resources for: + +- omitted storage produces `emptyDir`; +- dynamic storage produces the expected PVC and claim mount; +- both policies omit cross-namespace owner references and record sandbox UID provenance; +- Delete removes the namespace; Retain preserves namespace + PVC; +- existing claims do not produce a PVC object; +- bootstrap produces ConfigMap volume, mount and init container; +- Router does not mount the workspace claim; +- suspended sandboxes retain PVC reconciliation with replicas zero; +- immutable changes set the expected condition. + +### 13.3 Bootstrap tests + +Verify: + +- `IfMissing` initializes absent files; +- `IfMissing` preserves modified files; +- `Always` replaces allowed files; +- forbidden filenames fail; +- symlink source and destination attacks fail; +- file contents never appear in logs; +- ownership and mode are correct; +- manifest digests match initialized files; +- an interrupted copy cannot leave a partial destination. + +### 13.4 End-to-end tests + +A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath-backed PVC, must prove: + +1. Create a sandbox with persistent workspace and bootstrap ConfigMap. +2. Wait for `StorageReady=True/ClaimBound` and `Ready=True`. +3. Verify `SOUL.md` and `HEARTBEAT.md` exist. +4. Modify `SOUL.md` and create representative session/workspace state through a legitimate runtime-facing flow. +5. Delete the Pod. +6. Verify the replacement Pod sees the modified file and state. +7. Set `spec.suspended=true`, wait for zero replicas, then resume. +8. Verify state remains. +9. Delete a Retain sandbox and verify the PVC remains. +10. Create a new sandbox with `existingClaim` and verify explicit recovery. +11. Delete a Delete-policy sandbox and verify its claim is removed. + +Do not use `kubectl exec` into the agent container in AKS E2E because the validating admission policy correctly blocks that path. Use an approved test runtime, init-container evidence, `kars connect`, or a purpose-built test probe. + +## 14. Observability + +Add metrics: + +```text +kars_workspace_storage_reconcile_total{result,mode} +kars_workspace_storage_ready{sandbox,mode} +kars_workspace_storage_requested_bytes{sandbox} +kars_workspace_bootstrap_total{result,policy} +kars_workspace_bootstrap_files_total{result,policy} +``` + +Do not label metrics with PVC UID, ConfigMap content, user identifiers or filenames if that creates unbounded cardinality. + +Log claim names, mode, condition reason and bootstrap resourceVersion. Never log workspace file contents or Secret values. + +## 15. Documentation updates + +Implementation must update: + +- `docs/api/crd-reference.md` with storage and OpenClaw workspace fields; +- `docs/api/lifecycle.md` with PVC creation, suspension and deletion behavior; +- `docs/runtimes/CONTRACT.md` to distinguish ephemeral and persistent `/sandbox`; +- `docs/security.md` with storage trust boundary and encryption responsibilities; +- `docs/cli-reference.md` with new flags; +- `docs/getting-started.md` with persistence opt-in and cost warning; +- existing CRD comments that currently claim suspension restores state without qualifying storage mode. + +## 16. Acceptance criteria + +The feature is complete when all of the following are true: + +1. An old `KarsSandbox` without storage still runs with `emptyDir`. +2. A sandbox with dynamic storage gets a unique Bound PVC mounted at `/sandbox`. +3. Pod recreation and scale-to-zero preserve OpenClaw workspace and runtime state. +4. A same-namespace existing claim can be explicitly attached without Controller adoption or deletion. +5. Retain is the default and deleting the CR leaves the claim intact. +6. Delete is explicit and removes the generated claim through namespace cascading deletion. +7. A bootstrap ConfigMap initializes only the allowed files. +8. `IfMissing` preserves runtime/user edits across restart. +9. `Always` deterministically reapplies operator content. +10. `MEMORY.md`, credentials and runtime databases cannot be supplied through bootstrap. +11. Missing/incompatible claims and invalid bootstrap data prevent false `Ready=True` and expose actionable conditions. +12. No Secret or workspace content is emitted to logs or status. +13. Controller, CRD, CLI and E2E tests cover the behaviors above. +14. Documentation no longer claims state survives suspension when `/sandbox` is ephemeral. + +## 17. Future extensions + +Potential follow-up specs may add: + +- Feishu and other per-sandbox channel configuration; +- a wake gateway and durable message queue for scale-from-zero; +- volume snapshots, backup and restore; +- controlled workspace export/import; +- storage quota metrics and alerts; +- RWX and active/passive failover; +- per-file bootstrap policy; +- OCI-based signed workspace bundles; +- explicit migration jobs between claims or storage classes. diff --git a/docs/plans/2026-08-07-persistent-workspace-implementation.md b/docs/plans/2026-08-07-persistent-workspace-implementation.md new file mode 100644 index 000000000..2919ee44f --- /dev/null +++ b/docs/plans/2026-08-07-persistent-workspace-implementation.md @@ -0,0 +1,130 @@ +# Persistent Workspace Implementation Plan + +**Status:** Implemented on `feature/persistent-workspace`. + +Implementation notes: + +- Storage/bootstrap pure helpers remain in `controller/src/reconciler/mod.rs` + with focused tests in `controller/src/reconciler/tests.rs`; a separate + `workspace_storage.rs` module was not introduced because the helpers share + the reconciler's status and finalizer vocabulary. +- The bootstrap implementation is the image-baked + `sandbox-images/openclaw/workspace-bootstrap.sh`, tested by executing it + against temporary filesystems (including file and directory symlink attacks). +- Cluster-facing validation uses controller tests, Helm/Rust schema parity, + live API-server CRD dry-run, and generated-manifest checks. A destructive + persistence lifecycle E2E was not added to `tests/e2e/run.sh`; that suite + requires a CSI provisioner and is tracked as follow-up coverage. + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add opt-in per-sandbox persistent `/sandbox` storage and declarative OpenClaw workspace bootstrap files while preserving the existing `emptyDir` default. + +**Architecture:** Typed CRD fields describe dynamic or existing workspace claims and an optional same-namespace bootstrap ConfigMap. The Controller builds a pure workspace plan, reconciles storage before the Deployment, mounts the resulting volume only into the runtime container, and initializes approved Markdown files through a least-privilege init container. Storage readiness is surfaced independently and gates overall readiness. + +**Tech Stack:** Rust 2024, kube-rs, k8s-openapi, schemars, Kubernetes PVC/ConfigMap/Deployment APIs, Helm CRD OpenAPI/CEL, shell entrypoint tests. + +--- + +### Task 1: CRD storage and workspace schema + +**Files:** +- Modify: `controller/src/crd.rs` +- Modify: `deploy/helm/kars/templates/crd.yaml` +- Test: `controller/src/crd.rs` + +1. Add failing serialization/default tests for dynamic storage, existing claims and OpenClaw bootstrap configuration. +2. Add typed Rust schema with `Retain`, `ReadWriteOnce`, and `IfMissing` defaults. +3. Add matching Helm OpenAPI fields and CEL mutual-exclusion validation. +4. Run focused controller CRD tests and schema diagnostics. + +### Task 2: Pure workspace resource planning + +**Files:** +- Create: `controller/src/reconciler/workspace_storage.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` + +1. Add failing tests for ephemeral, dynamically provisioned and existing-claim plans. +2. Implement a pure planner that returns the `sandbox-data` volume source and optional PVC object. +3. Verify Retain/Delete ownership metadata and immutable field validation. +4. Run focused planner tests. + +### Task 3: PVC reconciliation and Pod mount + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Modify: `controller/src/reconciler/tests.rs` + +1. Add failing tests for the generated Pod volume and runtime-only mount. +2. Apply dynamic PVCs after namespace creation and before Deployment reconciliation. +3. Resolve existing claims and block Deployment readiness on missing/incompatible claims. +4. Replace only `sandbox-data.emptyDir` with `persistentVolumeClaim` when configured. +5. Verify existing pod-shape tests and new storage tests. + +### Task 4: Bootstrap ConfigMap and init container + +**Files:** +- Modify: `controller/src/reconciler/workspace_storage.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` + +1. Add failing tests for allowed file validation, unsafe keys and init-container security shape. +2. Resolve the same-namespace ConfigMap before Deployment apply. +3. Add read-only bootstrap volume and a non-networked, non-privileged init container. +4. Implement `IfMissing` and `Always` atomic-copy behavior without logging contents. +5. Verify generated Pod JSON and security settings. + +### Task 5: StorageReady status gating + +**Files:** +- Modify: `controller/src/status/conditions.rs` +- Modify: `controller/src/status/mod.rs` +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/status/mod.rs` + +1. Add failing tests preventing `Ready=True` together with `StorageReady=False`. +2. Add StorageReady reasons and status construction. +3. Gate Deployment/Running status on claim and bootstrap readiness. +4. Verify status idempotency and suspended behavior. + +### Task 6: OpenClaw default workspace preservation + +**Files:** +- Modify: `sandbox-images/openclaw/entrypoint.sh` +- Create: `sandbox-images/openclaw/testM_workspace_defaults.sh` + +1. Add a failing shell regression proving existing `AGENTS.md`, `SOUL.md`, and `TOOLS.md` survive initialization. +2. Extract create-if-missing helpers and preserve current first-run templates. +3. Verify shell syntax and the regression script. + +### Task 7: Retain deletion semantics + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Test: `controller/src/reconciler/workspace_storage.rs` +- Modify: `docs/plans/2026-08-07-persistent-workspace-design.md` only if the feasible namespace ownership model differs from the approved design. + +1. Add a failing test that demonstrates namespace deletion destroys an in-namespace retained claim. +2. Implement a feasible retention model before claiming `Retain` support. Do not rely solely on owner-reference omission. +3. Ensure `Delete` remains explicit and destructive. +4. Verify CR deletion, namespace deletion warning, and explicit recovery behavior. + +### Task 8: CLI, docs, and end-to-end validation + +**Files:** +- Modify: `cli/src/commands/add.ts` +- Modify: `cli/src/commands/add.test.ts` +- Modify: `docs/api/crd-reference.md` +- Modify: `docs/api/lifecycle.md` +- Modify: `docs/runtimes/CONTRACT.md` +- Modify: `docs/security.md` +- Modify: `docs/cli-reference.md` +- Modify: `docs/getting-started.md` +- Modify: `tests/e2e/run.sh` + +1. Add failing CLI tests for storage/bootstrap flags and destructive warnings. +2. Generate the new CR fields from CLI options. +3. Update public documentation and remove unconditional state-preservation claims. +4. Add lifecycle E2E coverage using a test PVC without bypassing the agent exec admission policy. +5. Run controller tests, CLI tests/typecheck/lint, shell checks and diff validation. diff --git a/docs/runtimes/CONTRACT.md b/docs/runtimes/CONTRACT.md index 2f6d47936..9f35f1149 100644 --- a/docs/runtimes/CONTRACT.md +++ b/docs/runtimes/CONTRACT.md @@ -130,7 +130,8 @@ The controller mounts these paths into the runtime container. | `/etc/kars/a2a-card/agent.json` (optional) | ConfigMap (A2AAgent compiled) | Router (mounts `/.well-known/agent.json` + `/a2a` routes when present) | ✅ router | | `/etc/kars/trustgraph/projection.json` (optional) | ConfigMap (TrustGraph projection per sandbox) | Runtime + router | ⚠️ TrustGraph reconciler exists; runtime-side consumption is planned | | `/sandbox/agent/` | OCI artifact or git via `spec.openclaw.config.agentCode` (and per-runtime equivalent) | Runtime entrypoint — user-supplied agent code lands here | ✅ all runtimes that support `agentCode` | -| `/sandbox/.openclaw/`, `/sandbox/.hermes/`, etc. | emptyDir | Runtime writable state (sessions, memory cache, prekeys) | ✅ all runtimes; per-runtime subdir | +| `/sandbox/.openclaw/`, `/sandbox/.hermes/`, etc. | `emptyDir` by default; optional per-sandbox PVC via `spec.storage.workspace` | Runtime writable state (sessions, memory cache, prekeys) | ✅ all runtimes; PVC preserves state across Pod recreation and suspension | +| `/etc/kars/workspace-bootstrap` | Same-namespace ConfigMap mirrored by the controller | OpenClaw bootstrap init container | ✅ OpenClaw; allowed files are `AGENTS.md`, `SOUL.md`, `HEARTBEAT.md`, `TOOLS.md`, `USER.md` | | `/tmp` (4 GiB tmpfs by default) | pod spec | Runtime scratch space | ✅ all runtimes | **Read-only root filesystem**: all of `/`, `/usr`, `/opt`, `/etc` (except mounted ConfigMaps/Secrets) is RO on AKS + local-k8s. Runtimes that need writable state under those paths must mirror to `/tmp` at entrypoint time (see `sandbox-images/openclaw/entrypoint.sh:42-52` for the OpenClaw mirror pattern). diff --git a/docs/security.md b/docs/security.md index 5d7927b68..bd616427b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -59,7 +59,15 @@ Applied to every sandbox pod: | User | Non-root (`runAsNonRoot: true`) — agent UID 1000, router UID 1001 | | Privilege escalation | Blocked (`allowPrivilegeEscalation: false`) | | Capabilities | All dropped (`drop: [ALL]`) | -| Writable paths | `/sandbox` and `/tmp` only (emptyDir) | +| Writable paths | `/sandbox` and `/tmp` only. `/sandbox` is `emptyDir` by default or an optional per-sandbox PVC; `/tmp` remains memory-backed `emptyDir`. | + +PVC persistence does not weaken UID, seccomp, read-only-rootfs, or egress +boundaries, but it extends the lifetime of data written by the runtime. The +operator remains responsible for StorageClass encryption, snapshots, backup, +regional placement, and customer-managed keys. `retainPolicy: Retain` protects +against deletion of the `KarsSandbox`; manually deleting the retained namespace +still deletes namespaced claims unless the backing PV policy independently +retains them. ### Layer 4 — Kernel confinement (seccomp) diff --git a/sandbox-images/openclaw/Dockerfile b/sandbox-images/openclaw/Dockerfile index 9a78b039f..962b9c7c6 100644 --- a/sandbox-images/openclaw/Dockerfile +++ b/sandbox-images/openclaw/Dockerfile @@ -175,7 +175,8 @@ RUN mkdir -p /etc/kars/policies /etc/kars/blocklist && \ # Copy entrypoint that auto-configures OpenClaw from mounted secrets COPY sandbox-images/openclaw/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY sandbox-images/openclaw/workspace-bootstrap.sh /usr/local/bin/workspace-bootstrap.sh +RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/workspace-bootstrap.sh # Labels LABEL org.opencontainers.image.title="kars OpenClaw Sandbox" \ diff --git a/sandbox-images/openclaw/entrypoint.sh b/sandbox-images/openclaw/entrypoint.sh index 6a90b173e..78c1088e4 100644 --- a/sandbox-images/openclaw/entrypoint.sh +++ b/sandbox-images/openclaw/entrypoint.sh @@ -383,7 +383,9 @@ else export OPENCLAW_DISABLE_BUNDLED_PLUGINS=1 fi -# Always (re)generate config + workspace seed files on every container start. +# Always regenerate env-driven config on every container start. Workspace seed +# files are created only when missing so PVC-backed user/bootstrap content +# survives restarts. # # Previously this block was guarded by `[ ! -f "$OPENCLAW_CONFIG" ]` for "idempotency", # but on AKS `/sandbox` is a persistent volume and OpenClaw's runtime workspace @@ -1029,8 +1031,10 @@ RCEOF printf '\n# kars env (managed by entrypoint)\n[ -f /sandbox/.kars-env.sh ] && . /sandbox/.kars-env.sh\n' >> /sandbox/.bashrc fi - # Write minimal workspace files so OpenClaw doesn't need onboarding - cat > "$WORKSPACE_DIR/AGENTS.md" << AGENTSEOF + # Write defaults only when a bootstrap init container or prior runtime has + # not already supplied the file. + if [ ! -e "$WORKSPACE_DIR/AGENTS.md" ]; then + cat > "$WORKSPACE_DIR/AGENTS.md" << AGENTSEOF # kars Agent You are a helpful AI assistant running inside an **kars** sandbox — a secure, @@ -1143,9 +1147,11 @@ Network egress starts in **learn mode** — all domains are allowed and recorded The operator can graduate to enforcement with \`kars egress --enforce\`, which promotes learned domains to the allowlist. After that, new domains require approval. AGENTSEOF + fi # Write TOOLS.md describing available Foundry endpoints - cat > "$WORKSPACE_DIR/TOOLS.md" << 'TOOLSEOF' + if [ ! -e "$WORKSPACE_DIR/TOOLS.md" ]; then + cat > "$WORKSPACE_DIR/TOOLS.md" << 'TOOLSEOF' # kars Tools All tools are accessed via the inference router at http://localhost:8443. @@ -1210,8 +1216,10 @@ curl -s -X POST http://localhost:8443/egress/fetch \ **IMPORTANT:** Do NOT use `curl https://...` directly — it will time out. Always use `curl http://localhost:8443/egress/fetch` with the target URL in the body. TOOLSEOF + fi - cat > "$WORKSPACE_DIR/SOUL.md" << SOULEOF + if [ ! -e "$WORKSPACE_DIR/SOUL.md" ]; then + cat > "$WORKSPACE_DIR/SOUL.md" << SOULEOF # Soul You are **kars Agent** — a secure, sandboxed AI assistant powered by Azure AI Foundry. @@ -1239,6 +1247,7 @@ for clarification; interpret the task as given and deliver your best work. For memory: write important facts, preferences, and decisions to memory files so they persist across sessions. Use \`foundry_memory\` for cross-agent/cross-session recall. SOULEOF + fi echo "[kars] OpenClaw configured — model: ${MODEL}, endpoint: ${ENDPOINT}" else diff --git a/sandbox-images/openclaw/workspace-bootstrap.sh b/sandbox-images/openclaw/workspace-bootstrap.sh new file mode 100644 index 000000000..214e85672 --- /dev/null +++ b/sandbox-images/openclaw/workspace-bootstrap.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +set -euo pipefail + +source_dir="${KARS_WORKSPACE_BOOTSTRAP_SOURCE:-/etc/kars/workspace-bootstrap}" +destination_dir="${KARS_WORKSPACE_BOOTSTRAP_DESTINATION:-/sandbox/.openclaw/workspace}" +state_dir="${KARS_WORKSPACE_BOOTSTRAP_STATE_DIR:-/sandbox/.kars}" +policy="${KARS_WORKSPACE_OVERWRITE_POLICY:-IfMissing}" +config_map_uid="${KARS_WORKSPACE_BOOTSTRAP_CONFIG_MAP_UID:-}" +resource_version="${KARS_WORKSPACE_BOOTSTRAP_RESOURCE_VERSION:-}" + +case "$policy" in + IfMissing|Always) ;; + *) + echo "workspace bootstrap received invalid overwrite policy: $policy" >&2 + exit 1 + ;; +esac + +reject_symlink_components() { + local path="$1" + local current="" + local component + if [[ "$path" != /* ]]; then + echo "workspace bootstrap requires an absolute destination path: $path" >&2 + exit 1 + fi + IFS='/' read -r -a components <<< "$path" + for component in "${components[@]}"; do + [ -n "$component" ] || continue + current="$current/$component" + if [ -L "$current" ]; then + echo "workspace bootstrap refused symlink path component: $current" >&2 + exit 1 + fi + if [ -e "$current" ] && [ ! -d "$current" ]; then + echo "workspace bootstrap path component is not a directory: $current" >&2 + exit 1 + fi + done +} + +reject_symlink_components "$destination_dir" +reject_symlink_components "$state_dir" +mkdir -p "$destination_dir" "$state_dir" +umask 027 +manifest_tmp=$(mktemp "$state_dir/.bootstrap-state.XXXXXX") +copy_tmp="" +cleanup() { + [ -z "$copy_tmp" ] || rm -f -- "$copy_tmp" + rm -f -- "$manifest_tmp" +} +trap cleanup EXIT HUP INT TERM + +printf '{"configMapUid":"%s","resourceVersion":"%s","policy":"%s","files":{' \ + "$config_map_uid" "$resource_version" "$policy" > "$manifest_tmp" +first_file=true +for filename in AGENTS.md SOUL.md HEARTBEAT.md TOOLS.md USER.md; do + source_file="$source_dir/$filename" + destination_file="$destination_dir/$filename" + [ -f "$source_file" ] || continue + + if [ -L "$destination_file" ]; then + echo "workspace bootstrap refused symlink destination: $filename" >&2 + exit 1 + fi + if [ "$policy" = "Always" ] || [ ! -e "$destination_file" ]; then + copy_tmp=$(mktemp "$destination_dir/.${filename}.kars-bootstrap.XXXXXX") + cat -- "$source_file" > "$copy_tmp" + chmod 0640 "$copy_tmp" + mv -f -- "$copy_tmp" "$destination_file" + copy_tmp="" + fi + + digest=$(sha256sum "$destination_file" | awk '{print $1}') + if [ "$first_file" = true ]; then + first_file=false + else + printf ',' >> "$manifest_tmp" + fi + printf '"%s":"%s"' "$filename" "$digest" >> "$manifest_tmp" +done +printf '}}\n' >> "$manifest_tmp" +chmod 0640 "$manifest_tmp" +mv -f -- "$manifest_tmp" "$state_dir/bootstrap-state.json" +trap - EXIT HUP INT TERM From d3d45c8aea0b4bd833f5c581649e6aa0321f16b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 9 Aug 2026 04:02:36 +0000 Subject: [PATCH 2/5] docs: align workspace spec with implementation --- .../2026-08-07-persistent-workspace-design.md | 125 ++++++++++++------ 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/docs/plans/2026-08-07-persistent-workspace-design.md b/docs/plans/2026-08-07-persistent-workspace-design.md index 72fbe8054..83fbae9b0 100644 --- a/docs/plans/2026-08-07-persistent-workspace-design.md +++ b/docs/plans/2026-08-07-persistent-workspace-design.md @@ -1,22 +1,24 @@ # Kars Persistent Workspace Design -**Status:** Approved design +**Status:** Implemented in [PR #494](https://github.com/Azure/kars/pull/494); follow-up validation and observability are listed in Sections 13, 14, and 17. **Date:** 2026-08-07 -**Scope:** Per-sandbox persistent workspace storage and declarative OpenClaw workspace bootstrap files. +**Scope:** Runtime-agnostic per-sandbox persistent `/sandbox` storage, plus declarative OpenClaw workspace bootstrap files. **Out of scope:** Feishu channel integration, message-triggered wake-up, cross-cluster volume migration, multi-replica RWX, online volume expansion. ## 1. Problem -Every `KarsSandbox` runtime currently mounts `/sandbox` from an `emptyDir` volume. OpenClaw stores its workspace, sessions, runtime configuration, pairing state, dynamic bindings, local memory and AgentMesh identity below this directory. The data survives a container restart inside the same Pod but is lost when the Pod is recreated, the Deployment is rolled out, or a suspended sandbox scales from zero back to one. +Before this change, every `KarsSandbox` runtime mounted `/sandbox` from an `emptyDir` volume. Runtime state survived a container restart inside the same Pod but was lost when the Pod was recreated, the Deployment rolled out, or a suspended sandbox scaled from zero back to one. -The Controller also has no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source. OpenClaw's entrypoint currently rewrites Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. Although `runtime.openclaw.config` exists in the CRD schema, the OpenClaw deployment planner does not consume it. +The persistent volume is a platform-level capability shared by OpenClaw, Hermes, OpenAI Agents, Microsoft Agent Framework, LangGraph, Anthropic, PydanticAI, and BYO runtimes. It preserves only state the runtime writes below `/sandbox`; the controller cannot make framework state persistent when an adapter stores that state elsewhere or only in memory. + +Before this change, the Controller had no supported way to initialize runtime-owned files such as `SOUL.md` and `HEARTBEAT.md` from a declarative source, and OpenClaw's entrypoint rewrote Kars-provided `AGENTS.md`, `SOUL.md`, and `TOOLS.md` on every startup. The implementation now uses a typed workspace block and create-if-missing defaults; the unrelated unstructured `runtime.openclaw.config` field remains outside this feature. ## 2. Goals -1. Give each `KarsSandbox` an optional, dedicated persistent volume mounted at `/sandbox`. -2. Preserve sessions, workspace files, pairing state, dynamic bindings and runtime identity across Pod recreation and scale-to-zero. +1. Give every supported `KarsSandbox` runtime an optional, dedicated persistent volume mounted at `/sandbox`. +2. Preserve runtime state written below `/sandbox` across Pod recreation and scale-to-zero. 3. Let an operator initialize selected OpenClaw workspace Markdown files from a same-namespace ConfigMap. 4. Preserve user changes by default after the initial bootstrap. 5. Retain dynamically provisioned storage by default when a `KarsSandbox` is deleted. @@ -60,11 +62,13 @@ Dynamically created claims default to `Retain`. Deleting the `KarsSandbox` remov The default `IfMissing` policy copies each managed file only when the destination does not exist. A ConfigMap update does not overwrite files already modified on the PVC. -## 5. Proposed API +## 5. API + +The storage block is runtime-agnostic. The nested OpenClaw workspace block is intentionally runtime-specific because `SOUL.md`, `HEARTBEAT.md`, and related files are OpenClaw contracts, not portable Kars runtime contracts. ### 5.1 KarsSandbox storage -Add the following optional block to `KarsSandboxSpec`: +`KarsSandboxSpec` exposes the following optional block: ```yaml apiVersion: kars.azure.com/v1alpha1 @@ -116,7 +120,7 @@ pub struct WorkspaceStorageSpec { pub storage_class_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub access_modes: Option>, + pub access_modes: Vec, #[serde(default)] pub retain_policy: WorkspaceRetainPolicy, @@ -222,7 +226,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: teaching-agent-workspace - namespace: kars-teaching-agent + namespace: kars-system # same namespace as the KarsSandbox CR data: SOUL.md: | # Soul @@ -245,7 +249,7 @@ data: ## 6. Admission and validation -The generated CRD and Helm CRD must enforce these constraints with schema and CEL where possible: +The generated CRD and Helm CRD enforce these constraints with schema and CEL where possible: 1. `existingClaim` must not appear with dynamic provisioning fields. 2. `size` must parse as a positive Kubernetes quantity. @@ -269,18 +273,24 @@ Required metadata: ```yaml metadata: labels: - kars.azure.com/managed: "true" + app.kubernetes.io/managed-by: kars-controller kars.azure.com/sandbox: teaching-agent kars.azure.com/storage-role: workspace annotations: kars.azure.com/retain-policy: Retain + kars.azure.com/sandbox-uid: ``` The source `KarsSandbox` and generated PVC live in different namespaces, so a -namespaced owner reference would be invalid. Both policies use labels plus a -`kars.azure.com/sandbox-uid` provenance annotation. `Delete` relies on deletion -of the generated sandbox namespace; `Retain` preserves that namespace and its -PVC while removing labelled workload resources. +namespaced owner reference would be invalid. Both policies use strict management +labels plus a `kars.azure.com/sandbox-uid` provenance annotation. + +Any PVC in the generated namespace prevents direct namespace cascading. On +deletion, the controller only deletes a PVC when the current CR still declares +a dynamically generated `retainPolicy: Delete` workspace and the claim name, +management labels, storage-role label, and sandbox UID all match. It then +requeues until the claim is gone. `Retain`, `existingClaim`, and omitted storage +never authorize PVC deletion, even if a claim carries stale annotations. ### 7.2 Immutable fields and drift @@ -290,7 +300,7 @@ The Controller reconciles desired PVC shape without attempting invalid in-place - Decreasing `size` is rejected. - Changing `storageClassName` is rejected after creation. - Changing `accessModes` is rejected after creation. -- Changing from a generated claim to `existingClaim`, or the reverse, is rejected while either claim contains active state. +- Changing from a generated claim to `existingClaim`, the reverse, or changing existing claim names requires `spec.suspended: true`; active transitions fail closed with `StorageReady=False/ImmutableFieldChanged`. These errors set `StorageReady=False` and leave the last-known-good Deployment and claim untouched. @@ -315,6 +325,10 @@ volumeMounts: `/tmp` remains a memory-backed `emptyDir`. +PVC-backed or bootstrap-enabled Deployments use the `Recreate` strategy. This +prevents two runtime Pods from concurrently writing a ReadWriteOnce workspace +and avoids rolling-update multi-attach deadlocks. + The inference-router does not receive access to the workspace PVC unless a separately reviewed feature requires it. The least-privilege boundary remains unchanged. ### 7.4 Bootstrap ConfigMap mount @@ -329,10 +343,10 @@ Do not mount the ConfigMap directly over the writable OpenClaw workspace. ### 7.5 Bootstrap init container -Add an init container before the runtime starts. It mounts: +The controller adds an init container before the runtime starts. It mounts: - `sandbox-data` at `/sandbox`; -- the bootstrap ConfigMap at `/bootstrap`, read-only. +- the bootstrap ConfigMap at `/etc/kars/workspace-bootstrap`, read-only. Its responsibilities are: @@ -348,6 +362,11 @@ Its responsibilities are: The init container must not log file contents. +The Pod disables automatic service-account-token mounting. A projected +Kubernetes token is mounted explicitly into the runtime and router containers, +while `egress-guard` and `workspace-bootstrap` receive neither that token nor an +Azure Workload Identity token. + ### 7.6 Default Kars templates If no bootstrap ConfigMap is configured, the existing Kars default `AGENTS.md`, `SOUL.md`, and `TOOLS.md` remain available, but entrypoint behavior changes from unconditional overwrite to create-if-missing. @@ -376,18 +395,21 @@ For `retainPolicy: Retain`: 1. Delete the Deployment and normal sandbox-owned resources. 2. Leave the PVC and backing PV intact. -3. Add an event and final status message naming the retained claim before the CR disappears. -4. Do not remove the PVC protection finalizer. -5. Do not automatically expose the retained claim to another sandbox. +3. Preserve the generated namespace because PVCs are namespaced resources. +4. Do not automatically expose the retained claim to another sandbox instance. -A future sandbox may use the retained data only by explicitly setting `existingClaim`. +A future sandbox may use the retained data only by explicitly setting +`existingClaim`. Because the claim remains in `kars-`, recovery +normally recreates the same sandbox name. ### 8.2 Delete For `retainPolicy: Delete`: -- the finalizer deletes the generated sandbox namespace; -- namespace cascading deletion removes the claim; +- the current CR spec must still explicitly request a dynamic Delete workspace; +- the controller verifies the generated claim's name, management labels, + storage-role label, and sandbox UID before deleting it; +- the finalizer waits for the claim to disappear, then deletes the namespace; - backing-volume deletion follows the StorageClass/PV reclaim policy. The CLI must show a destructive warning before creating or updating a sandbox to `Delete`. @@ -396,9 +418,9 @@ The CLI must show a destructive warning before creating or updating a sandbox to Kubernetes namespace deletion can delete both Retain and Delete claims. `retainPolicy: Retain` protects against deletion of the `KarsSandbox`, not deletion of its namespace. Documentation and CLI output must state this explicitly. -## 9. Status and events +## 9. Status conditions -Add a `StorageReady` condition to `KarsSandbox.status.conditions`. +The controller adds a `StorageReady` condition to `KarsSandbox.status.conditions`. | Status | Reason | Meaning | |---|---|---| @@ -408,21 +430,28 @@ Add a `StorageReady` condition to `KarsSandbox.status.conditions`. | `False` | `ClaimNotFound` | Referenced existing claim does not exist. | | `False` | `ClaimIncompatible` | Access mode or claim shape is unsupported. | | `False` | `ImmutableFieldChanged` | Requested storage mutation cannot be applied safely. | + +`BootstrapReady` is a separate condition: + +| Status | Reason | Meaning | +|---|---|---| +| `True` | `Reconciled` | The bootstrap init container completed successfully. | +| `False` | `Creating` | The bootstrap Pod/init container is pending. | | `False` | `BootstrapConfigNotFound` | Referenced ConfigMap does not exist. | | `False` | `BootstrapInvalid` | ConfigMap contains unsupported or unsafe entries. | -| `False` | `BootstrapFailed` | Init container failed to initialize the workspace. | +| `False` | `BootstrapFailed` | The init container terminated or crash-looped. | -`Ready=True` requires `StorageReady=True` when persistent storage or bootstrap is configured. +`Ready=True` requires both storage and configured bootstrap initialization to be ready. The controller aggregates all active Pods with `Failed > Pending > Ready` precedence so an old successful Pod cannot hide a new failing rollout. -The Controller emits Kubernetes Events for claim creation, retention, incompatible mutation, missing bootstrap ConfigMap and bootstrap failure. +Kubernetes Events for storage/bootstrap transitions are not part of PR #494 and remain follow-up work. ## 10. Security 1. ConfigMap data is non-sensitive. Admission documentation prohibits secrets in workspace bootstrap files. 2. Credentials remain in `-credentials` and are injected through `envFrom` or mounted Secret files. -3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials, service account token or network access. +3. The init container runs with only the permissions needed to write the workspace volume; it receives no cloud credentials or service-account token. Existing Pod-level NetworkPolicy still governs the Pod network; the bootstrap script itself performs no network operations. 4. Bootstrap source and destination paths are fixed; user-controlled path fields are not supported. -5. Symlinks are rejected at both source and destination. +5. ConfigMap projection symlinks are accepted as read-only sources; destination files and every writable destination/state directory component reject symlinks. 6. Atomic writes prevent partially initialized files. 7. The runtime remains UID 1000 with a read-only root filesystem. 8. PVCs are per sandbox and same namespace. Cross-namespace claim references are impossible. @@ -431,7 +460,7 @@ The Controller emits Kubernetes Events for claim creation, retention, incompatib ## 11. CLI behavior -Add storage flags to `kars add`: +`kars add` exposes these storage flags: ```text --workspace-storage Enable a generated workspace PVC, e.g. 10Gi @@ -515,10 +544,15 @@ Verify generated resources for: - omitted storage produces `emptyDir`; - dynamic storage produces the expected PVC and claim mount; - both policies omit cross-namespace owner references and record sandbox UID provenance; -- Delete removes the namespace; Retain preserves namespace + PVC; +- Delete removes only a strictly matched generated claim, then its namespace; +- Retain, existingClaim, and omitted storage preserve any namespace containing PVCs; - existing claims do not produce a PVC object; - bootstrap produces ConfigMap volume, mount and init container; - Router does not mount the workspace claim; +- bootstrap init containers do not receive Kubernetes or Azure identity tokens; +- PVC/bootstrap Deployments use `Recreate`; +- valid pending existing claims remain `ClaimPending` with zero replicas; +- valid pending dynamic claims retain one consumer Pod for `WaitForFirstConsumer`; - suspended sandboxes retain PVC reconciliation with replicas zero; - immutable changes set the expected condition. @@ -538,6 +572,10 @@ Verify: ### 13.4 End-to-end tests +**Follow-up, not delivered by PR #494.** The PR includes controller/CLI unit +tests, Rust/Helm schema parity, API-server CRD dry-run, and executable bootstrap +filesystem tests. A CSI-backed lifecycle suite should additionally prove: + A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath-backed PVC, must prove: 1. Create a sandbox with persistent workspace and bootstrap ConfigMap. @@ -551,12 +589,14 @@ A local Kind test with a CSI-capable test provisioner, or a pre-created hostPath 9. Delete a Retain sandbox and verify the PVC remains. 10. Create a new sandbox with `existingClaim` and verify explicit recovery. 11. Delete a Delete-policy sandbox and verify its claim is removed. +12. Repeat the write/recreate/suspend sentinel flow for Hermes, LangGraph, and + BYO to verify each adapter places recoverable state below `/sandbox`. Do not use `kubectl exec` into the agent container in AKS E2E because the validating admission policy correctly blocks that path. Use an approved test runtime, init-container evidence, `kars connect`, or a purpose-built test probe. ## 14. Observability -Add metrics: +**Follow-up, not delivered by PR #494.** Proposed metrics: ```text kars_workspace_storage_reconcile_total{result,mode} @@ -572,7 +612,7 @@ Log claim names, mode, condition reason and bootstrap resourceVersion. Never log ## 15. Documentation updates -Implementation must update: +The implementation updates: - `docs/api/crd-reference.md` with storage and OpenClaw workspace fields; - `docs/api/lifecycle.md` with PVC creation, suspension and deletion behavior; @@ -584,21 +624,21 @@ Implementation must update: ## 16. Acceptance criteria -The feature is complete when all of the following are true: +The implemented acceptance criteria are: 1. An old `KarsSandbox` without storage still runs with `emptyDir`. 2. A sandbox with dynamic storage gets a unique Bound PVC mounted at `/sandbox`. -3. Pod recreation and scale-to-zero preserve OpenClaw workspace and runtime state. +3. Pod recreation and scale-to-zero preserve state that any runtime writes below `/sandbox`. 4. A same-namespace existing claim can be explicitly attached without Controller adoption or deletion. 5. Retain is the default and deleting the CR leaves the claim intact. -6. Delete is explicit and removes the generated claim through namespace cascading deletion. +6. Delete is explicit, requires strict current-spec and PVC provenance checks, and removes the generated claim before namespace deletion. 7. A bootstrap ConfigMap initializes only the allowed files. 8. `IfMissing` preserves runtime/user edits across restart. 9. `Always` deterministically reapplies operator content. 10. `MEMORY.md`, credentials and runtime databases cannot be supplied through bootstrap. 11. Missing/incompatible claims and invalid bootstrap data prevent false `Ready=True` and expose actionable conditions. 12. No Secret or workspace content is emitted to logs or status. -13. Controller, CRD, CLI and E2E tests cover the behaviors above. +13. Controller, CRD, CLI, and executable bootstrap tests cover the implemented behaviors; CSI-backed multi-runtime E2E remains tracked follow-up work. 14. Documentation no longer claims state survives suspension when `/sandbox` is ephemeral. ## 17. Future extensions @@ -614,3 +654,8 @@ Potential follow-up specs may add: - per-file bootstrap policy; - OCI-based signed workspace bundles; - explicit migration jobs between claims or storage classes. +- runtime-specific bootstrap contracts for Hermes or other harnesses (rather + than reusing OpenClaw Markdown semantics); +- a CSI-backed persistence matrix for Hermes, LangGraph, BYO, and the remaining + shipping adapters; +- workspace lifecycle Events and Prometheus metrics listed in Section 14. From 92bb3be95af19c796f7bb280b9ae5d09507f7cf2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 9 Aug 2026 04:27:53 +0000 Subject: [PATCH 3/5] feat(runtime): generalize persistent workspace support --- cli/src/commands/add.test.ts | 90 ++++++++++++++---- cli/src/commands/add.ts | 92 ++++++++++++------- cli/src/commands/up.ts | 22 +++++ cli/src/commands/up/sandbox_bringup.ts | 19 ++++ cli/src/lib/testM_workspace_storage.test.ts | 49 ++++++++++ cli/src/lib/workspace-storage.ts | 30 ++++++ controller/src/reconciler/mod.rs | 19 ++++ controller/src/reconciler/tests.rs | 20 ++++ docs/cli-reference.md | 22 +++++ docs/getting-started.md | 5 + .../2026-08-07-persistent-workspace-design.md | 11 +++ docs/runtimes/CONTRACT.md | 25 +++++ 12 files changed, 349 insertions(+), 55 deletions(-) create mode 100644 cli/src/lib/testM_workspace_storage.test.ts create mode 100644 cli/src/lib/workspace-storage.ts diff --git a/cli/src/commands/add.test.ts b/cli/src/commands/add.test.ts index d5f61c879..86ed16600 100644 --- a/cli/src/commands/add.test.ts +++ b/cli/src/commands/add.test.ts @@ -2,6 +2,9 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; +import { validateRuntimeSpecificAddFlags } from "./add.js"; +import { buildRuntimeBlock, type RuntimeKind } from "../runtime.js"; +import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; /** * Tests for the `add` command's sandbox manifest generation logic. @@ -42,6 +45,7 @@ interface AddOptions { workspaceRetainPolicy?: "Retain" | "Delete"; workspaceBootstrap?: string; workspaceOverwrite?: "IfMissing" | "Always"; + runtimeKind?: RuntimeKind; } function defaultOptions(overrides: Partial = {}): AddOptions { @@ -60,19 +64,21 @@ function defaultOptions(overrides: Partial = {}): AddOptions { /** Build the KarsSandbox manifest object (mirrors add.ts action logic). */ function buildSandboxManifest(name: string, options: AddOptions) { + const runtimeKind = options.runtimeKind ?? "OpenClaw"; const sandbox: Record = { apiVersion: "kars.azure.com/v1alpha1", kind: "KarsSandbox", metadata: { name, namespace: "kars-system" }, spec: { - runtime: { - kind: "OpenClaw", - openclaw: { - version: "2026.3.13", - ...(options.image ? { image: options.image } : {}), - config: { agent: { model: `azure/${options.model}` } }, - }, - }, + runtime: buildRuntimeBlock({ + kind: runtimeKind, + openclawVersion: "2026.3.13", + model: options.model, + image: options.image, + byoImage: runtimeKind === "BYO" ? "example.invalid/byo:latest" : undefined, + byoContractVersion: "v1", + mafLanguage: "python", + }), sandbox: { isolation: options.isolation, seccompProfile: options.isolation === "standard" ? "RuntimeDefault" : "kars-strict", @@ -119,18 +125,9 @@ function buildSandboxManifest(name: string, options: AddOptions) { np.egressMode = "Learn"; } - if (options.workspaceStorage || options.workspaceExistingClaim) { - const workspace = options.workspaceExistingClaim - ? { existingClaim: options.workspaceExistingClaim } - : { - size: options.workspaceStorage, - ...(options.workspaceStorageClass - ? { storageClassName: options.workspaceStorageClass } - : {}), - accessModes: ["ReadWriteOnce"], - retainPolicy: options.workspaceRetainPolicy ?? "Retain", - }; - (sandbox.spec as Record).storage = { workspace }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandbox.spec as Record).storage = storage; } if (options.workspaceBootstrap) { @@ -377,6 +374,59 @@ describe("KarsSandbox manifest generation", () => { expect(spec.sandbox.allowPrivilegeEscalation).toBe(false); expect(spec.sandbox.writablePaths).toEqual(["/sandbox", "/tmp"]); }); + + it.each([ + "OpenClaw", + "Hermes", + "OpenAIAgents", + "MicrosoftAgentFramework", + "LangGraph", + "Anthropic", + "PydanticAi", + "BYO", + ] satisfies RuntimeKind[])("configures workspace storage for %s", (runtimeKind) => { + const manifest = buildSandboxManifest( + "persistent-agent", + defaultOptions({ runtimeKind, workspaceStorage: "10Gi" }), + ); + const spec = manifest.spec as any; + expect(spec.runtime.kind).toBe(runtimeKind); + expect(spec.storage.workspace).toEqual({ + size: "10Gi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }); + }); +}); + +describe("runtime-specific add flag validation", () => { + it("allows channels for Hermes", () => { + expect( + validateRuntimeSpecificAddFlags("Hermes", { + channels: "telegram", + telegramToken: "test-token", + }), + ).toEqual([]); + }); + + it("rejects channels for runtimes without channel adapters", () => { + expect( + validateRuntimeSpecificAddFlags("LangGraph", { channels: "telegram" }), + ).toEqual([ + "--channels is only valid with --runtime openclaw or --runtime hermes.", + ]); + }); + + it("keeps skills, plugins, and image overrides OpenClaw-only", () => { + expect( + validateRuntimeSpecificAddFlags("Hermes", { + skills: "browser", + image: "example.invalid/custom:latest", + }), + ).toEqual([ + "--skills, --image are only valid with --runtime openclaw.", + ]); + }); }); describe("channel and plugin secret generation", () => { diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index a7f0d5beb..d02bbe083 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -12,6 +12,56 @@ import { inferenceRefName, toolPolicyRefName, } from "../refs.js"; +import type { RuntimeKind } from "../runtime.js"; +import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; + +export function validateRuntimeSpecificAddFlags( + runtimeKind: RuntimeKind, + options: Record, +): string[] { + const errors: string[] = []; + const channelFlags: Array<[string, unknown]> = [ + ["--channels", options.channels], + ["--telegram-token", options.telegramToken], + ["--telegram-allow-from", options.telegramAllowFrom], + ["--slack-token", options.slackToken], + ["--discord-token", options.discordToken], + ]; + if (runtimeKind !== "OpenClaw" && runtimeKind !== "Hermes") { + const used = channelFlags + .filter(([, value]) => value !== undefined && value !== "" && value !== false) + .map(([flag]) => flag); + if (used.length > 0) { + errors.push( + `${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` + + `--runtime openclaw or --runtime hermes.`, + ); + } + } + + const openClawOnlyFlags: Array<[string, unknown]> = [ + ["--skills", options.skills], + ["--brave-api-key", options.braveApiKey], + ["--tavily-api-key", options.tavilyApiKey], + ["--exa-api-key", options.exaApiKey], + ["--firecrawl-api-key", options.firecrawlApiKey], + ["--perplexity-api-key", options.perplexityApiKey], + ["--openai-api-key", options.openaiApiKey], + ["--image", options.image], + ]; + if (runtimeKind !== "OpenClaw") { + const used = openClawOnlyFlags + .filter(([, value]) => value !== undefined && value !== "" && value !== false) + .map(([flag]) => flag); + if (used.length > 0) { + errors.push( + `${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` + + `--runtime openclaw.`, + ); + } + } + return errors; +} export function addCommand(): Command { const cmd = new Command("add"); @@ -102,29 +152,10 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. // Validate runtime-specific flag combinations before doing any work. // Reject incompatible flags up-front with a clear, actionable error // — better than silently ignoring a user's intent. - const openClawOnlyFlags: Array<[string, unknown]> = [ - ["--channels", options.channels], - ["--telegram-token", options.telegramToken], - ["--telegram-allow-from", options.telegramAllowFrom], - ["--slack-token", options.slackToken], - ["--discord-token", options.discordToken], - ["--skills", options.skills], - ["--brave-api-key", options.braveApiKey], - ["--tavily-api-key", options.tavilyApiKey], - ["--exa-api-key", options.exaApiKey], - ["--firecrawl-api-key", options.firecrawlApiKey], - ["--perplexity-api-key", options.perplexityApiKey], - ["--openai-api-key", options.openaiApiKey], - ["--image", options.image], - ]; - if (runtimeKind !== "OpenClaw") { - const used = openClawOnlyFlags.filter(([, v]) => v !== undefined && v !== "" && v !== false).map(([f]) => f); - if (used.length > 0) { - console.error(chalk.red(`\n Error: ${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with --runtime openclaw.`)); - console.error(chalk.dim(` Channels, skills, and plugin API keys are OpenClaw-specific entrypoint features.`)); - console.error(chalk.dim(` For ${options.runtime}, configure equivalents inside the agent's own code.\n`)); - process.exit(1); - } + const runtimeFlagErrors = validateRuntimeSpecificAddFlags(runtimeKind, options); + if (runtimeFlagErrors.length > 0) { + console.error(chalk.red(`\n Error: ${runtimeFlagErrors.join(" ")}\n`)); + process.exit(1); } if (runtimeKind !== "BYO" && (options.byoImage || (options.byoContractVersion && options.byoContractVersion !== "v1"))) { console.error(chalk.red(`\n Error: --byo-image / --byo-contract-version are only valid with --runtime byo.\n`)); @@ -224,18 +255,9 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. }, }; - if (options.workspaceStorage || options.workspaceExistingClaim) { - const workspace = options.workspaceExistingClaim - ? { existingClaim: options.workspaceExistingClaim } - : { - size: options.workspaceStorage, - ...(options.workspaceStorageClass - ? { storageClassName: options.workspaceStorageClass } - : {}), - accessModes: ["ReadWriteOnce"], - retainPolicy: options.workspaceRetainPolicy, - }; - (sandbox.spec as Record).storage = { workspace }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandbox.spec as Record).storage = storage; } // Add Foundry agent config if provided diff --git a/cli/src/commands/up.ts b/cli/src/commands/up.ts index 8e1a71340..a852f399d 100644 --- a/cli/src/commands/up.ts +++ b/cli/src/commands/up.ts @@ -51,6 +51,11 @@ export function upCommand(): Command { .option("--release [version]", "Import the PUBLIC, signed GHCR release images (ghcr.io/azure/*) into your ACR instead of building from source. Bare --release uses the latest release; pass a tag (e.g. v0.1.4) to pin. No Rust/Docker build needed.") .option("--build", "Build images locally and push to ACR (developer mode)", false) .option("--skip-runtime-images", "Skip building/importing the 7 multi-runtime adapter images (faster first deploy; only OpenClaw + BYO will be runnable)", false) + // ── Initial sandbox workspace ─────────────────────────────────────── + .option("--workspace-storage ", "Create a persistent workspace PVC for the initial sandbox, e.g. 10Gi") + .option("--workspace-storage-class ", "StorageClass for the initial sandbox workspace PVC") + .option("--workspace-existing-claim ", "Use an existing PVC in the initial sandbox namespace") + .option("--workspace-retain-policy ", "Initial workspace deletion policy: Retain | Delete", "Retain") // ── Foundry / Azure OpenAI ──────────────────────────────────────── .option("--foundry-endpoint ", "Existing Azure AI Foundry project endpoint (services.ai.azure.com)") .option("--openai-endpoint ", "Existing Azure OpenAI endpoint (openai.azure.com, derived from Foundry if omitted)") @@ -84,6 +89,7 @@ Flag groups: Cluster / region: --region, --cluster-name, --isolation, --resource-group Infrastructure: --skip-infra, --force-infra, --skip-preflight Images: --source-acr, --build, --skip-runtime-images + Workspace: --workspace-storage, --workspace-existing-claim, --workspace-retain-policy Foundry: --foundry-endpoint, --openai-endpoint Mesh federation: --mesh-peer / --no-mesh-peer, --global-registry, --expose-registry, --mesh-trust=anonymous|entra Output / lifecycle: --dry-run, --upgrade, --from-scratch @@ -115,6 +121,22 @@ Auto-resume: console.error(chalk.red(`\n Error: --policy must be one of: ${policyPresets.join(" | ")} (got "${options.policy}").\n`)); process.exit(1); } + if (options.workspaceStorage && options.workspaceExistingClaim) { + console.error(chalk.red("\n Error: --workspace-storage and --workspace-existing-claim are mutually exclusive.\n")); + process.exit(1); + } + if (options.workspaceStorageClass && !options.workspaceStorage) { + console.error(chalk.red("\n Error: --workspace-storage-class requires --workspace-storage .\n")); + process.exit(1); + } + if (!["Retain", "Delete"].includes(options.workspaceRetainPolicy)) { + console.error(chalk.red("\n Error: --workspace-retain-policy must be Retain or Delete.\n")); + process.exit(1); + } + if (options.workspaceExistingClaim && options.workspaceRetainPolicy !== "Retain") { + console.error(chalk.red("\n Error: --workspace-retain-policy applies only to generated PVCs.\n")); + process.exit(1); + } const { execa } = await import("execa"); diff --git a/cli/src/commands/up/sandbox_bringup.ts b/cli/src/commands/up/sandbox_bringup.ts index 78aee6294..462201b5f 100644 --- a/cli/src/commands/up/sandbox_bringup.ts +++ b/cli/src/commands/up/sandbox_bringup.ts @@ -23,6 +23,7 @@ import { inferenceRefName, toolPolicyRefName, } from "../../refs.js"; +import { buildWorkspaceStorageSpec } from "../../lib/workspace-storage.js"; export interface SandboxBringUpContext { options: { @@ -30,6 +31,10 @@ export interface SandboxBringUpContext { model: string; region: string; isolation: string; + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; [key: string]: unknown; }; baseName: string; @@ -516,6 +521,10 @@ export async function bringUpSandbox(ctx: SandboxBringUpContext): Promise }, }, }; + const storage = buildWorkspaceStorageSpec(options); + if (storage) { + (sandboxManifest.spec as Record).storage = storage; + } // KarsMemory binding — only meaningful with a Foundry project endpoint // (Memory Store is a Foundry feature). Gives the sandbox the same // controller-managed binding `kars dev` creates, instead of relying purely @@ -638,6 +647,16 @@ export async function bringUpSandbox(ctx: SandboxBringUpContext): Promise kvLine("Sandbox", options.name); kvLine("Model", `${options.model} (Azure OpenAI, Entra ID auth)`); kvLine("Isolation", isolationDesc[options.isolation] || options.isolation); + if (options.workspaceExistingClaim) { + kvLine("Workspace", `existing PVC ${String(options.workspaceExistingClaim)}`); + } else if (options.workspaceStorage) { + kvLine( + "Workspace", + `${String(options.workspaceStorage)} (${String(options.workspaceStorageClass || "default StorageClass")}, ${String(options.workspaceRetainPolicy || "Retain")})`, + ); + } else { + kvLine("Workspace", "ephemeral emptyDir"); + } kvLine("Region", options.region); kvLine("Cluster", `${baseName}-aks`); kvLine("ACR", acrLoginServer); diff --git a/cli/src/lib/testM_workspace_storage.test.ts b/cli/src/lib/testM_workspace_storage.test.ts new file mode 100644 index 000000000..da17b9f20 --- /dev/null +++ b/cli/src/lib/testM_workspace_storage.test.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { buildWorkspaceStorageSpec } from "./workspace-storage.js"; + +describe("buildWorkspaceStorageSpec", () => { + it("omits storage when persistence is not requested", () => { + expect(buildWorkspaceStorageSpec({})).toBeUndefined(); + }); + + it("builds a retained dynamic workspace", () => { + expect( + buildWorkspaceStorageSpec({ + workspaceStorage: "20Gi", + workspaceStorageClass: "managed-csi", + workspaceRetainPolicy: "Retain", + }), + ).toEqual({ + workspace: { + size: "20Gi", + storageClassName: "managed-csi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Retain", + }, + }); + }); + + it("builds a destructive dynamic workspace only when requested", () => { + expect( + buildWorkspaceStorageSpec({ + workspaceStorage: "1.5Gi", + workspaceRetainPolicy: "Delete", + }), + ).toEqual({ + workspace: { + size: "1.5Gi", + accessModes: ["ReadWriteOnce"], + retainPolicy: "Delete", + }, + }); + }); + + it("builds a pure existing-claim reference", () => { + expect( + buildWorkspaceStorageSpec({ workspaceExistingClaim: "restored-workspace" }), + ).toEqual({ workspace: { existingClaim: "restored-workspace" } }); + }); +}); \ No newline at end of file diff --git a/cli/src/lib/workspace-storage.ts b/cli/src/lib/workspace-storage.ts new file mode 100644 index 000000000..33c411f82 --- /dev/null +++ b/cli/src/lib/workspace-storage.ts @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export interface WorkspaceStorageOptions { + workspaceStorage?: string; + workspaceStorageClass?: string; + workspaceExistingClaim?: string; + workspaceRetainPolicy?: "Retain" | "Delete"; +} + +export function buildWorkspaceStorageSpec( + options: WorkspaceStorageOptions, +): Record | undefined { + if (options.workspaceExistingClaim) { + return { + workspace: { existingClaim: options.workspaceExistingClaim }, + }; + } + if (!options.workspaceStorage) return undefined; + return { + workspace: { + size: options.workspaceStorage, + ...(options.workspaceStorageClass + ? { storageClassName: options.workspaceStorageClass } + : {}), + accessModes: ["ReadWriteOnce"], + retainPolicy: options.workspaceRetainPolicy ?? "Retain", + }, + }; +} \ No newline at end of file diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 0268d7819..675e4819d 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -648,6 +648,20 @@ fn workspace_desired_replicas( } } +fn runtime_state_dir(kind: &crate::crd::RuntimeKind) -> &'static str { + match kind { + crate::crd::RuntimeKind::OpenClaw => "/sandbox/.openclaw", + crate::crd::RuntimeKind::Hermes => "/sandbox/.hermes", + crate::crd::RuntimeKind::OpenAIAgents => "/sandbox/.openai-agents", + crate::crd::RuntimeKind::MicrosoftAgentFramework => "/sandbox/.maf", + crate::crd::RuntimeKind::LangGraph => "/sandbox/.langgraph", + crate::crd::RuntimeKind::Anthropic => "/sandbox/.anthropic", + crate::crd::RuntimeKind::PydanticAi => "/sandbox/.pydantic-ai", + crate::crd::RuntimeKind::BYO => "/sandbox/.byo", + crate::crd::RuntimeKind::SemanticKernel => "/sandbox/.semantic-kernel", + } +} + fn kube_api_access_mount() -> serde_json::Value { json!({ "name": "kube-api-access", @@ -2676,6 +2690,11 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result` | controller runtime-kind mapping | Recommended private state directory for sessions, checkpoints, identities, and runtime metadata. Adapters may expose a native alias such as `HERMES_HOME`. | ✅ all Kubernetes runtimes | ### Conditionally present @@ -136,6 +138,29 @@ The controller mounts these paths into the runtime container. **Read-only root filesystem**: all of `/`, `/usr`, `/opt`, `/etc` (except mounted ConfigMaps/Secrets) is RO on AKS + local-k8s. Runtimes that need writable state under those paths must mirror to `/tmp` at entrypoint time (see `sandbox-images/openclaw/entrypoint.sh:42-52` for the OpenClaw mirror pattern). +### Persistent workspace contract + +`spec.storage.workspace` is a platform-level contract shared by every wired +runtime. The controller guarantees only that the same filesystem is mounted at +`/sandbox` after Pod recreation or suspension. Each runtime adapter or user +application remains responsible for placing recoverable state there. + +| Runtime | Known state below `/sandbox` | Persistence boundary | +|---|---|---| +| OpenClaw | `/sandbox/.openclaw` | Sessions, workspace, memory cache, pairing/bindings, and mesh identity use the PVC. OpenClaw-only ConfigMap bootstrap is supported. | +| Hermes | `/sandbox/.hermes` and `HOME=/sandbox` | Hermes config, sessions, memory, channel-local state, plugins, and AGT identity use the PVC. No declarative Hermes bootstrap yet. | +| OpenAI Agents | `/sandbox/agent` | User code and files persist. Conversation/session state persists only if the application configures its store/checkpoint under `/sandbox`. | +| Microsoft Agent Framework Python | `/sandbox/agent` | User code and files persist. Thread/conversation persistence remains application-defined. | +| LangGraph Python/TypeScript | `/sandbox/agent` | User code and files persist. Agents must configure a durable checkpointer (for example SQLite under `/sandbox` or an external database); in-memory checkpointers do not become durable merely because a PVC is mounted. | +| Anthropic | `/sandbox/agent` | User code and files persist. SDK session state remains application-defined. | +| PydanticAI | `/sandbox/agent` | User code and files persist. Application message history/state must be written under `/sandbox` or to an external store. | +| BYO | `/sandbox` by contract | The image must write every state item it expects to recover below `/sandbox`; writes elsewhere are not covered. | + +The PVC capability is implemented on Kubernetes targets (AKS and local-k8s). +The Docker `kars dev` target has no `KarsSandbox` CRD or PVC and continues to +use its Docker bind/volume behavior. Do not interpret `--workspace-storage` as +a portable Docker-dev flag. + --- ## HTTP contract — runtime ↔ router From a6cfe5b5a03053f6b368a13435491869a9065667 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 9 Aug 2026 15:47:04 +0000 Subject: [PATCH 4/5] feat(channels): add secure Feishu WebSocket support --- cli/src/commands/add.test.ts | 127 ++- cli/src/commands/add.ts | 224 ++++- cli/src/commands/credentials.test.ts | 143 +++ cli/src/commands/credentials.ts | 306 ++++-- cli/src/commands/operator/dialogs/spawn.ts | 4 +- cli/src/config.test.ts | 2 + cli/src/config.ts | 4 + cli/src/lib/feishu-secret-reference.ts | 28 + controller/src/config_hash.rs | 13 + controller/src/crd.rs | 140 +++ controller/src/crd_validations.rs | 69 +- controller/src/helm_drift.rs | 72 ++ controller/src/reconciler/mod.rs | 898 +++++++++++++++++- controller/src/reconciler/runtime.rs | 103 +- controller/src/reconciler/tests.rs | 501 +++++++++- controller/src/status/conditions.rs | 12 + deploy/helm/kars/templates/crd.yaml | 69 ++ docs/api/conditions.md | 7 + docs/api/crd-reference.md | 25 +- docs/channels-plugins.md | 48 +- docs/cli-reference.md | 37 +- docs/hermes-plugin.md | 9 +- docs/operations/image-versioning.md | 18 + ...26-08-09-feishu-channel-contract-design.md | 706 ++++++++++++++ ...-feishu-channel-contract-implementation.md | 119 +++ docs/runtimes.md | 6 +- docs/security.md | 4 + sandbox-images/hermes/Dockerfile | 14 +- sandbox-images/hermes/entrypoint.sh | 76 +- .../hermes/kars-channel-feishu-ready | 7 + .../hermes/patch-hermes-feishu-policy.py | 142 +++ sandbox-images/hermes/testM_feishu_channel.sh | 95 ++ sandbox-images/openclaw/Dockerfile | 13 +- sandbox-images/openclaw/Dockerfile.base | 20 +- sandbox-images/openclaw/entrypoint.sh | 71 ++ .../openclaw/kars-channel-feishu-ready | 16 + .../openclaw/patch-feishu-proxy.cjs | 98 ++ .../openclaw/testM_feishu_channel.sh | 142 +++ 38 files changed, 4276 insertions(+), 112 deletions(-) create mode 100644 cli/src/commands/credentials.test.ts create mode 100644 cli/src/lib/feishu-secret-reference.ts create mode 100644 docs/plans/2026-08-09-feishu-channel-contract-design.md create mode 100644 docs/plans/2026-08-09-feishu-channel-contract-implementation.md create mode 100644 sandbox-images/hermes/kars-channel-feishu-ready create mode 100644 sandbox-images/hermes/patch-hermes-feishu-policy.py create mode 100644 sandbox-images/hermes/testM_feishu_channel.sh create mode 100644 sandbox-images/openclaw/kars-channel-feishu-ready create mode 100644 sandbox-images/openclaw/patch-feishu-proxy.cjs create mode 100644 sandbox-images/openclaw/testM_feishu_channel.sh diff --git a/cli/src/commands/add.test.ts b/cli/src/commands/add.test.ts index 86ed16600..e0dab3c68 100644 --- a/cli/src/commands/add.test.ts +++ b/cli/src/commands/add.test.ts @@ -2,7 +2,12 @@ // Licensed under the MIT License. import { describe, it, expect } from "vitest"; -import { validateRuntimeSpecificAddFlags } from "./add.js"; +import { + buildFeishuChannelSpec, + buildFeishuSecrets, + buildCredentialSecretManifest, + validateRuntimeSpecificAddFlags, +} from "./add.js"; import { buildRuntimeBlock, type RuntimeKind } from "../runtime.js"; import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; @@ -31,6 +36,14 @@ interface AddOptions { telegramToken?: string; slackToken?: string; discordToken?: string; + feishuAppId?: string; + feishuAppSecret?: string; + feishuDomain?: "feishu" | "lark"; + feishuDmPolicy?: "pairing" | "allowlist" | "disabled"; + feishuAllowFrom?: string; + feishuGroupPolicy?: "allowlist" | "disabled"; + feishuGroupAllowFrom?: string; + feishuRequireMention?: boolean; braveApiKey?: string; tavilyApiKey?: string; exaApiKey?: string; @@ -191,6 +204,32 @@ function buildSecrets(options: AddOptions) { // --- Tests --- describe("KarsSandbox manifest generation", () => { + it("does not treat Feishu policy defaults as explicitly selected flags", () => { + expect(validateRuntimeSpecificAddFlags("OpenClaw", { + feishuDomain: "feishu", + feishuDmPolicy: "pairing", + feishuGroupPolicy: "allowlist", + feishuRequireMention: true, + })).toEqual([]); + }); + + it("builds an immutable managed Feishu Secret manifest", () => { + expect(buildCredentialSecretManifest( + "agent-feishu-credentials", + "kars-agent", + { FEISHU_APP_ID: "cli_test", FEISHU_APP_SECRET: "secret" }, + { + immutable: true, + labels: { "channels.kars.azure.com/managed-rotation": "true" }, + }, + )).toMatchObject({ + immutable: true, + metadata: { + labels: { "channels.kars.azure.com/managed-rotation": "true" }, + }, + }); + }); + it("generates correct apiVersion and kind", () => { const manifest = buildSandboxManifest("agent1", defaultOptions()); expect(manifest.apiVersion).toBe("kars.azure.com/v1alpha1"); @@ -429,6 +468,92 @@ describe("runtime-specific add flag validation", () => { }); }); +describe("Feishu channel contract", () => { + it.each(["OpenClaw", "Hermes"] satisfies RuntimeKind[])( + "builds typed policy for %s without credentials", + (runtimeKind) => { + const channel = buildFeishuChannelSpec(runtimeKind, { + channels: "feishu", + feishuDomain: "lark", + feishuDmPolicy: "allowlist", + feishuAllowFrom: "ou_teacher,ou_admin", + feishuGroupPolicy: "allowlist", + feishuGroupAllowFrom: "oc_teaching", + feishuRequireMention: false, + }); + expect(channel).toEqual({ + type: "Feishu", + feishu: { + domain: "Lark", + connectionMode: "WebSocket", + directMessages: { + policy: "Allowlist", + allowFrom: ["ou_teacher", "ou_admin"], + }, + groups: { + policy: "Allowlist", + allowFrom: ["oc_teaching"], + requireMention: false, + }, + }, + }); + expect(JSON.stringify(channel)).not.toContain("secret"); + }, + ); + + it("maps App credentials only into Secret env keys", () => { + expect( + buildFeishuSecrets({ + channels: "feishu", + feishuAppId: "cli_test", + feishuAppSecret: "top-secret", + }), + ).toEqual({ + FEISHU_APP_ID: "cli_test", + FEISHU_APP_SECRET: "top-secret", + }); + }); + + it("rejects partial credentials and unsupported runtimes", () => { + expect(() => + buildFeishuSecrets({ channels: "feishu", feishuAppId: "cli_test" }), + ).toThrow("both --feishu-app-id and --feishu-app-secret"); + expect(() => buildFeishuChannelSpec("LangGraph", { channels: "feishu" })).toThrow( + "Feishu is only supported by OpenClaw and Hermes", + ); + expect(() => buildFeishuSecrets({ channels: "feishu" })).toThrow( + "both --feishu-app-id and --feishu-app-secret", + ); + }); + + it("rejects invalid Feishu policy values and IDs", () => { + expect(() => buildFeishuChannelSpec("OpenClaw", { + channels: "feishu", + feishuDomain: "example", + })).toThrow("--feishu-domain"); + expect(() => buildFeishuChannelSpec("OpenClaw", { + channels: "feishu", + feishuDmPolicy: "allowlist", + })).toThrow("requires --feishu-allow-from"); + expect(() => buildFeishuChannelSpec("OpenClaw", { + channels: "feishu", + feishuGroupAllowFrom: "group-1", + })).toThrow("oc_ group chat IDs"); + }); + + it("builds an apply manifest without putting values in kubectl arguments", () => { + const manifest = buildCredentialSecretManifest("agent-credentials", "kars-agent", { + FEISHU_APP_ID: "cli_test", + FEISHU_APP_SECRET: "top-secret", + }); + expect(manifest).toMatchObject({ + kind: "Secret", + metadata: { name: "agent-credentials", namespace: "kars-agent" }, + stringData: { FEISHU_APP_SECRET: "top-secret" }, + }); + }); +}); + describe("channel and plugin secret generation", () => { it("maps telegram token to TELEGRAM_BOT_TOKEN env var", () => { const secrets = buildSecrets( diff --git a/cli/src/commands/add.ts b/cli/src/commands/add.ts index d02bbe083..6ef9d521f 100644 --- a/cli/src/commands/add.ts +++ b/cli/src/commands/add.ts @@ -4,6 +4,7 @@ import { Command } from "commander"; import chalk from "chalk"; import ora from "ora"; +import { randomBytes } from "node:crypto"; import { loadContext, resolveSecret } from "../config.js"; import { assertRuntimeWired, buildRuntimeBlock, flagToKind } from "../runtime.js"; import { @@ -14,6 +15,112 @@ import { } from "../refs.js"; import type { RuntimeKind } from "../runtime.js"; import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js"; +import { + classifyFeishuSecretReference, + shouldCleanupStagedFeishuSecret, + type FeishuSecretReferenceState, +} from "../lib/feishu-secret-reference.js"; + +type FeishuAddOptions = { + channels?: string; + feishuAppId?: string; + feishuAppSecret?: string; + feishuDomain?: string; + feishuDmPolicy?: string; + feishuAllowFrom?: string; + feishuGroupPolicy?: string; + feishuGroupAllowFrom?: string; + feishuRequireMention?: boolean; +}; + +function includesChannel(channels: string | undefined, name: string): boolean { + return (channels ?? "") + .split(",") + .map((channel) => channel.trim().toLowerCase()) + .includes(name); +} + +function csv(value: string | undefined): string[] { + return (value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); +} + +export function buildCredentialSecretManifest( + name: string, + namespace: string, + values: Record, + options: { immutable?: boolean; labels?: Record } = {}, +): Record { + return { + apiVersion: "v1", + kind: "Secret", + metadata: { name, namespace, ...(options.labels ? { labels: options.labels } : {}) }, + type: "Opaque", + ...(options.immutable ? { immutable: true } : {}), + stringData: values, + }; +} + +export function buildFeishuChannelSpec( + runtimeKind: RuntimeKind, + options: FeishuAddOptions, +): Record | undefined { + if (!includesChannel(options.channels, "feishu")) return undefined; + if (runtimeKind !== "OpenClaw" && runtimeKind !== "Hermes") { + throw new Error("Feishu is only supported by OpenClaw and Hermes"); + } + const domain = (options.feishuDomain ?? "feishu").toLowerCase(); + const dmPolicy = (options.feishuDmPolicy ?? "pairing").toLowerCase(); + const groupPolicy = (options.feishuGroupPolicy ?? "allowlist").toLowerCase(); + if (!new Set(["feishu", "lark"]).has(domain)) { + throw new Error("--feishu-domain must be feishu or lark"); + } + if (!new Set(["pairing", "allowlist", "disabled"]).has(dmPolicy)) { + throw new Error("--feishu-dm-policy must be pairing, allowlist, or disabled"); + } + if (!new Set(["allowlist", "disabled"]).has(groupPolicy)) { + throw new Error("--feishu-group-policy must be allowlist or disabled"); + } + const allowFrom = csv(options.feishuAllowFrom); + const groupAllowFrom = csv(options.feishuGroupAllowFrom); + if (dmPolicy === "allowlist" && allowFrom.length === 0) { + throw new Error("--feishu-dm-policy allowlist requires --feishu-allow-from"); + } + if (allowFrom.some((id) => !/^ou_[A-Za-z0-9_-]+$/.test(id))) { + throw new Error("--feishu-allow-from values must be ou_ user open IDs"); + } + if (groupAllowFrom.some((id) => !/^oc_[A-Za-z0-9_-]+$/.test(id))) { + throw new Error("--feishu-group-allow-from values must be oc_ group chat IDs"); + } + return { + type: "Feishu", + feishu: { + domain: domain === "lark" ? "Lark" : "Feishu", + connectionMode: "WebSocket", + directMessages: { + policy: dmPolicy === "allowlist" ? "Allowlist" : dmPolicy === "disabled" ? "Disabled" : "Pairing", + allowFrom, + }, + groups: { + policy: groupPolicy === "disabled" ? "Disabled" : "Allowlist", + allowFrom: groupAllowFrom, + requireMention: options.feishuRequireMention ?? true, + }, + }, + }; +} + +export function buildFeishuSecrets(options: FeishuAddOptions): Record { + if (!includesChannel(options.channels, "feishu")) return {}; + const appId = options.feishuAppId?.trim(); + const appSecret = options.feishuAppSecret?.trim(); + if (!appId || !appSecret) { + throw new Error("Feishu requires both --feishu-app-id and --feishu-app-secret"); + } + return { FEISHU_APP_ID: appId, FEISHU_APP_SECRET: appSecret }; +} export function validateRuntimeSpecificAddFlags( runtimeKind: RuntimeKind, @@ -26,6 +133,10 @@ export function validateRuntimeSpecificAddFlags( ["--telegram-allow-from", options.telegramAllowFrom], ["--slack-token", options.slackToken], ["--discord-token", options.discordToken], + ["--feishu-app-id", options.feishuAppId], + ["--feishu-app-secret", options.feishuAppSecret], + ["--feishu-allow-from", options.feishuAllowFrom], + ["--feishu-group-allow-from", options.feishuGroupAllowFrom], ]; if (runtimeKind !== "OpenClaw" && runtimeKind !== "Hermes") { const used = channelFlags @@ -38,6 +149,22 @@ export function validateRuntimeSpecificAddFlags( ); } } + const feishuFlags: Array<[string, unknown]> = [ + ["--feishu-app-id", options.feishuAppId], + ["--feishu-app-secret", options.feishuAppSecret], + ["--feishu-allow-from", options.feishuAllowFrom], + ["--feishu-group-allow-from", options.feishuGroupAllowFrom], + ["--feishu-domain", options.feishuDomain !== "feishu" ? options.feishuDomain : undefined], + ["--feishu-dm-policy", options.feishuDmPolicy !== "pairing" ? options.feishuDmPolicy : undefined], + ["--feishu-group-policy", options.feishuGroupPolicy !== "allowlist" ? options.feishuGroupPolicy : undefined], + ["--no-feishu-require-mention", options.feishuRequireMention === false], + ]; + const usedFeishuFlags = feishuFlags + .filter(([, value]) => value !== undefined && value !== "" && value !== false) + .map(([flag]) => flag); + if (usedFeishuFlags.length > 0 && !includesChannel(options.channels as string | undefined, "feishu")) { + errors.push(`${usedFeishuFlags.join(", ")} require --channels feishu.`); + } const openClawOnlyFlags: Array<[string, unknown]> = [ ["--skills", options.skills], @@ -104,11 +231,20 @@ export function addCommand(): Command { .option("--agent-tools ", "Foundry tools: file_search,web_search,code_interpreter (comma-separated)") // ── Runtime-specific: OpenClaw + Hermes (channel-capable runtimes) ─ - .option("--channels ", "[OpenClaw + Hermes] Channels to enable: telegram,slack,discord,whatsapp (comma-separated)") + .option("--channels ", "[OpenClaw + Hermes] Channels to enable: telegram,slack,discord,whatsapp,feishu (comma-separated)") .option("--telegram-token ", "[OpenClaw + Hermes] Telegram bot token (from BotFather)") .option("--telegram-allow-from ", "[OpenClaw + Hermes] Telegram user IDs allowed to DM (comma-separated)") .option("--slack-token ", "[OpenClaw + Hermes] Slack bot OAuth token") .option("--discord-token ", "[OpenClaw + Hermes] Discord bot token") + .option("--feishu-app-id ", "[OpenClaw + Hermes] Feishu/Lark App ID") + .option("--feishu-app-secret ", "[OpenClaw + Hermes] Feishu/Lark App Secret") + .option("--feishu-domain ", "[OpenClaw + Hermes] Feishu domain: feishu | lark", "feishu") + .option("--feishu-dm-policy ", "[OpenClaw + Hermes] Feishu DM policy: pairing | allowlist | disabled", "pairing") + .option("--feishu-allow-from ", "[OpenClaw + Hermes] Feishu user open_ids (comma-separated)") + .option("--feishu-group-policy ", "[OpenClaw + Hermes] Feishu group policy: allowlist | disabled", "allowlist") + .option("--feishu-group-allow-from ", "[OpenClaw + Hermes] Feishu group chat_ids (comma-separated)") + .option("--feishu-require-mention", "[OpenClaw + Hermes] Require direct bot mention in groups", true) + .option("--no-feishu-require-mention", "[OpenClaw + Hermes] Respond in allowed groups without mention") // ── Runtime-specific: OpenClaw only (skills + plugin API keys) ───── .option("--skills ", "[OpenClaw only] Skills to activate: browser,github,summarize,weather (comma-separated)") .option("--brave-api-key ", "[OpenClaw only] Brave Search API key") @@ -213,6 +349,22 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. byoContractVersion: options.byoContractVersion, mafLanguage: options.mafLanguage as "python" | "dotnet", }); + let feishuChannel: Record | undefined; + let feishuSecrets: Record = {}; + try { + feishuChannel = buildFeishuChannelSpec(runtimeKind, options); + feishuSecrets = buildFeishuSecrets({ + ...options, + feishuAppId: resolveSecret(options.feishuAppId, "feishu-app-id"), + feishuAppSecret: resolveSecret(options.feishuAppSecret, "feishu-app-secret"), + }); + } catch (error) { + console.error(chalk.red(`\n Error: ${(error as Error).message}\n`)); + process.exit(1); + } + const feishuSecretName = feishuChannel + ? `${name}-feishu-${randomBytes(6).toString("hex")}` + : undefined; if (options.workspaceBootstrap) { const openclaw = runtimeBlock.openclaw as Record; openclaw.workspace = { @@ -259,6 +411,10 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. if (storage) { (sandbox.spec as Record).storage = storage; } + if (feishuChannel) { + feishuChannel.credentialSecretRef = { name: feishuSecretName }; + (sandbox.spec as Record).channels = [feishuChannel]; + } // Add Foundry agent config if provided if (options.agentInstructions || options.agentTools) { @@ -307,7 +463,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. discord: "DISCORD_BOT_TOKEN", whatsapp: "WHATSAPP_ENABLED", }; - const knownChannels = new Set(["telegram", "slack", "discord", "whatsapp"]); + const knownChannels = new Set(["telegram", "slack", "discord", "whatsapp", "feishu"]); if (options.channels) { const channels = options.channels.split(",").map((c: string) => c.trim().toLowerCase()); @@ -467,6 +623,7 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. } const spinner = ora(`Creating sandbox '${name}' (${options.isolation}, ${options.model})...`).start(); + let stagedFeishuSecretCreated = false; try { // Verify cluster is reachable @@ -535,21 +692,39 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. }; if (Object.keys(allSecrets).length > 0) { spinner.text = "Creating credential secret..."; - try { - // Ensure namespace exists - await execa("kubectl", ["create", "namespace", namespace], { stdio: "pipe" }).catch(() => {}); - const secretArgs = ["create", "secret", "generic", `${name}-credentials`, "-n", namespace]; - for (const [envVar, value] of Object.entries(allSecrets)) { - secretArgs.push(`--from-literal=${envVar}=${value}`); - } - await execa("kubectl", secretArgs, { stdio: "pipe" }).catch(async () => { - // Already exists — delete and recreate with updated values - await execa("kubectl", ["delete", "secret", `${name}-credentials`, "-n", namespace], { stdio: "pipe" }).catch(() => {}); - return execa("kubectl", secretArgs, { stdio: "pipe" }); - }); - } catch { - // Non-fatal — controller can still create pod without credential secret - } + // Ensure namespace exists + await execa("kubectl", ["create", "namespace", namespace], { stdio: "pipe" }).catch(() => {}); + const secretManifest = buildCredentialSecretManifest( + `${name}-credentials`, + namespace, + allSecrets, + ); + await execa("kubectl", ["apply", "--server-side", "--field-manager=kars-cli", "-f", "-"], { + input: JSON.stringify(secretManifest), + stdio: ["pipe", "pipe", "pipe"], + }); + } + if (Object.keys(feishuSecrets).length > 0) { + spinner.text = "Creating immutable Feishu credential secret..."; + await execa("kubectl", ["create", "namespace", namespace], { stdio: "pipe" }).catch(() => {}); + const feishuSecretManifest = buildCredentialSecretManifest( + feishuSecretName!, + namespace, + feishuSecrets, + { + immutable: true, + labels: { + "channels.kars.azure.com/managed-rotation": "true", + "channels.kars.azure.com/revision-state": "staged", + "kars.azure.com/sandbox": name, + }, + }, + ); + await execa("kubectl", ["create", "-f", "-"], { + input: JSON.stringify(feishuSecretManifest), + stdio: ["pipe", "pipe", "pipe"], + }); + stagedFeishuSecretCreated = true; } spinner.text = `Creating sandbox '${name}'...`; // Apply InferencePolicy + (optional) ToolPolicy + KarsSandbox as a @@ -700,6 +875,21 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs. console.log(chalk.dim(` Remove: kars destroy ${name}\n`)); } catch (error) { + if (stagedFeishuSecretCreated && feishuSecretName) { + let referenceState: FeishuSecretReferenceState = "unknown"; + try { + const { stdout } = await execa("kubectl", [ + "get", "karssandbox", name, "-n", "kars-system", + "--ignore-not-found=true", "-o", "json", + ], { stdio: "pipe" }); + referenceState = classifyFeishuSecretReference(stdout, feishuSecretName); + } catch { /* preserve the Secret when the live reference is unknown */ } + if (shouldCleanupStagedFeishuSecret(feishuSecretName, referenceState)) { + await execa("kubectl", [ + "delete", "secret", feishuSecretName, "-n", `kars-${name}`, "--ignore-not-found", + ], { stdio: "pipe" }).catch(() => {}); + } + } spinner.fail("Failed to create sandbox"); const message = error instanceof Error ? error.message : String(error); if (message.includes("karssandboxes.kars.azure.com")) { // lgtm[js/incomplete-url-substring-sanitization] — error message check, not URL validation diff --git a/cli/src/commands/credentials.test.ts b/cli/src/commands/credentials.test.ts new file mode 100644 index 000000000..5751782cf --- /dev/null +++ b/cli/src/commands/credentials.test.ts @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "vitest"; +import { + CREDENTIAL_FLAG_TO_ENV, + buildFeishuChannelSecretPatch, + buildFeishuRotationSecretName, + planCredentialSecretUpdates, + selectSandboxForCredentialUpdate, + validateCredentialUpdates, +} from "./credentials.js"; +import { + classifyFeishuSecretReference, + shouldCleanupStagedFeishuSecret, +} from "../lib/feishu-secret-reference.js"; + +describe("credentials update channel mappings", () => { + it("maps Feishu credential flags to runtime environment variables", () => { + expect(CREDENTIAL_FLAG_TO_ENV.feishuAppId).toBe("FEISHU_APP_ID"); + expect(CREDENTIAL_FLAG_TO_ENV.feishuAppSecret).toBe("FEISHU_APP_SECRET"); + }); + + it("requires Feishu App ID and App Secret to rotate together", () => { + expect(() => validateCredentialUpdates({ FEISHU_APP_ID: "cli_new" })).toThrow( + "must be updated together", + ); + expect(() => + validateCredentialUpdates({ + FEISHU_APP_ID: "cli_new", + FEISHU_APP_SECRET: "secret", + }), + ).not.toThrow(); + expect(() => + validateCredentialUpdates({ + FEISHU_APP_ID: "cli_new", + FEISHU_APP_SECRET: "secret", + TELEGRAM_BOT_TOKEN: "telegram", + }), + ).toThrow("separately"); + }); + + it("routes Feishu and ordinary credentials to separate declared targets", () => { + const sandbox = { + spec: { + channels: [{ type: "Feishu", credentialSecretRef: { name: "custom-feishu" } }], + }, + }; + expect(planCredentialSecretUpdates( + "agent", + sandbox, + { TELEGRAM_BOT_TOKEN: "telegram" }, + )).toEqual([ + { kind: "conventional", secretName: "agent-credentials", updates: { TELEGRAM_BOT_TOKEN: "telegram" } }, + ]); + expect(planCredentialSecretUpdates( + "agent", + sandbox, + { + FEISHU_APP_ID: "cli_new", + FEISHU_APP_SECRET: "secret", + }, + )).toEqual([ + { + kind: "feishu", + secretName: "custom-feishu", + updates: { FEISHU_APP_ID: "cli_new", FEISHU_APP_SECRET: "secret" }, + }, + ]); + expect(() => planCredentialSecretUpdates( + "agent", + { spec: { channels: [] } }, + { FEISHU_APP_ID: "cli_new", FEISHU_APP_SECRET: "secret" }, + )).toThrow("does not declare a Feishu channel"); + }); + + it("builds an immutable Secret name and non-sensitive channel-ref patch", () => { + expect(buildFeishuRotationSecretName("custom-feishu", "a1b2c3")).toBe( + "custom-feishu-rotation-a1b2c3", + ); + const patch = buildFeishuChannelSecretPatch( + "12345", + [{ type: "Feishu", feishu: { domain: "Feishu" } }], + "custom-feishu-rotation-a1b2c3", + ); + expect(patch).toEqual([ + { op: "test", path: "/metadata/resourceVersion", value: "12345" }, + { op: "test", path: "/spec/channels/0/type", value: "Feishu" }, + { + op: "add", + path: "/spec/channels/0/credentialSecretRef", + value: { name: "custom-feishu-rotation-a1b2c3" }, + }, + ]); + expect(JSON.stringify(patch)).not.toContain("FEISHU_APP"); + + expect(buildFeishuChannelSecretPatch( + "12346", + [{ type: "Feishu", credentialSecretRef: { name: "old" } }], + "new", + )).toEqual([ + { op: "test", path: "/metadata/resourceVersion", value: "12346" }, + { op: "test", path: "/spec/channels/0/type", value: "Feishu" }, + { + op: "replace", + path: "/spec/channels/0/credentialSecretRef/name", + value: "new", + }, + ]); + }); + + it("selects one namespaced sandbox and rejects ambiguous names", () => { + const selected = selectSandboxForCredentialUpdate("agent", { + items: [{ metadata: { name: "agent", namespace: "team-a" } }], + }); + expect(selected.metadata?.namespace).toBe("team-a"); + expect(() => + selectSandboxForCredentialUpdate("agent", { + items: [ + { metadata: { name: "agent", namespace: "team-a" } }, + { metadata: { name: "agent", namespace: "team-b" } }, + ], + }), + ).toThrow("multiple KarsSandboxes"); + }); + + it("preserves a staged Secret when an ambiguous patch committed or cannot be checked", () => { + const committed = JSON.stringify({ + spec: { channels: [{ type: "Feishu", credentialSecretRef: { name: "new-revision" } }] }, + }); + const conflict = JSON.stringify({ + spec: { channels: [{ type: "Feishu", credentialSecretRef: { name: "old-revision" } }] }, + }); + expect(classifyFeishuSecretReference(committed, "new-revision")).toBe("referenced"); + expect(classifyFeishuSecretReference(conflict, "new-revision")).toBe("unreferenced"); + expect(classifyFeishuSecretReference("", "new-revision")).toBe("unreferenced"); + expect(classifyFeishuSecretReference("not-json", "new-revision")).toBe("unknown"); + expect(shouldCleanupStagedFeishuSecret("new-revision", "referenced")).toBe(false); + expect(shouldCleanupStagedFeishuSecret("new-revision", "unreferenced")).toBe(true); + expect(shouldCleanupStagedFeishuSecret("new-revision", "unknown")).toBe(false); + expect(shouldCleanupStagedFeishuSecret(undefined, "unreferenced")).toBe(false); + }); +}); diff --git a/cli/src/commands/credentials.ts b/cli/src/commands/credentials.ts index 4b689b29f..e169a69bd 100644 --- a/cli/src/commands/credentials.ts +++ b/cli/src/commands/credentials.ts @@ -3,11 +3,139 @@ import { Command } from "commander"; import chalk from "chalk"; +import { randomBytes } from "node:crypto"; import { banner, section } from "../stepper.js"; import { promptAndSaveCredentials, SECRETS_FILE, KNOWN_SECRETS, loadSecrets, setSecret, getSecret, deleteSecret, listSecretVariants, } from "../config.js"; +import { buildCredentialSecretManifest } from "./add.js"; +import { + classifyFeishuSecretReference, + shouldCleanupStagedFeishuSecret, + type FeishuSecretReferenceState, +} from "../lib/feishu-secret-reference.js"; + +export const CREDENTIAL_FLAG_TO_ENV: Record = { + telegramToken: "TELEGRAM_BOT_TOKEN", + telegramAllowFrom: "TELEGRAM_ALLOW_FROM", + slackToken: "SLACK_BOT_TOKEN", + discordToken: "DISCORD_BOT_TOKEN", + feishuAppId: "FEISHU_APP_ID", + feishuAppSecret: "FEISHU_APP_SECRET", + braveApiKey: "BRAVE_API_KEY", + tavilyApiKey: "TAVILY_API_KEY", + exaApiKey: "EXA_API_KEY", + firecrawlApiKey: "FIRECRAWL_API_KEY", + perplexityApiKey: "PERPLEXITY_API_KEY", + openaiApiKey: "OPENAI_API_KEY", +}; + +const FEISHU_CREDENTIAL_KEYS = new Set(["FEISHU_APP_ID", "FEISHU_APP_SECRET"]); + +type SandboxCredentialView = { + metadata?: { name?: string; namespace?: string; resourceVersion?: string }; + status?: { namespace?: string }; + spec?: { + channels?: Array<{ + type?: string; + credentialSecretRef?: { name?: string }; + [key: string]: unknown; + }>; + }; +}; + +export function selectSandboxForCredentialUpdate( + sandboxName: string, + list: { items?: SandboxCredentialView[] }, +): SandboxCredentialView { + const matches = (list.items ?? []).filter((item) => item.metadata?.name === sandboxName); + if (matches.length === 0) { + throw new Error(`KarsSandbox '${sandboxName}' was not found`); + } + if (matches.length > 1) { + throw new Error( + `multiple KarsSandboxes named '${sandboxName}' exist; update credentials with kubectl in the intended namespace`, + ); + } + return matches[0]; +} + +export type CredentialSecretUpdatePlan = { + kind: "conventional" | "feishu"; + secretName: string; + updates: Record; +}; + +export function validateCredentialUpdates(updates: Record): void { + const hasAppId = Boolean(updates.FEISHU_APP_ID); + const hasAppSecret = Boolean(updates.FEISHU_APP_SECRET); + if (hasAppId !== hasAppSecret) { + throw new Error("Feishu App ID and App Secret must be updated together"); + } + if (hasAppId && Object.keys(updates).some((key) => !FEISHU_CREDENTIAL_KEYS.has(key))) { + throw new Error("Feishu credentials must be rotated separately from other credentials"); + } +} + +export function planCredentialSecretUpdates( + sandboxName: string, + sandbox: SandboxCredentialView, + updates: Record, +): CredentialSecretUpdatePlan[] { + validateCredentialUpdates(updates); + const conventionalSecret = `${sandboxName}-credentials`; + const feishuChannel = sandbox.spec?.channels?.find((channel) => channel.type === "Feishu"); + if (updates.FEISHU_APP_ID && !feishuChannel) { + throw new Error(`KarsSandbox '${sandboxName}' does not declare a Feishu channel`); + } + const feishuSecret = feishuChannel?.credentialSecretRef?.name || conventionalSecret; + const conventionalUpdates: Record = {}; + const feishuUpdates: Record = {}; + for (const [key, value] of Object.entries(updates)) { + if (FEISHU_CREDENTIAL_KEYS.has(key)) { + feishuUpdates[key] = value; + } else { + conventionalUpdates[key] = value; + } + } + const plans: CredentialSecretUpdatePlan[] = []; + if (Object.keys(conventionalUpdates).length > 0) { + plans.push({ kind: "conventional", secretName: conventionalSecret, updates: conventionalUpdates }); + } + if (Object.keys(feishuUpdates).length > 0) { + plans.push({ kind: "feishu", secretName: feishuSecret, updates: feishuUpdates }); + } + return plans; +} + +export function buildFeishuRotationSecretName(baseName: string, suffix: string): string { + const trailer = `-rotation-${suffix}`; + return `${baseName.slice(0, 253 - trailer.length).replace(/[.-]+$/, "")}${trailer}`; +} + +export function buildFeishuChannelSecretPatch( + resourceVersion: string, + channels: NonNullable["channels"], + secretName: string, +): Array> { + if (!resourceVersion) { + throw new Error("KarsSandbox resourceVersion is required for Feishu credential rotation"); + } + const channelIndex = (channels ?? []).findIndex((channel) => channel.type === "Feishu"); + if (channelIndex < 0) { + throw new Error("KarsSandbox does not declare a Feishu channel"); + } + const channel = channels![channelIndex]; + const refPath = `/spec/channels/${channelIndex}/credentialSecretRef`; + return [ + { op: "test", path: "/metadata/resourceVersion", value: resourceVersion }, + { op: "test", path: `/spec/channels/${channelIndex}/type`, value: "Feishu" }, + channel.credentialSecretRef + ? { op: "replace", path: `${refPath}/name`, value: secretName } + : { op: "add", path: refPath, value: { name: secretName } }, + ]; +} export function credentialsCommand(): Command { const cmd = new Command("credentials"); @@ -26,8 +154,7 @@ export function credentialsCommand(): Command { if (Object.keys(secrets).length > 0) { console.log(chalk.dim(" Currently stored:")); for (const key of Object.keys(secrets).sort()) { - const val = secrets[key]; - const masked = val.length > 8 ? "••••" + val.slice(-4) : "••••"; + const masked = "••••"; const info = KNOWN_SECRETS[key.includes(".") ? key.slice(0, key.indexOf(".")) : key]; const label = info ? chalk.dim(` (${info.label})`) : ""; console.log(` ${chalk.cyan(key)} = ${masked}${label}`); @@ -41,6 +168,7 @@ export function credentialsCommand(): Command { { name: "Telegram — bot token, allowed users", value: "telegram" }, { name: "Slack — bot OAuth token", value: "slack" }, { name: "Discord — bot token", value: "discord" }, + { name: "Feishu — App ID and App Secret", value: "feishu" }, { name: "Search APIs — Brave, Tavily, Exa, Perplexity", value: "search" }, { name: "Other APIs — Firecrawl, OpenAI", value: "other" }, new inquirer.Separator(), @@ -71,6 +199,10 @@ export function credentialsCommand(): Command { discord: [ { key: "discord-token", label: "Discord bot token", allowSuffix: true }, ], + feishu: [ + { key: "feishu-app-id", label: "Feishu App ID", allowSuffix: true }, + { key: "feishu-app-secret", label: "Feishu App Secret", allowSuffix: true }, + ], search: [ { key: "brave-api-key", label: "Brave Search API key" }, { key: "tavily-api-key", label: "Tavily search API key" }, @@ -103,7 +235,7 @@ export function credentialsCommand(): Command { } const currentVal = getSecret(finalKey); - const currentHint = currentVal ? chalk.dim(` (current: ••••${currentVal.slice(-4)})`) : ""; + const currentHint = currentVal ? chalk.dim(" (current: set)") : ""; const { value } = await inquirer.prompt([{ type: "password", @@ -114,8 +246,7 @@ export function credentialsCommand(): Command { if (value && value.trim()) { setSecret(finalKey, value.trim()); - const masked = value.length > 8 ? "••••" + value.slice(-4) : "••••"; - console.log(chalk.green(` ✔ ${finalKey} = ${masked}`)); + console.log(chalk.green(` ✔ ${finalKey} = ••••`)); } else if (currentVal) { console.log(chalk.dim(` Kept existing value for ${finalKey}`)); } else { @@ -171,9 +302,7 @@ export function credentialsCommand(): Command { // Note: `setSecret` runs `normalizeSecretValue` so Telegram `bot` // prefix stripping happens uniformly across all write paths. setSecret(key, value!); - const stored = (await import("../config.js")).getSecret(key) ?? value!; - const masked = stored.length > 8 ? "••••" + stored.slice(-4) : "••••"; - console.log(chalk.green(` ✔ ${key} = ${masked}`)); + console.log(chalk.green(` ✔ ${key} = ••••`)); console.log(chalk.dim(` Saved to ${SECRETS_FILE}`)); if (info || baseInfo) { console.log(chalk.dim(` → env var: ${(info || baseInfo)!.env}`)); @@ -194,8 +323,7 @@ export function credentialsCommand(): Command { } console.log(chalk.bold("\n Stored secrets:\n")); for (const key of keys.sort()) { - const val = secrets[key]; - const masked = val.length > 8 ? "••••" + val.slice(-4) : "••••"; + const masked = "••••"; const info = KNOWN_SECRETS[key]; let label = ""; if (info) { @@ -233,12 +361,14 @@ export function credentialsCommand(): Command { // Subcommand: update credentials for a running AKS sandbox const update = new Command("update"); update - .description("Update credentials for a running AKS sandbox (updates secret + restarts pod)") + .description("Update credentials for a running AKS sandbox (updates Secret + coordinates pod restart)") .argument("", "Sandbox name") .option("--telegram-token ", "New Telegram bot token") .option("--telegram-allow-from ", "Telegram allowed user IDs (comma-separated)") .option("--slack-token ", "New Slack bot token") .option("--discord-token ", "New Discord bot token") + .option("--feishu-app-id ", "New Feishu App ID") + .option("--feishu-app-secret ", "New Feishu App Secret") .option("--brave-api-key ", "New Brave Search API key") .option("--tavily-api-key ", "New Tavily API key") .option("--exa-api-key ", "New Exa API key") @@ -250,22 +380,9 @@ export function credentialsCommand(): Command { const { execa } = await import("execa"); const ora = (await import("ora")).default; - const flagToEnv: Record = { - telegramToken: "TELEGRAM_BOT_TOKEN", - telegramAllowFrom: "TELEGRAM_ALLOW_FROM", - slackToken: "SLACK_BOT_TOKEN", - discordToken: "DISCORD_BOT_TOKEN", - braveApiKey: "BRAVE_API_KEY", - tavilyApiKey: "TAVILY_API_KEY", - exaApiKey: "EXA_API_KEY", - firecrawlApiKey: "FIRECRAWL_API_KEY", - perplexityApiKey: "PERPLEXITY_API_KEY", - openaiApiKey: "OPENAI_API_KEY", - }; - // Collect new values const updates: Record = {}; - for (const [flag, env] of Object.entries(flagToEnv)) { + for (const [flag, env] of Object.entries(CREDENTIAL_FLAG_TO_ENV)) { if (options[flag]) updates[env] = options[flag]; } @@ -273,50 +390,108 @@ export function credentialsCommand(): Command { console.error(chalk.red(" No credentials specified. Use --telegram-token, --brave-api-key, etc.")); process.exit(1); } + const rotatesFeishu = Boolean(updates.FEISHU_APP_ID); + if (rotatesFeishu && options.restart === false) { + console.error(chalk.red(" Feishu credential rotation does not support --no-restart; the controller must claim the new App before rollout.")); + process.exit(1); + } - const namespace = `kars-${name}`; - const secretName = `${name}-credentials`; const spinner = ora(`Updating credentials for '${name}'...`).start(); + let stagedFeishuSecret: string | undefined; + let stagedFeishuNamespace: string | undefined; + let feishuControlNamespace: string | undefined; try { - // Read existing secret (if any) and merge with new values - let existing: Record = {}; - try { - const { stdout } = await execa("kubectl", [ - "get", "secret", secretName, "-n", namespace, - "-o", "jsonpath={.data}", - ], { stdio: "pipe" }); - if (stdout && stdout !== "{}") { - const data = JSON.parse(stdout); - for (const [k, v] of Object.entries(data)) { - existing[k] = Buffer.from(v as string, "base64").toString(); + const { stdout: sandboxJson } = await execa("kubectl", [ + "get", "karssandboxes", "-A", "-o", "json", + ], { stdio: "pipe" }); + const sandbox = selectSandboxForCredentialUpdate( + name, + JSON.parse(sandboxJson) as { items?: SandboxCredentialView[] }, + ); + const controlNamespace = sandbox.metadata?.namespace || "kars-system"; + feishuControlNamespace = controlNamespace; + const namespace = sandbox.status?.namespace || `kars-${name}`; + const plans = planCredentialSecretUpdates(name, sandbox, updates); + + for (const plan of plans) { + let existing: Record = {}; + try { + const { stdout } = await execa("kubectl", [ + "get", "secret", plan.secretName, "-n", namespace, + "-o", "jsonpath={.data}", + ], { stdio: "pipe" }); + if (stdout && stdout !== "{}") { + const data = JSON.parse(stdout); + for (const [key, value] of Object.entries(data)) { + existing[key] = Buffer.from(value as string, "base64").toString(); + } } + } catch { /* secret doesn't exist yet */ } + + const targetSecretName = plan.kind === "feishu" + ? buildFeishuRotationSecretName(plan.secretName, randomBytes(6).toString("hex")) + : plan.secretName; + const retained = plan.kind === "feishu" + ? Object.fromEntries( + Object.entries(existing).filter(([key]) => FEISHU_CREDENTIAL_KEYS.has(key)), + ) + : existing; + const manifest = buildCredentialSecretManifest( + targetSecretName, + namespace, + { ...retained, ...plan.updates }, + plan.kind === "feishu" + ? { + immutable: true, + labels: { + "channels.kars.azure.com/managed-rotation": "true", + "channels.kars.azure.com/revision-state": "staged", + "kars.azure.com/sandbox": name, + }, + } + : {}, + ); + const secretCommand = plan.kind === "feishu" + ? ["create", "-f", "-"] + : ["apply", "--server-side", "--field-manager=kars-cli", "-f", "-"]; + await execa("kubectl", secretCommand, { + input: JSON.stringify(manifest), + stdio: ["pipe", "pipe", "pipe"], + }); + if (plan.kind === "feishu") { + stagedFeishuSecret = targetSecretName; + stagedFeishuNamespace = namespace; } - } catch { /* secret doesn't exist yet */ } - - const merged = { ...existing, ...updates }; + } - // Create/replace the secret - const secretArgs = ["create", "secret", "generic", secretName, "-n", namespace, "--dry-run=client", "-o", "yaml"]; - for (const [env, val] of Object.entries(merged)) { - secretArgs.push(`--from-literal=${env}=${val}`); + if (stagedFeishuSecret) { + await execa("kubectl", [ + "patch", "karssandbox", name, "-n", controlNamespace, + "--type=json", "-p", JSON.stringify( + buildFeishuChannelSecretPatch( + sandbox.metadata?.resourceVersion ?? "", + sandbox.spec?.channels, + stagedFeishuSecret, + ), + ), + ], { stdio: "pipe" }); } - const { stdout: yaml } = await execa("kubectl", secretArgs, { stdio: "pipe" }); - await execa("kubectl", ["apply", "-f", "-"], { input: yaml, stdio: ["pipe", "pipe", "pipe"] }); spinner.succeed("Secret updated"); // Show what changed - for (const [env, val] of Object.entries(updates)) { - console.log(chalk.dim(` ${env} = ••••${val.slice(-4)}`)); + for (const env of Object.keys(updates)) { + console.log(chalk.dim(` ${env} updated`)); } - // Restart pod unless --no-restart if (options.restart !== false) { const restartSpinner = ora("Restarting pod...").start(); - await execa("kubectl", [ - "rollout", "restart", `deploy/${name}`, "-n", namespace, - ], { stdio: "pipe" }); + if (!rotatesFeishu) { + await execa("kubectl", [ + "rollout", "restart", `deploy/${name}`, "-n", namespace, + ], { stdio: "pipe" }); + } // Wait for rollout try { @@ -333,6 +508,31 @@ export function credentialsCommand(): Command { console.log(chalk.dim(` Restart manually: kubectl rollout restart deploy/${name} -n ${namespace}`)); } } catch (err: any) { + let referenceState: FeishuSecretReferenceState = "unknown"; + if (stagedFeishuSecret && feishuControlNamespace) { + try { + const { stdout } = await execa("kubectl", [ + "get", "karssandbox", name, "-n", feishuControlNamespace, + "--ignore-not-found=true", "-o", "json", + ], { stdio: "pipe" }); + referenceState = classifyFeishuSecretReference(stdout, stagedFeishuSecret); + } catch { /* preserve the Secret when the live reference is unknown */ } + } + if ( + shouldCleanupStagedFeishuSecret(stagedFeishuSecret, referenceState) + && stagedFeishuNamespace + ) { + try { + await execa("kubectl", [ + "delete", "secret", stagedFeishuSecret!, "-n", stagedFeishuNamespace, + "--ignore-not-found=true", + ], { stdio: "pipe" }); + } catch { + console.error(chalk.yellow( + ` Warning: failed to remove unreferenced Secret '${stagedFeishuSecret}'`, + )); + } + } spinner.fail(`Failed: ${err.message}`); process.exit(1); } diff --git a/cli/src/commands/operator/dialogs/spawn.ts b/cli/src/commands/operator/dialogs/spawn.ts index 5e5a2185d..c73c35414 100644 --- a/cli/src/commands/operator/dialogs/spawn.ts +++ b/cli/src/commands/operator/dialogs/spawn.ts @@ -137,8 +137,8 @@ export function openSpawnDialog(ctx: SpawnDialogContext): void { const variants = storedTokens[state.channel] || []; const matchedVariant = variants.find(v => v.value === tokenVal); const display = matchedVariant - ? `{green-fg}${matchedVariant.label}{/} (●●●●${tokenVal.slice(-4)})` - : tokenVal ? "●●●●" + tokenVal.slice(-4) : "{gray-fg}(press Enter to type){/}"; + ? `{green-fg}${matchedVariant.label}{/} (set)` + : tokenVal ? "●●●●" : "{gray-fg}(press Enter to type){/}"; const hint = variants.length > 1 ? ` {gray-fg}←→ ${variants.length} stored{/}` : ""; const label = state.channel.charAt(0).toUpperCase() + state.channel.slice(1); lines.push(`${sel} {bold}${label} Token:{/} ${display}${hint}`); diff --git a/cli/src/config.test.ts b/cli/src/config.test.ts index 426b67def..15bac3136 100644 --- a/cli/src/config.test.ts +++ b/cli/src/config.test.ts @@ -359,6 +359,8 @@ describe("secrets store", () => { it("KNOWN_SECRETS has correct env mappings", () => { expect(KNOWN_SECRETS["telegram-token"].env).toBe("TELEGRAM_BOT_TOKEN"); expect(KNOWN_SECRETS["slack-token"].env).toBe("SLACK_BOT_TOKEN"); + expect(KNOWN_SECRETS["feishu-app-id"].env).toBe("FEISHU_APP_ID"); + expect(KNOWN_SECRETS["feishu-app-secret"].env).toBe("FEISHU_APP_SECRET"); expect(KNOWN_SECRETS["azure-openai-key"].env).toBe("AZURE_OPENAI_API_KEY"); }); diff --git a/cli/src/config.ts b/cli/src/config.ts index 7617725db..cab7cd27e 100644 --- a/cli/src/config.ts +++ b/cli/src/config.ts @@ -45,6 +45,8 @@ export const KNOWN_SECRETS: Record = { "telegram-allow-from": { env: "TELEGRAM_ALLOW_FROM", label: "Telegram allowed user IDs" }, "slack-token": { env: "SLACK_BOT_TOKEN", label: "Slack bot OAuth token" }, "discord-token": { env: "DISCORD_BOT_TOKEN", label: "Discord bot token" }, + "feishu-app-id": { env: "FEISHU_APP_ID", label: "Feishu App ID" }, + "feishu-app-secret": { env: "FEISHU_APP_SECRET", label: "Feishu App Secret" }, "brave-api-key": { env: "BRAVE_API_KEY", label: "Brave Search API key" }, "tavily-api-key": { env: "TAVILY_API_KEY", label: "Tavily search API key" }, "exa-api-key": { env: "EXA_API_KEY", label: "Exa search API key" }, @@ -59,6 +61,8 @@ export const FLAG_TO_SECRET: Record = { telegramAllowFrom:"telegram-allow-from", slackToken: "slack-token", discordToken: "discord-token", + feishuAppId: "feishu-app-id", + feishuAppSecret: "feishu-app-secret", braveApiKey: "brave-api-key", tavilyApiKey: "tavily-api-key", exaApiKey: "exa-api-key", diff --git a/cli/src/lib/feishu-secret-reference.ts b/cli/src/lib/feishu-secret-reference.ts new file mode 100644 index 000000000..085ab027f --- /dev/null +++ b/cli/src/lib/feishu-secret-reference.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export type FeishuSecretReferenceState = "referenced" | "unreferenced" | "unknown"; + +export function classifyFeishuSecretReference( + sandboxJson: string, + secretName: string, +): FeishuSecretReferenceState { + if (!sandboxJson.trim()) return "unreferenced"; + try { + const sandbox = JSON.parse(sandboxJson) as { + spec?: { channels?: Array<{ type?: string; credentialSecretRef?: { name?: string } }> }; + }; + if (!Array.isArray(sandbox.spec?.channels)) return "unknown"; + const channel = sandbox.spec.channels.find((candidate) => candidate.type === "Feishu"); + return channel?.credentialSecretRef?.name === secretName ? "referenced" : "unreferenced"; + } catch { + return "unknown"; + } +} + +export function shouldCleanupStagedFeishuSecret( + stagedSecretName: string | undefined, + referenceState: FeishuSecretReferenceState, +): boolean { + return Boolean(stagedSecretName) && referenceState === "unreferenced"; +} \ No newline at end of file diff --git a/controller/src/config_hash.rs b/controller/src/config_hash.rs index 5a99ec205..e4c183d7e 100644 --- a/controller/src/config_hash.rs +++ b/controller/src/config_hash.rs @@ -32,6 +32,11 @@ use prometheus::{IntGaugeVec, opts, register_int_gauge_vec}; use sha2::{Digest, Sha256}; use std::sync::LazyLock; +pub fn sha256_hex_prefix(bytes: &[u8], prefix_bytes: usize) -> String { + let digest = Sha256::digest(bytes); + hex::encode(&digest[..prefix_bytes.min(digest.len())]) +} + /// Env var names that contribute to the controller config hash. /// /// Adding/removing entries from this list is itself a config-hash @@ -128,6 +133,14 @@ pub fn record_config_hash(config_hash: &str) { #[cfg(test)] mod tests { use super::*; + + #[test] + fn sha256_hex_prefix_matches_known_vector() { + assert_eq!( + sha256_hex_prefix(b"abc", 16), + "ba7816bf8f01cfea414140de5dae2223" + ); + } use std::collections::HashMap; fn lookup_fn(map: HashMap<&'static str, &'static str>) -> impl Fn(&str) -> Option { diff --git a/controller/src/crd.rs b/controller/src/crd.rs index d3c677b44..e7ca23721 100644 --- a/controller/src/crd.rs +++ b/controller/src/crd.rs @@ -77,6 +77,11 @@ pub struct KarsSandboxSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub storage: Option, + /// Runtime-facing messaging channels. Policy is non-sensitive; channel + /// credentials remain in the per-sandbox Kubernetes Secret. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub channels: Vec, + /// Network policy pub network_policy: Option, @@ -248,6 +253,93 @@ pub enum WorkspaceRetainPolicy { Delete, } +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ChannelSpec { + #[serde(rename = "type")] + pub type_: ChannelType, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub credential_secret_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub feishu: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum ChannelType { + Feishu, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct FeishuChannelSpec { + #[serde(default)] + pub domain: FeishuDomain, + #[serde(default)] + pub connection_mode: FeishuConnectionMode, + #[serde(default)] + pub direct_messages: DirectMessagePolicy, + #[serde(default)] + pub groups: GroupPolicy, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum FeishuDomain { + #[default] + Feishu, + Lark, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum FeishuConnectionMode { + #[default] + WebSocket, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct DirectMessagePolicy { + #[serde(default)] + pub policy: DirectMessageAccess, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub allow_from: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum DirectMessageAccess { + #[default] + Pairing, + Allowlist, + Disabled, +} + +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct GroupPolicy { + #[serde(default)] + pub policy: GroupAccess, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub allow_from: Vec, + #[serde(default = "default_true")] + pub require_mention: bool, +} + +impl Default for GroupPolicy { + fn default() -> Self { + Self { + policy: GroupAccess::Allowlist, + allow_from: Vec::new(), + require_mention: true, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, JsonSchema, PartialEq, Eq)] +pub enum GroupAccess { + #[default] + Allowlist, + Disabled, +} + /// Per-sandbox mesh authentication mode. /// /// Two terminal modes are supported: @@ -1405,6 +1497,54 @@ mod tests { assert!(cfg.allowlist_ref.is_none()); } + #[test] + fn feishu_channel_defaults_are_fail_closed() { + let channel: ChannelSpec = serde_json::from_value(serde_json::json!({ + "type": "Feishu", + "feishu": {} + })) + .expect("Feishu channel uses safe defaults"); + assert_eq!(channel.type_, ChannelType::Feishu); + let feishu = channel.feishu.expect("Feishu config"); + assert_eq!(feishu.domain, FeishuDomain::Feishu); + assert_eq!(feishu.connection_mode, FeishuConnectionMode::WebSocket); + assert_eq!(feishu.direct_messages.policy, DirectMessageAccess::Pairing); + assert!(feishu.direct_messages.allow_from.is_empty()); + assert_eq!(feishu.groups.policy, GroupAccess::Allowlist); + assert!(feishu.groups.allow_from.is_empty()); + assert!(feishu.groups.require_mention); + } + + #[test] + fn feishu_channel_round_trips_camel_case_policy() { + let channel: ChannelSpec = serde_json::from_value(serde_json::json!({ + "type": "Feishu", + "credentialSecretRef": {"name": "agent-credentials"}, + "feishu": { + "domain": "Lark", + "connectionMode": "WebSocket", + "directMessages": { + "policy": "Allowlist", + "allowFrom": ["ou_teacher"] + }, + "groups": { + "policy": "Allowlist", + "allowFrom": ["oc_teaching"], + "requireMention": false + } + } + })) + .unwrap(); + let value = serde_json::to_value(channel).unwrap(); + assert_eq!(value["credentialSecretRef"]["name"], "agent-credentials"); + assert_eq!(value["feishu"]["connectionMode"], "WebSocket"); + assert_eq!( + value["feishu"]["directMessages"]["allowFrom"][0], + "ou_teacher" + ); + assert_eq!(value["feishu"]["groups"]["allowFrom"][0], "oc_teaching"); + } + #[test] fn egress_mode_default_is_learn() { // Slice 5b: default egress mode is `Learn` so manifests that omit diff --git a/controller/src/crd_validations.rs b/controller/src/crd_validations.rs index d626a5da9..09213477c 100644 --- a/controller/src/crd_validations.rs +++ b/controller/src/crd_validations.rs @@ -752,7 +752,7 @@ pub fn kars_sre_action_crd() -> CustomResourceDefinition { .expect("kube-rs derive must produce a spec property on KarsSREAction") } -fn inject_kars_sandbox_workspace_validations( +fn inject_kars_sandbox_validations( mut crd: CustomResourceDefinition, ) -> Option { let root = crd @@ -789,15 +789,76 @@ fn inject_kars_sandbox_workspace_validations( ..ValidationRule::default() }, ]); - Some(crd) + let channels = root + .properties + .as_mut()? + .get_mut("spec")? + .properties + .as_mut()? + .get_mut("channels")?; + channels.x_kubernetes_validations = Some(vec![ + ValidationRule { + rule: "self.filter(channel, channel.type == 'Feishu').size() <= 1".into(), + message: Some("spec.channels may contain at most one Feishu channel".into()), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ValidationRule { + rule: "self.all(channel, (channel.type == 'Feishu') == has(channel.feishu))".into(), + message: Some("channels[].feishu must be set iff type is Feishu".into()), + reason: Some("FieldValueInvalid".into()), + ..ValidationRule::default() + }, + ]); + + // kube-rs/schemars cannot express Kubernetes list bounds and nested + // string patterns consistently for this discriminated channel block. + // Apply the same structural facets as the hand-written Helm CRD. + let mut value = serde_json::to_value(&crd).ok()?; + const CHANNELS: &str = + "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/channels"; + value + .pointer_mut(CHANNELS)? + .as_object_mut()? + .insert("maxItems".into(), serde_json::json!(8)); + let credential_name = + format!("{CHANNELS}/items/properties/credentialSecretRef/properties/name"); + let credential_schema = value.pointer_mut(&credential_name)?.as_object_mut()?; + credential_schema.insert("maxLength".into(), serde_json::json!(253)); + credential_schema.insert( + "pattern".into(), + serde_json::json!(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"), + ); + for (field, pattern) in [ + ("directMessages", "^ou_[A-Za-z0-9_-]+$"), + ("groups", "^oc_[A-Za-z0-9_-]+$"), + ] { + let allow_from = + format!("{CHANNELS}/items/properties/feishu/properties/{field}/properties/allowFrom"); + let allow_from_schema = value.pointer_mut(&allow_from)?.as_object_mut()?; + allow_from_schema.insert("maxItems".into(), serde_json::json!(256)); + allow_from_schema + .get_mut("items")? + .as_object_mut()? + .insert("pattern".into(), serde_json::json!(pattern)); + } + let direct_messages = format!("{CHANNELS}/items/properties/feishu/properties/directMessages"); + value.pointer_mut(&direct_messages)?.as_object_mut()?.insert( + "x-kubernetes-validations".into(), + serde_json::json!([{ + "rule": "!has(self.policy) || self.policy != 'Allowlist' || (has(self.allowFrom) && self.allowFrom.size() > 0)", + "message": "directMessages.allowFrom must be non-empty when policy is Allowlist" + }]), + ); + serde_json::from_value(value).ok() } /// `KarsSandbox` CRD with workspace storage CEL injected into the generated /// schema. The hand-written Helm CRD carries the same rule. #[must_use] pub fn kars_sandbox_crd() -> CustomResourceDefinition { - inject_kars_sandbox_workspace_validations(crate::crd::KarsSandbox::crd()) - .expect("kube-rs derive must produce spec.storage.workspace") + inject_kars_sandbox_validations(crate::crd::KarsSandbox::crd()) + .expect("kube-rs derive must produce workspace and channel schemas") } #[cfg(test)] diff --git a/controller/src/helm_drift.rs b/controller/src/helm_drift.rs index e65e407bc..05be1d876 100644 --- a/controller/src/helm_drift.rs +++ b/controller/src/helm_drift.rs @@ -234,6 +234,78 @@ mod tests { } } + #[test] + fn helm_and_rust_expose_feishu_channel_contract() { + let rust = serde_json::to_value(kars_sandbox_crd()).expect("rust CRD serializes"); + let helm_text = std::fs::read_to_string(KARSSANDBOX_HELM_CRD_PATH) + .expect("KarsSandbox Helm CRD exists"); + let helm: serde_json::Value = serde_yaml::Deserializer::from_str(&helm_text) + .next() + .map(serde_json::Value::deserialize) + .expect("KarsSandbox document exists") + .expect("KarsSandbox Helm CRD parses"); + const SPEC: &str = "/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties"; + + for (label, crd) in [("Rust", &rust), ("Helm", &helm)] { + let channels = crd + .pointer(&format!("{SPEC}/channels")) + .unwrap_or_else(|| panic!("{label} schema must expose spec.channels")); + assert_eq!( + channels.pointer("/items/properties/type/enum/0"), + Some(&serde_json::json!("Feishu")) + ); + assert_eq!( + channels.pointer("/items/properties/feishu/properties/connectionMode/enum/0"), + Some(&serde_json::json!("WebSocket")) + ); + assert_eq!(channels.get("maxItems"), Some(&serde_json::json!(8))); + assert_eq!( + channels.pointer("/items/properties/credentialSecretRef/properties/name/maxLength"), + Some(&serde_json::json!(253)) + ); + assert_eq!( + channels.pointer("/items/properties/credentialSecretRef/properties/name/pattern"), + Some(&serde_json::json!( + r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + )) + ); + assert_eq!( + channels.pointer("/items/properties/feishu/properties/directMessages/properties/allowFrom/maxItems"), + Some(&serde_json::json!(256)) + ); + assert_eq!( + channels.pointer("/items/properties/feishu/properties/directMessages/properties/allowFrom/items/pattern"), + Some(&serde_json::json!("^ou_[A-Za-z0-9_-]+$")) + ); + assert_eq!( + channels.pointer( + "/items/properties/feishu/properties/groups/properties/allowFrom/items/pattern" + ), + Some(&serde_json::json!("^oc_[A-Za-z0-9_-]+$")) + ); + let dm_validations = channels + .pointer( + "/items/properties/feishu/properties/directMessages/x-kubernetes-validations", + ) + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("{label} Feishu DM CEL validations")); + assert!(dm_validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("Allowlist") && rule.contains("allowFrom")) + })); + let validations = channels + .get("x-kubernetes-validations") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("{label} channel CEL validations")); + assert!(validations.iter().any(|validation| { + validation["rule"] + .as_str() + .is_some_and(|rule| rule.contains("Feishu") && rule.contains("size")) + })); + } + } + #[test] fn helm_toolpolicy_crd_matches_rust_schema() { let rust_crd_value = diff --git a/controller/src/reconciler/mod.rs b/controller/src/reconciler/mod.rs index 675e4819d..e09d3049b 100644 --- a/controller/src/reconciler/mod.rs +++ b/controller/src/reconciler/mod.rs @@ -21,7 +21,7 @@ use k8s_openapi::api::{ }; use kube::{ Client, ResourceExt, - api::{Api, DeleteParams, ListParams, Patch, PatchParams}, + api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams}, runtime::{ controller::{Action, Controller}, reflector::ObjectRef, @@ -41,6 +41,609 @@ mod mcp_egress; pub(crate) mod trustgraph_mount; use mcp_egress::mcp_egress_rule; +const FEISHU_CLAIM_OWNER_UID: &str = "channels.kars.azure.com/owner-uid"; +const FEISHU_CLAIM_OWNER_NAME: &str = "channels.kars.azure.com/owner-name"; +const FEISHU_CLAIM_OWNER_NAMESPACE: &str = "channels.kars.azure.com/owner-namespace"; +const FEISHU_CLAIM_FINGERPRINT: &str = "channels.kars.azure.com/app-fingerprint"; +const FEISHU_CLAIM_OWNER_UID_LABEL: &str = "channels.kars.azure.com/owner-uid"; +const FEISHU_SECRET_MANAGED_LABEL: &str = "channels.kars.azure.com/managed-rotation"; +const FEISHU_SECRET_REVISION_STATE_LABEL: &str = "channels.kars.azure.com/revision-state"; +const FEISHU_SECRET_SANDBOX_LABEL: &str = "kars.azure.com/sandbox"; +const FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: &str = + "channels.kars.azure.com/credentials-secret-uid"; +const FEISHU_POD_CREDENTIALS_SECRET_ANNOTATION: &str = "channels.kars.azure.com/credentials-secret"; + +fn build_feishu_app_claim( + app_id: &str, + owner_namespace: &str, + owner_name: &str, + owner_uid: &str, +) -> ConfigMap { + let fingerprint = crate::config_hash::sha256_hex_prefix(app_id.as_bytes(), 16); + let claim_name = format!("feishu-app-{fingerprint}"); + serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": claim_name, + "labels": { + "app.kubernetes.io/name": "kars", + "app.kubernetes.io/component": "channel-app-claim", + FEISHU_CLAIM_OWNER_UID_LABEL: owner_uid + }, + "annotations": { + FEISHU_CLAIM_OWNER_UID: owner_uid, + FEISHU_CLAIM_OWNER_NAME: owner_name, + FEISHU_CLAIM_OWNER_NAMESPACE: owner_namespace, + FEISHU_CLAIM_FINGERPRINT: fingerprint + } + } + })) + .expect("Feishu App claim shape is valid") +} + +fn feishu_app_claim_owner(claim: &ConfigMap) -> Option { + claim + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FEISHU_CLAIM_OWNER_UID)) + .cloned() +} + +fn feishu_app_claim_matches(claim: &ConfigMap, owner_uid: &str) -> bool { + feishu_app_claim_owner(claim).as_deref() == Some(owner_uid) +} + +fn feishu_app_claims_to_release( + claims: &[ConfigMap], + owner_uid: &str, + current_claim_name: Option<&str>, +) -> Vec { + claims + .iter() + .filter(|claim| { + feishu_app_claim_matches(claim, owner_uid) + && claim.metadata.name.as_deref() != current_claim_name + }) + .filter_map(|claim| claim.metadata.name.clone()) + .collect() +} + +fn feishu_claim_rollout_complete( + pods: &[Pod], + runtime_container_name: &str, + credentials_version: Option<&str>, +) -> bool { + if credentials_version.is_some() && pods.is_empty() { + return false; + } + pods.iter().all(|pod| match credentials_version { + Some(version) => feishu_pod_connected(pod, runtime_container_name, Some(version)), + None => pod + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION)) + .is_none(), + }) +} + +fn feishu_claim_release_ready_on_delete(pods: &[Pod]) -> bool { + pods.is_empty() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum FeishuCredentialRevisionPlan { + UseCurrent(String), + KeepDeployed(String), + Unavailable, +} + +fn plan_feishu_credential_revision( + desired_secret: &str, + deployed_secret: Option<&str>, + deployed_version: Option<&str>, + secret_version: Option<&str>, +) -> FeishuCredentialRevisionPlan { + let Some(secret_version) = secret_version else { + return FeishuCredentialRevisionPlan::Unavailable; + }; + match (deployed_secret, deployed_version) { + (None, None) => FeishuCredentialRevisionPlan::UseCurrent(secret_version.to_string()), + (Some(deployed_secret), _) if deployed_secret != desired_secret => { + FeishuCredentialRevisionPlan::UseCurrent(secret_version.to_string()) + } + (Some(_), Some(deployed)) if deployed == secret_version => { + FeishuCredentialRevisionPlan::UseCurrent(secret_version.to_string()) + } + (Some(_), Some(deployed)) => { + FeishuCredentialRevisionPlan::KeepDeployed(deployed.to_string()) + } + _ => FeishuCredentialRevisionPlan::Unavailable, + } +} + +fn deployment_feishu_credentials_version(deployment: Option<&Deployment>) -> Option<&str> { + deployment + .and_then(|deployment| deployment.spec.as_ref()) + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|metadata| metadata.annotations.as_ref()) + .and_then(|annotations| annotations.get(FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION)) + .map(String::as_str) +} + +fn deployment_feishu_credentials_secret(deployment: Option<&Deployment>) -> Option<&str> { + deployment + .and_then(|deployment| deployment.spec.as_ref()) + .and_then(|spec| spec.template.metadata.as_ref()) + .and_then(|metadata| metadata.annotations.as_ref()) + .and_then(|annotations| annotations.get(FEISHU_POD_CREDENTIALS_SECRET_ANNOTATION)) + .map(String::as_str) +} + +fn feishu_app_id(secret: &Secret) -> Option { + secret + .data + .as_ref() + .and_then(|data| data.get("FEISHU_APP_ID")) + .and_then(|value| String::from_utf8(value.0.clone()).ok()) + .filter(|value| !value.is_empty()) +} + +async fn acquire_feishu_app_claim( + client: &Client, + app_id: &str, + owner_namespace: &str, + owner_name: &str, + owner_uid: &str, +) -> std::result::Result<(), kube::Error> { + let claim_namespace = std::env::var("POD_NAMESPACE") + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "kars-system".to_string()); + let api: Api = Api::namespaced(client.clone(), &claim_namespace); + let desired = build_feishu_app_claim(app_id, owner_namespace, owner_name, owner_uid); + match api.create(&PostParams::default(), &desired).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(error)) if error.code == 409 => { + let existing = api.get(&desired.name_any()).await?; + let annotations_match = existing + .metadata + .annotations + .as_ref() + .is_some_and(|actual| { + desired + .metadata + .annotations + .as_ref() + .is_some_and(|expected| { + actual.get(FEISHU_CLAIM_OWNER_UID) + == expected.get(FEISHU_CLAIM_OWNER_UID) + && actual.get(FEISHU_CLAIM_FINGERPRINT) + == expected.get(FEISHU_CLAIM_FINGERPRINT) + }) + }); + if annotations_match { + Ok(()) + } else { + Err(kube::Error::Api(error)) + } + } + Err(error) => Err(error), + } +} + +async fn release_feishu_app_claims( + client: &Client, + owner_uid: &str, + current_claim_name: Option<&str>, +) -> std::result::Result<(), kube::Error> { + let claim_namespace = std::env::var("POD_NAMESPACE") + .ok() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "kars-system".to_string()); + let api: Api = Api::namespaced(client.clone(), &claim_namespace); + let selector = format!("{FEISHU_CLAIM_OWNER_UID_LABEL}={owner_uid}"); + let claims = api + .list(&ListParams::default().labels(&selector)) + .await? + .items; + for claim_name in feishu_app_claims_to_release(&claims, owner_uid, current_claim_name) { + api.delete(&claim_name, &DeleteParams::default()).await?; + } + Ok(()) +} + +async fn cleanup_managed_feishu_secrets( + client: &Client, + namespace: &str, + sandbox_name: &str, + current_secret_name: Option<&str>, +) -> std::result::Result<(), kube::Error> { + let api = Api::::namespaced(client.clone(), namespace); + let selector = format!( + "{FEISHU_SECRET_MANAGED_LABEL}=true,{FEISHU_SECRET_REVISION_STATE_LABEL}=adopted,{FEISHU_SECRET_SANDBOX_LABEL}={sandbox_name}" + ); + for secret in api + .list(&ListParams::default().labels(&selector)) + .await? + .items + { + if managed_feishu_secret_cleanup_candidate(&secret, current_secret_name) { + match api + .delete(&secret.name_any(), &DeleteParams::default()) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(error), + } + } + } + Ok(()) +} + +fn managed_feishu_secret_cleanup_candidate( + secret: &Secret, + current_secret_name: Option<&str>, +) -> bool { + secret.metadata.name.as_deref() != current_secret_name + && secret + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(FEISHU_SECRET_REVISION_STATE_LABEL)) + .is_some_and(|state| state == "adopted") +} + +async fn adopt_managed_feishu_secret( + client: &Client, + namespace: &str, + sandbox_name: &str, + secret: &Secret, +) -> std::result::Result<(), kube::Error> { + let labels = secret.metadata.labels.as_ref(); + let managed = labels + .and_then(|labels| labels.get(FEISHU_SECRET_MANAGED_LABEL)) + .is_some_and(|value| value == "true"); + let owned = labels + .and_then(|labels| labels.get(FEISHU_SECRET_SANDBOX_LABEL)) + .is_some_and(|value| value == sandbox_name); + let adopted = labels + .and_then(|labels| labels.get(FEISHU_SECRET_REVISION_STATE_LABEL)) + .is_some_and(|value| value == "adopted"); + if !managed || !owned || adopted { + return Ok(()); + } + let Some(secret_name) = secret.metadata.name.as_deref() else { + return Ok(()); + }; + Api::::namespaced(client.clone(), namespace) + .patch( + secret_name, + &PatchParams::default(), + &Patch::Merge(json!({ + "metadata": {"labels": {FEISHU_SECRET_REVISION_STATE_LABEL: "adopted"}} + })), + ) + .await?; + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FeishuCredentialState { + Missing, + Partial, + Invalid, + Complete, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FeishuChannelRuntimeState { + Configured, + Connecting, + Failed, + Suspended, +} + +fn feishu_pod_connected( + pod: &Pod, + runtime_container_name: &str, + credentials_version: Option<&str>, +) -> bool { + let pod_credentials_version = pod + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION)) + .map(String::as_str); + if pod_credentials_version != credentials_version { + return false; + } + let has_channel_probe = pod.spec.as_ref().is_some_and(|spec| { + spec.containers.iter().any(|container| { + container.name == runtime_container_name + && container + .readiness_probe + .as_ref() + .and_then(|probe| probe.exec.as_ref()) + .and_then(|action| action.command.as_ref()) + .is_some_and(|command| { + command + .iter() + .any(|part| part.contains("kars-channel-feishu-ready")) + }) + }) + }); + has_channel_probe + && pod + .status + .as_ref() + .and_then(|status| status.container_statuses.as_ref()) + .is_some_and(|statuses| { + statuses + .iter() + .any(|status| status.name == runtime_container_name && status.ready) + }) +} + +fn feishu_pod_runtime_failed( + pod: &Pod, + runtime_container_name: &str, + credentials_version: Option<&str>, +) -> bool { + let pod_credentials_version = pod + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION)) + .map(String::as_str); + if pod_credentials_version != credentials_version { + return false; + } + pod.status + .as_ref() + .and_then(|status| status.container_statuses.as_ref()) + .and_then(|statuses| { + statuses + .iter() + .find(|status| status.name == runtime_container_name) + }) + .and_then(|status| status.state.as_ref()) + .is_some_and(|state| { + state.terminated.is_some() + || state + .waiting + .as_ref() + .and_then(|waiting| waiting.reason.as_deref()) + .is_some_and(|reason| { + matches!( + reason, + "CrashLoopBackOff" + | "CreateContainerConfigError" + | "CreateContainerError" + | "ErrImagePull" + | "ImagePullBackOff" + | "InvalidImageName" + | "RunContainerError" + ) + }) + }) +} + +fn feishu_channel_runtime_state( + pods: &[Pod], + runtime_container_name: &str, + credentials_version: Option<&str>, + suspended: bool, +) -> FeishuChannelRuntimeState { + if suspended { + return FeishuChannelRuntimeState::Suspended; + } + if pods + .iter() + .any(|pod| feishu_pod_connected(pod, runtime_container_name, credentials_version)) + { + FeishuChannelRuntimeState::Configured + } else if pods + .iter() + .any(|pod| feishu_pod_runtime_failed(pod, runtime_container_name, credentials_version)) + { + FeishuChannelRuntimeState::Failed + } else { + FeishuChannelRuntimeState::Connecting + } +} + +fn feishu_channel_status_conditions( + sandbox: &KarsSandbox, + state: FeishuChannelRuntimeState, +) -> Vec { + let prior = sandbox + .status + .as_ref() + .map(|status| status.conditions.as_slice()) + .unwrap_or(&[]); + let generation = sandbox.metadata.generation; + let (channel_status, reason, message) = match state { + FeishuChannelRuntimeState::Configured => ( + crate::status::conditions::status::TRUE, + crate::status::conditions::reason::CHANNEL_CONFIGURED, + "Feishu runtime adapter reports an active WebSocket connection", + ), + FeishuChannelRuntimeState::Connecting => ( + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CHANNEL_CONNECTING, + "Feishu configuration is valid; waiting for the runtime WebSocket connection", + ), + FeishuChannelRuntimeState::Failed => ( + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CHANNEL_CONNECTION_FAILED, + "Feishu runtime adapter failed to start or maintain its connection", + ), + FeishuChannelRuntimeState::Suspended => ( + crate::status::conditions::status::FALSE, + crate::status::conditions::reason::CHANNEL_SUSPENDED, + "sandbox is suspended; no Feishu WebSocket consumer is running", + ), + }; + let channel_ready = crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_CHANNEL_READY), + crate::status::conditions::TYPE_CHANNEL_READY, + channel_status, + reason, + message, + generation, + ); + let mut conditions = vec![channel_ready]; + if channel_status == crate::status::conditions::status::FALSE { + conditions.push(crate::status::conditions::preserve_transition_time( + crate::status::conditions::find(prior, crate::status::conditions::TYPE_READY), + crate::status::conditions::TYPE_READY, + crate::status::conditions::status::FALSE, + reason, + message, + generation, + )); + } + conditions +} + +fn channel_capability_failure_condition() -> (&'static str, &'static str) { + ( + crate::status::conditions::TYPE_CHANNEL_READY, + crate::status::conditions::reason::CHANNEL_UNSUPPORTED_RUNTIME, + ) +} + +fn is_feishu_id(value: &str, prefix: &str) -> bool { + value.strip_prefix(prefix).is_some_and(|suffix| { + !suffix.is_empty() + && suffix + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') + }) +} + +fn build_channel_policy_env( + channels: &[crate::crd::ChannelSpec], +) -> Result, String> { + let Some(channel) = channels + .iter() + .find(|channel| channel.type_ == crate::crd::ChannelType::Feishu) + else { + return Ok(Vec::new()); + }; + let feishu = channel + .feishu + .as_ref() + .ok_or_else(|| "Feishu channel requires spec.channels[].feishu".to_string())?; + if feishu.direct_messages.policy == crate::crd::DirectMessageAccess::Allowlist + && feishu.direct_messages.allow_from.is_empty() + { + return Err("Feishu DM allowlist policy requires at least one ou_ user ID".to_string()); + } + if let Some(invalid) = feishu + .direct_messages + .allow_from + .iter() + .find(|value| !is_feishu_id(value, "ou_")) + { + return Err(format!("invalid Feishu user Open ID `{invalid}`")); + } + if let Some(invalid) = feishu + .groups + .allow_from + .iter() + .find(|value| !is_feishu_id(value, "oc_")) + { + return Err(format!("invalid Feishu group chat ID `{invalid}`")); + } + + let domain = match feishu.domain { + crate::crd::FeishuDomain::Feishu => "feishu", + crate::crd::FeishuDomain::Lark => "lark", + }; + let connection_mode = match feishu.connection_mode { + crate::crd::FeishuConnectionMode::WebSocket => "websocket", + }; + let dm_policy = match feishu.direct_messages.policy { + crate::crd::DirectMessageAccess::Pairing => "pairing", + crate::crd::DirectMessageAccess::Allowlist => "allowlist", + crate::crd::DirectMessageAccess::Disabled => "disabled", + }; + let group_policy = match feishu.groups.policy { + crate::crd::GroupAccess::Allowlist => "allowlist", + crate::crd::GroupAccess::Disabled => "disabled", + }; + + Ok(vec![ + json!({"name": "FEISHU_DOMAIN", "value": domain}), + json!({"name": "FEISHU_CONNECTION_MODE", "value": connection_mode}), + json!({"name": "FEISHU_DM_POLICY", "value": dm_policy}), + json!({"name": "FEISHU_ALLOW_FROM", "value": feishu.direct_messages.allow_from.join(",")}), + json!({"name": "FEISHU_GROUP_POLICY", "value": group_policy}), + json!({"name": "FEISHU_GROUP_ALLOW_FROM", "value": feishu.groups.allow_from.join(",")}), + json!({"name": "FEISHU_REQUIRE_MENTION", "value": feishu.groups.require_mention.to_string()}), + ]) +} + +fn feishu_credential_state(secret: Option<&Secret>) -> FeishuCredentialState { + let value = |key: &str| -> Result, std::str::Utf8Error> { + let Some(bytes) = secret + .and_then(|secret| secret.data.as_ref()) + .and_then(|data| data.get(key)) + else { + return Ok(None); + }; + let decoded = std::str::from_utf8(&bytes.0)?; + Ok((!decoded.is_empty()).then_some(decoded)) + }; + match (value("FEISHU_APP_ID"), value("FEISHU_APP_SECRET")) { + (Err(_), _) | (_, Err(_)) => FeishuCredentialState::Invalid, + (Ok(None), Ok(None)) => FeishuCredentialState::Missing, + (Ok(Some(app_id)), Ok(Some(_))) if is_feishu_id(app_id, "cli_") => { + FeishuCredentialState::Complete + } + (Ok(Some(_)), Ok(Some(_))) => FeishuCredentialState::Invalid, + _ => FeishuCredentialState::Partial, + } +} + +fn channel_credential_secret_name(sandbox_name: &str, channel: &crate::crd::ChannelSpec) -> String { + channel + .credential_secret_ref + .as_ref() + .map(|reference| reference.name.clone()) + .unwrap_or_else(|| format!("{sandbox_name}-credentials")) +} + +fn feishu_credential_env( + sandbox_name: &str, + channels: &[crate::crd::ChannelSpec], +) -> Vec { + let Some(channel) = channels + .iter() + .find(|channel| channel.type_ == crate::crd::ChannelType::Feishu) + else { + return Vec::new(); + }; + let secret_name = channel_credential_secret_name(sandbox_name, channel); + ["FEISHU_APP_ID", "FEISHU_APP_SECRET"] + .into_iter() + .map(|key| { + json!({ + "name": key, + "valueFrom": {"secretKeyRef": {"name": secret_name, "key": key}} + }) + }) + .collect() +} + +fn runtime_credentials_secret_name(sandbox_name: &str) -> String { + format!("{sandbox_name}-credentials") +} #[derive(Debug, Clone, PartialEq)] struct WorkspaceStoragePlan { @@ -796,6 +1399,43 @@ async fn stamp_degraded_with_condition( } } +fn channel_fail_closed_deployment_patch() -> serde_json::Value { + json!({"spec": {"replicas": 0}}) +} + +async fn stamp_channel_failure_fail_closed( + client: &Client, + sandbox: &KarsSandbox, + sandbox_namespace: &str, + name: &str, + reason: &'static str, + message: &str, +) -> Result<(), kube::Error> { + let deployment_api = Api::::namespaced(client.clone(), sandbox_namespace); + match deployment_api + .patch( + name, + &PatchParams::default(), + &Patch::Merge(channel_fail_closed_deployment_patch()), + ) + .await + { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(error), + } + stamp_degraded_with_condition( + client, + sandbox, + name, + crate::status::conditions::TYPE_CHANNEL_READY, + reason, + message, + ) + .await; + Ok(()) +} + /// Build pod security context, conditionally including SELinux options and /// choosing between RuntimeDefault and Localhost seccomp profiles. /// For Kata (confidential), we use RuntimeDefault since the VM provides isolation. @@ -1299,6 +1939,25 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result::namespaced(client.clone(), &sandbox_ns); + let pods = match pod_api + .list(&ListParams::default().labels(&format!("kars.azure.com/sandbox={name}"))) + .await + { + Ok(pods) => pods.items, + Err(kube::Error::Api(error)) if error.code == 404 => Vec::new(), + Err(error) => return Err(error.into()), + }; + if !feishu_claim_release_ready_on_delete(&pods) { + return Ok(Action::requeue(Duration::from_secs(2))); + } + if let Err(error) = release_feishu_app_claims(client, owner_uid, None).await { + tracing::warn!(sandbox = %name, error = %error, "Feishu App claim cleanup failed"); + return Ok(Action::requeue(Duration::from_secs(10))); + } + } + // Remove the finalizer so K8s can complete CRD deletion let sandbox_api: Api = Api::namespaced(client.clone(), &sandbox.namespace().unwrap_or_default()); @@ -1353,6 +2012,36 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result env, + Err(message) => { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_POLICY_INVALID, + &message, + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(300))); + } + }; if let Some(workspace) = spec .storage .as_ref() @@ -1731,6 +2420,147 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result = None; + let mut feishu_credentials_version: Option = None; + if has_feishu_channel { + let secret_api: Api = Api::namespaced(client.clone(), &sandbox_ns); + let secret_name = feishu_credentials_secret + .as_deref() + .expect("Feishu channel has a credential Secret name"); + let secret = secret_api.get_opt(secret_name).await?; + let (reason, message) = match feishu_credential_state(secret.as_ref()) { + FeishuCredentialState::Complete => (None, None), + FeishuCredentialState::Missing => ( + Some(crate::status::conditions::reason::CHANNEL_CREDENTIALS_MISSING), + Some(format!( + "Feishu credentials are missing from Secret `{secret_name}`" + )), + ), + FeishuCredentialState::Partial => ( + Some(crate::status::conditions::reason::CHANNEL_CREDENTIALS_PARTIAL), + Some(format!( + "Feishu credentials are incomplete in Secret `{secret_name}`" + )), + ), + FeishuCredentialState::Invalid => ( + Some(crate::status::conditions::reason::CHANNEL_POLICY_INVALID), + Some(format!( + "Feishu credentials in Secret `{secret_name}` are malformed" + )), + ), + }; + if let (Some(reason), Some(message)) = (reason, message) { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + reason, + &message, + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(30))); + } + if secret.as_ref().and_then(|secret| secret.immutable) != Some(true) { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_POLICY_INVALID, + "Feishu credential Secret must set immutable: true", + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(300))); + } + if let Some(secret) = secret.as_ref() { + adopt_managed_feishu_secret(client, &sandbox_ns, &name, secret).await?; + } + let secret_version = secret + .as_ref() + .and_then(|secret| secret.metadata.uid.clone()); + let deployed = Api::::namespaced(client.clone(), &sandbox_ns) + .get_opt(&name) + .await?; + let revision_plan = plan_feishu_credential_revision( + secret_name, + deployment_feishu_credentials_secret(deployed.as_ref()), + deployment_feishu_credentials_version(deployed.as_ref()), + secret_version.as_deref(), + ); + match revision_plan { + FeishuCredentialRevisionPlan::KeepDeployed(_version) => { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_CONNECTING, + "Feishu credentials changed in place; use kars credentials update to stage an immutable Secret", + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(30))); + } + FeishuCredentialRevisionPlan::Unavailable => { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_CONNECTING, + "requested Feishu credential revision is unavailable; runtime remains stopped", + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(30))); + } + FeishuCredentialRevisionPlan::UseCurrent(version) => { + feishu_credentials_version = Some(version); + let Some(app_id) = secret.as_ref().and_then(feishu_app_id) else { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_POLICY_INVALID, + "Feishu credential App ID is malformed", + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(300))); + }; + let Some(owner_uid) = sandbox.metadata.uid.as_deref() else { + return Ok(Action::requeue(Duration::from_secs(5))); + }; + if let Err(error) = + acquire_feishu_app_claim(client, &app_id, &sandbox_self_ns, &name, owner_uid) + .await + { + if matches!(&error, kube::Error::Api(api_error) if api_error.code == 409) { + stamp_channel_failure_fail_closed( + client, + &sandbox, + &sandbox_ns, + &name, + crate::status::conditions::reason::CHANNEL_APP_ALREADY_CLAIMED, + "Feishu App credentials are already owned by another sandbox", + ) + .await?; + return Ok(Action::requeue(Duration::from_secs(300))); + } + return Err(error.into()); + } + let current_claim = + build_feishu_app_claim(&app_id, &sandbox_self_ns, &name, owner_uid); + current_feishu_claim_name = current_claim.metadata.name; + } + } + } + let claim_api: Api = Api::namespaced(client.clone(), &sandbox_ns); let namespace_claims = claim_api.list(&ListParams::default()).await?.items; let requested_existing_claim = spec @@ -2651,8 +3481,6 @@ async fn reconcile(sandbox: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result::namespaced(client.clone(), &sandbox_ns) + .list(&ListParams::default().labels(&format!("kars.azure.com/sandbox={name}"))) + .await?; + let channel_state = feishu_channel_runtime_state( + &pods.items, + runtime_container_name, + feishu_credentials_version.as_deref(), + spec.suspended.unwrap_or(false), + ); + extras.extend(feishu_channel_status_conditions(&sandbox, channel_state)); + if feishu_claim_rollout_complete( + &pods.items, + runtime_container_name, + feishu_credentials_version.as_deref(), + ) && current_feishu_claim_name.is_some() + && let Some(owner_uid) = sandbox.metadata.uid.as_deref() + { + release_feishu_app_claims(client, owner_uid, current_feishu_claim_name.as_deref()) + .await?; + if let Some(current_secret_name) = feishu_credentials_secret.as_deref() { + cleanup_managed_feishu_secrets( + client, + &sandbox_ns, + &name, + Some(current_secret_name), + ) + .await?; + } + } + } else if let Some(owner_uid) = sandbox.metadata.uid.as_deref() { + let pods = Api::::namespaced(client.clone(), &sandbox_ns) + .list(&ListParams::default().labels(&format!("kars.azure.com/sandbox={name}"))) + .await?; + if feishu_claim_rollout_complete(&pods.items, "", None) { + release_feishu_app_claims(client, owner_uid, None).await?; + cleanup_managed_feishu_secrets(client, &sandbox_ns, &name, None).await?; + } + } + // Phase G P1 #4: stamp Suspended condition when spec.suspended // is true, or surface Suspended=False/Active when there is a // prior Suspended condition that was operator-driven. We diff --git a/controller/src/reconciler/runtime.rs b/controller/src/reconciler/runtime.rs index 6580d0a93..1371c8c3a 100644 --- a/controller/src/reconciler/runtime.rs +++ b/controller/src/reconciler/runtime.rs @@ -34,9 +34,9 @@ use std::collections::BTreeMap; use crate::crd::{ - AgentCodeRef, AnthropicConfig, ByoRuntimeConfig, HermesConfig, LangGraphConfig, - LangGraphLanguage, MafLanguage, MicrosoftAgentFrameworkConfig, OpenAIAgentsConfig, - OpenClawConfig, PydanticAiConfig, RuntimeKind, RuntimeSpec, + AgentCodeRef, AnthropicConfig, ByoRuntimeConfig, ChannelSpec, ChannelType, HermesConfig, + LangGraphConfig, LangGraphLanguage, MafLanguage, MicrosoftAgentFrameworkConfig, + OpenAIAgentsConfig, OpenClawConfig, PydanticAiConfig, RuntimeKind, RuntimeSpec, }; /// Default container image for the OpenAI Agents Python runtime @@ -330,6 +330,45 @@ pub fn validate_runtime_shape(runtime: &RuntimeSpec) -> Result<(), RuntimePlanEr Ok(()) } +pub fn validate_channel_capabilities( + runtime_kind: &RuntimeKind, + channels: &[ChannelSpec], +) -> Result<(), RuntimePlanError> { + if channels.len() > 8 { + return Err(RuntimePlanError::ShapeInvalid( + "spec.channels may contain at most 8 entries".into(), + )); + } + let feishu_count = channels + .iter() + .filter(|channel| channel.type_ == ChannelType::Feishu) + .count(); + if feishu_count > 1 { + return Err(RuntimePlanError::ShapeInvalid( + "spec.channels may contain at most one Feishu channel".into(), + )); + } + for channel in channels { + match channel.type_ { + ChannelType::Feishu + if !matches!(runtime_kind, RuntimeKind::OpenClaw | RuntimeKind::Hermes) => + { + return Err(RuntimePlanError::ShapeInvalid(format!( + "Feishu channel is unsupported by runtime {}", + kind_str(runtime_kind) + ))); + } + ChannelType::Feishu if channel.feishu.is_none() => { + return Err(RuntimePlanError::ShapeInvalid( + "Feishu channel requires spec.channels[].feishu".into(), + )); + } + ChannelType::Feishu => {} + } + } + Ok(()) +} + /// Produce a [`RuntimeDeploymentPlan`] for the given runtime spec, or an /// [`AdapterMissing`](RuntimePlanError::AdapterMissing) error if the kind /// has no adapter in this build. @@ -758,9 +797,9 @@ fn plan_hermes(cfg: &HermesConfig) -> RuntimeDeploymentPlan { mod tests { use super::*; use crate::crd::{ - AgentCodeRef, AnthropicConfig, ByoRuntimeConfig, HermesConfig, LangGraphConfig, - LangGraphLanguage, MafLanguage, MicrosoftAgentFrameworkConfig, OciAgentCode, - OpenAIAgentsConfig, OpenClawConfig, PydanticAiConfig, SemanticKernelConfig, + AgentCodeRef, AnthropicConfig, ByoRuntimeConfig, FeishuChannelSpec, HermesConfig, + LangGraphConfig, LangGraphLanguage, MafLanguage, MicrosoftAgentFrameworkConfig, + OciAgentCode, OpenAIAgentsConfig, OpenClawConfig, PydanticAiConfig, SemanticKernelConfig, }; use std::sync::Mutex; @@ -825,6 +864,58 @@ mod tests { assert!(validate_runtime_shape(&rt).is_ok()); } + #[test] + fn feishu_channel_capability_accepts_openclaw_and_hermes() { + let channels = vec![ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }]; + assert!(validate_channel_capabilities(&RuntimeKind::OpenClaw, &channels).is_ok()); + assert!(validate_channel_capabilities(&RuntimeKind::Hermes, &channels).is_ok()); + } + + #[test] + fn feishu_channel_capability_rejects_other_runtimes() { + let channels = vec![ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }]; + for kind in [ + RuntimeKind::OpenAIAgents, + RuntimeKind::MicrosoftAgentFramework, + RuntimeKind::LangGraph, + RuntimeKind::Anthropic, + RuntimeKind::PydanticAi, + RuntimeKind::BYO, + ] { + let error = validate_channel_capabilities(&kind, &channels).unwrap_err(); + assert!(matches!(error, RuntimePlanError::ShapeInvalid(ref message) + if message.contains("Feishu") && message.contains(kind_str(&kind)))); + } + } + + #[test] + fn feishu_channel_shape_rejects_duplicates_and_missing_config() { + let missing = vec![ChannelSpec { + type_: crate::crd::ChannelType::Feishu, + credential_secret_ref: None, + feishu: None, + }]; + assert!(validate_channel_capabilities(&RuntimeKind::OpenClaw, &missing).is_err()); + + let channel = ChannelSpec { + type_: crate::crd::ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }; + assert!( + validate_channel_capabilities(&RuntimeKind::OpenClaw, &[channel.clone(), channel]) + .is_err() + ); + } + #[test] fn validate_rejects_kind_without_matching_variant_struct() { let mut rt = rt_only_kind(RuntimeKind::OpenAIAgents); diff --git a/controller/src/reconciler/tests.rs b/controller/src/reconciler/tests.rs index feabeb550..f0a216c39 100644 --- a/controller/src/reconciler/tests.rs +++ b/controller/src/reconciler/tests.rs @@ -13,11 +13,508 @@ use super::*; use crate::crd::{ - OpenClawConfig, OpenClawWorkspaceSpec, PersistentVolumeAccessMode, SandboxConfig, - SandboxStorageSpec, WorkspaceOverwritePolicy, WorkspaceRetainPolicy, WorkspaceStorageSpec, + ChannelSpec, ChannelType, DirectMessageAccess, DirectMessagePolicy, FeishuChannelSpec, + FeishuConnectionMode, FeishuDomain, GroupAccess, GroupPolicy, OpenClawConfig, + OpenClawWorkspaceSpec, PersistentVolumeAccessMode, SandboxConfig, SandboxStorageSpec, + WorkspaceOverwritePolicy, WorkspaceRetainPolicy, WorkspaceStorageSpec, }; use crate::mcp_server::LocalObjectRef; +#[test] +fn feishu_policy_env_uses_common_runtime_contract() { + let channels = vec![ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec { + domain: FeishuDomain::Lark, + connection_mode: FeishuConnectionMode::WebSocket, + direct_messages: DirectMessagePolicy { + policy: DirectMessageAccess::Allowlist, + allow_from: vec!["ou_teacher".into()], + }, + groups: GroupPolicy { + policy: GroupAccess::Allowlist, + allow_from: vec!["oc_teaching".into(), "oc_admin".into()], + require_mention: false, + }, + }), + }]; + + let env = build_channel_policy_env(&channels).expect("valid policy"); + let values = env + .iter() + .map(|entry| { + ( + entry["name"].as_str().unwrap(), + entry["value"].as_str().unwrap(), + ) + }) + .collect::>(); + assert_eq!(values["FEISHU_DOMAIN"], "lark"); + assert_eq!(values["FEISHU_CONNECTION_MODE"], "websocket"); + assert_eq!(values["FEISHU_DM_POLICY"], "allowlist"); + assert_eq!(values["FEISHU_ALLOW_FROM"], "ou_teacher"); + assert_eq!(values["FEISHU_GROUP_POLICY"], "allowlist"); + assert_eq!(values["FEISHU_GROUP_ALLOW_FROM"], "oc_teaching,oc_admin"); + assert_eq!(values["FEISHU_REQUIRE_MENTION"], "false"); +} + +#[test] +fn feishu_policy_rejects_invalid_user_and_group_ids() { + let mut channel = ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }; + channel.feishu.as_mut().unwrap().direct_messages = DirectMessagePolicy { + policy: DirectMessageAccess::Allowlist, + allow_from: vec!["user-1".into()], + }; + assert!(build_channel_policy_env(&[channel.clone()]).is_err()); + channel.feishu.as_mut().unwrap().direct_messages = DirectMessagePolicy::default(); + channel.feishu.as_mut().unwrap().groups.allow_from = vec!["group-1".into()]; + assert!(build_channel_policy_env(&[channel]).is_err()); +} + +#[test] +fn feishu_credential_state_never_exposes_values() { + let complete: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "agent-credentials"}, + "data": { + "FEISHU_APP_ID": "Y2xpX3Rlc3Q=", + "FEISHU_APP_SECRET": "c3VwZXItc2VjcmV0" + } + })) + .unwrap(); + assert_eq!( + feishu_credential_state(Some(&complete)), + FeishuCredentialState::Complete + ); + + let partial: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "agent-credentials"}, + "data": {"FEISHU_APP_ID": "Y2xpX3Rlc3Q="} + })) + .unwrap(); + assert_eq!( + feishu_credential_state(Some(&partial)), + FeishuCredentialState::Partial + ); + assert_eq!( + feishu_credential_state(None), + FeishuCredentialState::Missing + ); + assert!(!format!("{:?}", feishu_credential_state(Some(&complete))).contains("secret")); + + let invalid_utf8: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "agent-credentials"}, + "data": { + "FEISHU_APP_ID": "/w==", + "FEISHU_APP_SECRET": "c2VjcmV0" + } + })) + .unwrap(); + assert_eq!( + feishu_credential_state(Some(&invalid_utf8)), + FeishuCredentialState::Invalid + ); +} + +#[test] +fn feishu_secret_name_uses_explicit_ref_or_sandbox_default() { + let default = ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }; + assert_eq!( + channel_credential_secret_name("agent", &default), + "agent-credentials" + ); + let explicit = ChannelSpec { + credential_secret_ref: Some(LocalObjectRef { + name: "custom-feishu".into(), + }), + ..default + }; + assert_eq!( + channel_credential_secret_name("agent", &explicit), + "custom-feishu" + ); +} + +#[test] +fn channel_env_from_uses_declared_feishu_secret() { + let channels = vec![ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: Some(LocalObjectRef { + name: "custom-feishu".into(), + }), + feishu: Some(FeishuChannelSpec::default()), + }]; + assert_eq!( + channel_credential_secret_name("agent", &channels[0]), + "custom-feishu" + ); + assert_eq!( + runtime_credentials_secret_name("agent"), + "agent-credentials" + ); +} + +#[test] +fn feishu_credentials_use_explicit_secret_key_refs() { + let channel = ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: Some(LocalObjectRef { + name: "custom-feishu".into(), + }), + feishu: Some(FeishuChannelSpec::default()), + }; + let env = feishu_credential_env("agent", &[channel]); + assert_eq!(env.len(), 2); + assert!( + env.iter() + .all(|entry| { entry["valueFrom"]["secretKeyRef"]["name"] == "custom-feishu" }) + ); + assert_eq!( + runtime_credentials_secret_name("agent"), + "agent-credentials" + ); +} + +#[test] +fn channel_policy_env_names_are_feishu_only_and_non_secret() { + let channels = vec![ChannelSpec { + type_: ChannelType::Feishu, + credential_secret_ref: None, + feishu: Some(FeishuChannelSpec::default()), + }]; + let env = build_channel_policy_env(&channels).unwrap(); + let names = env + .iter() + .filter_map(|entry| entry["name"].as_str()) + .collect::>(); + assert!(names.iter().all(|name| name.starts_with("FEISHU_"))); + assert!(!names.contains(&"FEISHU_APP_ID")); + assert!(!names.contains(&"FEISHU_APP_SECRET")); +} + +#[test] +fn feishu_channel_status_tracks_runtime_probe_and_suspension() { + let ready_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-abc", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "42" + }}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "openclaw:test", + "readinessProbe": {"exec": {"command": ["sh", "-c", "kars-channel-feishu-ready"]}} + }]}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": true}]} + })) + .unwrap(); + let connecting_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-def"}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": false}]} + })) + .unwrap(); + + assert_eq!( + feishu_channel_runtime_state(&[ready_pod], "openclaw", Some("42"), false), + FeishuChannelRuntimeState::Configured + ); + assert_eq!( + feishu_channel_runtime_state(&[connecting_pod], "openclaw", Some("42"), false), + FeishuChannelRuntimeState::Connecting + ); + assert_eq!( + feishu_channel_runtime_state(&[], "openclaw", Some("42"), true), + FeishuChannelRuntimeState::Suspended + ); + + let generic_ready_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-old"}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "openclaw:old", + "readinessProbe": {"exec": {"command": ["sh", "-c", "test -f /proc/1/status"]}} + }]}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": true}]} + })) + .unwrap(); + assert_eq!( + feishu_channel_runtime_state(&[generic_ready_pod], "openclaw", Some("42"), false), + FeishuChannelRuntimeState::Connecting + ); + + let stale_ready_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-stale", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "41" + }}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "openclaw:test", + "readinessProbe": {"exec": {"command": ["sh", "-c", "kars-channel-feishu-ready"]}} + }]}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": true}]} + })) + .unwrap(); + assert_eq!( + feishu_channel_runtime_state(&[stale_ready_pod], "openclaw", Some("42"), false), + FeishuChannelRuntimeState::Connecting + ); + + let crash_looping_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-crash", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "42" + }}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "openclaw:test", + "readinessProbe": {"exec": {"command": ["sh", "-c", "kars-channel-feishu-ready"]}} + }]}, + "status": {"containerStatuses": [{ + "name": "openclaw", + "ready": false, + "state": {"waiting": {"reason": "CrashLoopBackOff"}} + }]} + })) + .unwrap(); + assert_eq!( + feishu_channel_runtime_state(&[crash_looping_pod], "openclaw", Some("42"), false), + FeishuChannelRuntimeState::Failed + ); +} + +#[test] +fn channel_readiness_condition_gates_overall_ready() { + let sandbox = KarsSandbox { + metadata: kube::api::ObjectMeta { + name: Some("agent".into()), + namespace: Some("default".into()), + generation: Some(2), + ..Default::default() + }, + spec: crate::crd::KarsSandboxSpec::default(), + status: None, + }; + let extras = feishu_channel_status_conditions(&sandbox, FeishuChannelRuntimeState::Connecting); + let patch = crate::status::build_running_status_patch_with_extras( + &sandbox, + "kars-agent", + "OpenClaw", + &extras, + ); + let conditions = patch["status"]["conditions"].as_array().unwrap(); + assert!(conditions.iter().any(|condition| { + condition["type"] == "ChannelReady" + && condition["status"] == "False" + && condition["reason"] == "Connecting" + })); + assert!( + conditions + .iter() + .any(|condition| { condition["type"] == "Ready" && condition["status"] == "False" }) + ); +} + +#[test] +fn feishu_app_claim_is_deterministic_and_uid_owned() { + let first = build_feishu_app_claim("cli_test", "default", "agent-a", "uid-a"); + let same = build_feishu_app_claim("cli_test", "default", "agent-a", "uid-a"); + let other_app = build_feishu_app_claim("cli_other", "default", "agent-a", "uid-a"); + + assert_eq!(first.metadata.name, same.metadata.name); + assert_ne!(first.metadata.name, other_app.metadata.name); + assert!(!format!("{first:?}").contains("cli_test")); + assert_eq!(feishu_app_claim_owner(&first).as_deref(), Some("uid-a")); + assert!(feishu_app_claim_matches(&first, "uid-a")); + assert!(!feishu_app_claim_matches(&first, "uid-b")); +} + +#[test] +fn feishu_app_claim_cleanup_preserves_only_current_app() { + let current = build_feishu_app_claim("cli_current", "default", "agent", "uid-a"); + let stale = build_feishu_app_claim("cli_stale", "default", "agent", "uid-a"); + let other_owner = build_feishu_app_claim("cli_other", "default", "other", "uid-b"); + let release = feishu_app_claims_to_release( + &[current.clone(), stale.clone(), other_owner], + "uid-a", + current.metadata.name.as_deref(), + ); + + assert_eq!(release, vec![stale.name_any()]); +} + +#[test] +fn feishu_secret_cleanup_ignores_unadopted_staged_revisions() { + let staged: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "agent-feishu-staged", + "labels": {FEISHU_SECRET_REVISION_STATE_LABEL: "staged"} + } + })) + .unwrap(); + let adopted: Secret = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "agent-feishu-adopted", + "labels": {FEISHU_SECRET_REVISION_STATE_LABEL: "adopted"} + } + })) + .unwrap(); + assert!(!managed_feishu_secret_cleanup_candidate( + &staged, + Some("agent-feishu-current") + )); + assert!(managed_feishu_secret_cleanup_candidate( + &adopted, + Some("agent-feishu-current") + )); + assert!(!managed_feishu_secret_cleanup_candidate( + &adopted, + Some("agent-feishu-adopted") + )); +} + +#[test] +fn feishu_app_claim_cleanup_waits_for_pod_rollout() { + let old_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-old", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "41" + }}, + "spec": {"containers": [{"name": "openclaw", "image": "runtime:latest"}]} + })) + .unwrap(); + let unready_new_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-new", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "42" + }}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "runtime:latest", + "readinessProbe": {"exec": {"command": ["kars-channel-feishu-ready"]}} + }]}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": false}]} + })) + .unwrap(); + + let ready_new_pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-new", "annotations": { + FEISHU_POD_CREDENTIALS_VERSION_ANNOTATION: "42" + }}, + "spec": {"containers": [{ + "name": "openclaw", + "image": "runtime:latest", + "readinessProbe": {"exec": {"command": ["kars-channel-feishu-ready"]}} + }]}, + "status": {"containerStatuses": [{"name": "openclaw", "ready": true}]} + })) + .unwrap(); + + assert!(!feishu_claim_rollout_complete( + &[old_pod, ready_new_pod.clone()], + "openclaw", + Some("42"), + )); + assert!(!feishu_claim_rollout_complete( + &[unready_new_pod], + "openclaw", + Some("42"), + )); + assert!(feishu_claim_rollout_complete( + &[ready_new_pod], + "openclaw", + Some("42"), + )); + assert!(!feishu_claim_rollout_complete(&[], "openclaw", Some("42"),)); + assert!(feishu_claim_rollout_complete(&[], "openclaw", None)); +} + +#[test] +fn feishu_app_claim_delete_waits_for_all_pods_to_terminate() { + let pod: Pod = serde_json::from_value(json!({ + "apiVersion": "v1", + "kind": "Pod", + "metadata": {"name": "agent-terminating"}, + "spec": {"containers": [{"name": "openclaw", "image": "runtime:latest"}]} + })) + .unwrap(); + assert!(!feishu_claim_release_ready_on_delete(&[pod])); + assert!(feishu_claim_release_ready_on_delete(&[])); +} + +#[test] +fn feishu_credential_revision_requires_explicit_rotation_after_initial_deploy() { + assert_eq!( + plan_feishu_credential_revision("agent-credentials", None, None, Some("41")), + FeishuCredentialRevisionPlan::UseCurrent("41".into()) + ); + assert_eq!( + plan_feishu_credential_revision( + "agent-credentials-rotation-new", + Some("agent-credentials"), + Some("41"), + Some("42"), + ), + FeishuCredentialRevisionPlan::UseCurrent("42".into()) + ); + assert_eq!( + plan_feishu_credential_revision( + "agent-credentials", + Some("agent-credentials"), + Some("41"), + Some("42"), + ), + FeishuCredentialRevisionPlan::KeepDeployed("41".into()) + ); + assert_eq!( + plan_feishu_credential_revision("agent-credentials", None, Some("41"), Some("42")), + FeishuCredentialRevisionPlan::Unavailable + ); +} + +#[test] +fn unsupported_feishu_runtime_uses_channel_condition_reason() { + let (condition_type, reason) = channel_capability_failure_condition(); + assert_eq!(condition_type, "ChannelReady"); + assert_eq!(reason, "UnsupportedByRuntime"); +} + +#[test] +fn channel_fail_closed_patch_stops_existing_runtime() { + assert_eq!( + channel_fail_closed_deployment_patch(), + json!({"spec": {"replicas": 0}}) + ); +} + #[test] fn workspace_bootstrap_config_map_accepts_declarative_files() { let config_map: ConfigMap = serde_json::from_value(json!({ diff --git a/controller/src/status/conditions.rs b/controller/src/status/conditions.rs index 693e277ef..eda7733dd 100644 --- a/controller/src/status/conditions.rs +++ b/controller/src/status/conditions.rs @@ -82,6 +82,9 @@ pub const TYPE_STORAGE_READY: &str = "StorageReady"; /// bootstrap ConfigMap is configured. pub const TYPE_BOOTSTRAP_READY: &str = "BootstrapReady"; +/// Declared runtime messaging channels are configured and connected. +pub const TYPE_CHANNEL_READY: &str = "ChannelReady"; + /// Phase 2 S12.e — `AllowlistVerified`: the controller fetched the /// signed OCI artifact referenced by /// `spec.networkPolicy.allowlistRef`, verified its cosign signature @@ -175,6 +178,15 @@ pub mod reason { pub const BOOTSTRAP_CONFIG_NOT_FOUND: &str = "BootstrapConfigNotFound"; pub const BOOTSTRAP_INVALID: &str = "BootstrapInvalid"; pub const BOOTSTRAP_FAILED: &str = "BootstrapFailed"; + pub const CHANNEL_CONFIGURED: &str = "Configured"; + pub const CHANNEL_CONNECTING: &str = "Connecting"; + pub const CHANNEL_CONNECTION_FAILED: &str = "ConnectionFailed"; + pub const CHANNEL_SUSPENDED: &str = "Suspended"; + pub const CHANNEL_APP_ALREADY_CLAIMED: &str = "AppAlreadyClaimed"; + pub const CHANNEL_CREDENTIALS_MISSING: &str = "CredentialsMissing"; + pub const CHANNEL_CREDENTIALS_PARTIAL: &str = "CredentialsPartial"; + pub const CHANNEL_UNSUPPORTED_RUNTIME: &str = "UnsupportedByRuntime"; + pub const CHANNEL_POLICY_INVALID: &str = "PolicyInvalid"; /// Phase 2 S12.b — `Verified`: signed allowlist artifact fetched, /// cosign signature passed, signer identity matched cluster /// SignerPolicy, canonical form re-validated. diff --git a/deploy/helm/kars/templates/crd.yaml b/deploy/helm/kars/templates/crd.yaml index edcd575e5..89cfbb45b 100644 --- a/deploy/helm/kars/templates/crd.yaml +++ b/deploy/helm/kars/templates/crd.yaml @@ -506,6 +506,75 @@ spec: retainPolicy: type: string enum: ["Retain", "Delete"] + channels: + type: array + description: "Runtime-facing messaging channel policy. Credentials remain in a per-sandbox Secret." + maxItems: 8 + x-kubernetes-validations: + - rule: "self.filter(channel, channel.type == 'Feishu').size() <= 1" + message: "spec.channels may contain at most one Feishu channel" + reason: "FieldValueInvalid" + - rule: "self.all(channel, (channel.type == 'Feishu') == has(channel.feishu))" + message: "channels[].feishu must be set iff type is Feishu" + reason: "FieldValueInvalid" + items: + type: object + required: ["type"] + properties: + type: + type: string + enum: ["Feishu"] + credentialSecretRef: + type: object + required: ["name"] + properties: + name: + type: string + maxLength: 253 + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + feishu: + type: object + properties: + domain: + type: string + enum: ["Feishu", "Lark"] + default: "Feishu" + connectionMode: + type: string + enum: ["WebSocket"] + default: "WebSocket" + directMessages: + type: object + properties: + policy: + type: string + enum: ["Pairing", "Allowlist", "Disabled"] + default: "Pairing" + allowFrom: + type: array + maxItems: 256 + items: + type: string + pattern: "^ou_[A-Za-z0-9_-]+$" + x-kubernetes-validations: + - rule: "!has(self.policy) || self.policy != 'Allowlist' || (has(self.allowFrom) && self.allowFrom.size() > 0)" + message: "directMessages.allowFrom must be non-empty when policy is Allowlist" + groups: + type: object + properties: + policy: + type: string + enum: ["Allowlist", "Disabled"] + default: "Allowlist" + allowFrom: + type: array + maxItems: 256 + items: + type: string + pattern: "^oc_[A-Za-z0-9_-]+$" + requireMention: + type: boolean + default: true agent: type: object description: "Foundry Agent Service configuration — controller creates a prompt agent on reconcile" diff --git a/docs/api/conditions.md b/docs/api/conditions.md index e666136bb..9a433466f 100644 --- a/docs/api/conditions.md +++ b/docs/api/conditions.md @@ -23,6 +23,7 @@ These types are reused across CRDs. | `Suspended` | Operator has paused the object via `spec.suspended: true`. | | `StorageReady` | Sandbox workspace storage is available to the runtime. | | `BootstrapReady` | OpenClaw workspace bootstrap init container completed. | +| `ChannelReady` | Every declared runtime messaging channel is connected. | `status: True` means the type predicate holds. For `Degraded`, that means the object **is** degraded; for `Ready`, that it **is** ready. @@ -48,6 +49,11 @@ means the object **is** degraded; for `Ready`, that it **is** ready. | `BootstrapConfigNotFound` | `KarsSandbox` | Referenced workspace bootstrap ConfigMap does not exist. | | `BootstrapInvalid` | `KarsSandbox` | Bootstrap ConfigMap contains unsupported files or binary data. | | `BootstrapFailed` | `KarsSandbox` | Bootstrap init container exited unsuccessfully. | +| `Configured` / `Connecting` / `ConnectionFailed` | `KarsSandbox` | Channel adapter is connected / waiting for its runtime WebSocket signal / unable to start or maintain its connection. | +| `CredentialsMissing` / `CredentialsPartial` | `KarsSandbox` | Required channel Secret keys are absent or incomplete. | +| `UnsupportedByRuntime` | `KarsSandbox` | The selected runtime has no adapter for the declared channel. | +| `AppAlreadyClaimed` | `KarsSandbox` | Another sandbox owns the non-secret App ID fingerprint. | +| `PolicyInvalid` | `KarsSandbox` | Channel IDs or access policy failed semantic validation. | ## KarsSandbox @@ -63,6 +69,7 @@ end-to-end runtime. | `RuntimeReady` | True/False | `AdapterMissing` (Falsey when the runtime adapter isn't wired) | | `StorageReady` | True/False | `EmptyDir`, `ClaimBound`, `ClaimPending` | | `BootstrapReady` | True/False | `Reconciled`, `Creating`, `BootstrapFailed` | +| `ChannelReady` | True/False | `Configured`, `Connecting`, `ConnectionFailed`, `CredentialsMissing`, `CredentialsPartial`, `UnsupportedByRuntime`, `AppAlreadyClaimed`, `PolicyInvalid`, `Suspended` | | `AllowlistVerified` | True/False | `Verified`, `Unsigned`, `FailedClosed` | | `AllowlistAuthoritative` | True/False | `Inline`, `Verified`, `StaleLKG`, `FailedClosed`, `InlineDiffersFromArtifact` | | `AllowlistDrift` | True/False | `InlineDiffersFromArtifact`, `InlineCleared` | diff --git a/docs/api/crd-reference.md b/docs/api/crd-reference.md index 840fe6a1b..c1b904f56 100644 --- a/docs/api/crd-reference.md +++ b/docs/api/crd-reference.md @@ -197,6 +197,20 @@ spec: name: shared-inference # required: sibling InferencePolicy memoryRef: # optional: sibling KarsMemory name: my-agent-memory + channels: + - type: Feishu + credentialSecretRef: + name: my-agent-credentials # optional; defaults to -credentials + feishu: + domain: Feishu # Feishu (default) | Lark + connectionMode: WebSocket # the only v1 transport + directMessages: + policy: Pairing # Pairing (default) | Allowlist | Disabled + allowFrom: [] # ou_... IDs; required for Allowlist + groups: + policy: Allowlist # Allowlist (default) | Disabled + allowFrom: [oc_teaching] # group chat IDs + requireMention: true # default true governance: # AGT governance defaults to enabled=true since v0.1.18 — AGT is # part of every kars deployment. To opt out, set enabled: false. @@ -235,6 +249,7 @@ status: | Field | Type | Purpose | |---|---|---| | `spec.memoryRef.name` | LocalObjectRef | Bind to a sibling `KarsMemory` (same namespace). | +| `spec.channels[]` | `[]ChannelSpec` | Typed messaging policy. V1 supports one `type: Feishu` entry on OpenClaw or Hermes only. Credentials remain in the referenced Secret, never in this CR. | | `spec.storage.workspace` | object | Optional PVC-backed `/sandbox`. Omit for legacy `emptyDir`; set `existingClaim` to attach an explicitly selected PVC, or set dynamic fields (`size`, `storageClassName`, `accessModes`, `retainPolicy`). | | `spec.runtime.openclaw.workspace.bootstrapConfigMapRef.name` | LocalObjectRef | Same-namespace ConfigMap containing only `AGENTS.md`, `SOUL.md`, `HEARTBEAT.md`, `TOOLS.md`, or `USER.md`. The controller mirrors it into the runtime namespace. | | `spec.runtime.openclaw.workspace.overwritePolicy` | enum | `IfMissing` (default) preserves files already on the PVC; `Always` reapplies declared bootstrap files on every Pod start. | @@ -270,6 +285,12 @@ For a retained workspace, deleting the `KarsSandbox` removes Kars-managed workloads but leaves the generated namespace and PVC. Reattach it explicitly with `existingClaim`; a new same-name CR cannot silently adopt a retained claim. +For Feishu, `ChannelReady=True/Configured` is emitted only after the runtime's +channel-specific probe reports an active WebSocket adapter. Complete credentials +or a generally-ready Pod are not enough. The controller atomically claims a +SHA-256 fingerprint of the App ID so the same external app cannot run in two +sandboxes; neither the App ID, App Secret, nor fingerprint appears in status. + ### `spec.runtime.hermes` (`HermesConfig`) {#hermesconfig} Runtime-kind config block used when `spec.runtime.kind: Hermes`. All fields are optional — defaults give a working smoke-test agent on first boot. @@ -279,7 +300,7 @@ spec: runtime: kind: Hermes hermes: - version: "0.15.2" # optional — Hermes Agent version pin (entrypoint reads HERMES_VERSION) + version: "0.16.0" # optional — Hermes Agent version pin (entrypoint reads HERMES_VERSION) agentCode: # optional — user-supplied agent code (mutually exclusive: oci | git) oci: image: myregistry.azurecr.io/my-hermes-agent:1.2.3 @@ -296,7 +317,7 @@ spec: | Field | Type | Notes | |---|---|---| -| `version` | string | Hermes Agent version pin (e.g. `"0.15.2"`). Stays opt-in — adapter image defaults to its latest-supported tag. Surfaces as `HERMES_VERSION` in the container env. | +| `version` | string | Hermes Agent version pin (e.g. `"0.16.0"`). Stays opt-in — adapter image defaults to its latest-supported tag. Surfaces as `HERMES_VERSION` in the container env. | | `agentCode.oci.image` | string | Pull agent code from an OCI image. Production path. | | `agentCode.git.url` | string | Clone agent code from a git URL. Development-iteration path. | | `agentCode.git.ref` | string | Branch / tag / commit SHA. Defaults to repo HEAD. | diff --git a/docs/channels-plugins.md b/docs/channels-plugins.md index 1dc747f77..2a0b45d42 100644 --- a/docs/channels-plugins.md +++ b/docs/channels-plugins.md @@ -1,6 +1,6 @@ # Channels & external plugins -Messaging channels (Telegram, Slack, Discord, WhatsApp) and **third-party** search/scrape API integrations (Brave, Tavily, Exa, Firecrawl, Perplexity, OpenAI) extend your kars agent with external communication and search capabilities. Configuration is via CLI flags — the sandbox entrypoint auto-configures everything from environment variables at startup. +Messaging channels (Telegram, Slack, Discord, WhatsApp, Feishu/Lark) and **third-party** search/scrape API integrations (Brave, Tavily, Exa, Firecrawl, Perplexity, OpenAI) extend your kars agent with external communication and search capabilities. Configuration is via CLI flags — the sandbox entrypoint auto-configures everything from environment variables at startup. > **Looking for the kars-owned plugins?** This page is about **external** integrations. For the kars-owned components: > - **[kars OpenClaw plugin](openclaw-plugin.md)** — the in-sandbox plugin (24 governance-aware tools, 10 skills) shipped with every OpenClaw-runtime sandbox. @@ -8,7 +8,7 @@ Messaging channels (Telegram, Slack, Discord, WhatsApp) and **third-party** sear > - **[`@kars/mesh` plugin](mesh-plugin.md)** — the companion local plugin (built from source, not yet published on npm) for pairing a **local** OpenClaw with a remote kars cluster (8 federation tools, 1 skill). > - **Cross-Framework Secure Mesh** — Hermes, OpenClaw, and LangGraph agents communicate over one AgentMesh fabric, every hop end-to-end encrypted with the Signal Protocol. See [mesh-plugin.md](mesh-plugin.md) and the [`kars mesh` reference](cli-reference.md#kars-mesh). -The channels and plugins documented below work identically with both runtimes — same CLI flag, same secret name (`-credentials`), same auto-config flow inside the entrypoint. The only difference is the config-file shape inside the agent container: OpenClaw writes to `~/.openclaw-data/config.yaml`, Hermes writes to `$HERMES_HOME/config.yaml`. The CLI hides that detail. +The channels and plugins documented below use the same secret name (`-credentials`) and auto-config flow in OpenClaw and Hermes. Feishu is supported only by these two runtimes; the CLI and controller reject every other runtime rather than silently dropping the channel. --- @@ -24,6 +24,7 @@ Channels connect your agent to messaging platforms. Pass channel flags to `kars | Slack | `--channels slack` | `--slack-token` | Bot User OAuth Token (`xoxb-...`) | | Discord | `--channels discord` | `--discord-token` | Bot token from Discord Developer Portal | | WhatsApp | `--channels whatsapp` | — | QR code pairing at runtime (no token needed) | +| Feishu/Lark | `--channels feishu` | `--feishu-app-id`, `--feishu-app-secret` | Outbound WebSocket; OpenClaw and Hermes only | Multiple channels can be enabled at once: @@ -74,6 +75,31 @@ kars dev --channels whatsapp Scan the QR code with WhatsApp on your phone to link the session. +### Feishu / Lark Setup + +1. Create one custom app in the [Feishu Open Platform](https://open.feishu.cn/) or [Lark Developer Console](https://open.larksuite.com/). +2. Enable the bot capability and the message receive/send permissions required by your app. +3. In Event Subscriptions, select **WebSocket / long connection**. Kars does not expose a webhook endpoint in v1. +4. Publish or install the app to the tenant, then add the bot to each allowed group. +5. Deploy with a dedicated App ID and App Secret: + +```bash +kars add teaching-agent \ + --runtime openclaw \ + --channels feishu \ + --feishu-app-id "$FEISHU_APP_ID" \ + --feishu-app-secret "$FEISHU_APP_SECRET" \ + --feishu-group-allow-from oc_teaching_a,oc_teaching_b \ + --workspace-storage 10Gi \ + --learn-egress +``` + +Use `--runtime hermes` for the Hermes adapter. `--feishu-domain lark` selects Lark; the default is `feishu`. + +Security defaults are DM pairing, group allowlist, and direct bot mention required. User IDs use Feishu open IDs (`ou_...`); group entries use chat IDs (`oc_...`). An empty group allowlist admits no groups. The same App ID cannot be active in two sandboxes: the controller stores a non-secret SHA-256 ownership claim and reports `ChannelReady=False/AppAlreadyClaimed` for a conflict. + +Pairing approvals and runtime dedup/session state live under `/sandbox`. Add `--workspace-storage` when those approvals must survive Pod replacement. Incoming Feishu messages do not wake a suspended sandbox. + --- ## Third-Party Plugins @@ -164,7 +190,7 @@ When deploying to AKS with `kars add`, channel tokens and plugin API keys are st CLI (kars add --telegram-token "...") │ ▼ -K8s Secret (kars-/-credentials) +K8s Secret (conventional: -credentials; Feishu: immutable versioned Secret) │ ▼ Controller mounts via envFrom in pod spec @@ -176,7 +202,7 @@ entrypoint.sh reads env vars → configures channels/plugins Agent process (pre-configured, never sees raw tokens) ``` -Secret naming convention: All credentials are stored in a **single secret** named `-credentials` in the `kars-` namespace. The secret contains keys mapped to environment variables: +Secret naming convention: ordinary channel/plugin credentials are stored in `-credentials` in the `kars-` namespace. Feishu App credentials are stored separately in a dedicated immutable, versioned Secret referenced by `spec.channels[].credentialSecretRef`. Secret keys map to these environment variables: | Credential Type | Secret Key | Environment Variable | |----------------|------------|---------------------| @@ -184,6 +210,8 @@ Secret naming convention: All credentials are stored in a **single secret** name | Telegram allowlist | `TELEGRAM_ALLOW_FROM` | `TELEGRAM_ALLOW_FROM` | | Slack token | `SLACK_BOT_TOKEN` | `SLACK_BOT_TOKEN` | | Discord token | `DISCORD_BOT_TOKEN` | `DISCORD_BOT_TOKEN` | +| Feishu App ID | `FEISHU_APP_ID` | `FEISHU_APP_ID` | +| Feishu App Secret | `FEISHU_APP_SECRET` | `FEISHU_APP_SECRET` | | WhatsApp | `WHATSAPP_ENABLED` | `WHATSAPP_ENABLED` | | Brave API key | `BRAVE_API_KEY` | `BRAVE_API_KEY` | | Tavily API key | `TAVILY_API_KEY` | `TAVILY_API_KEY` | @@ -211,9 +239,14 @@ kars credentials update my-agent --telegram-token "NEW" --brave-api-key "NEW" # Update without restarting the pod (apply on next restart) kars credentials update my-agent --telegram-token "NEW" --no-restart + +# Rotate Feishu credentials and restart the target sandbox +kars credentials update teaching-agent \ + --feishu-app-id "$NEW_APP_ID" \ + --feishu-app-secret "$NEW_APP_SECRET" ``` -The command updates the K8s secret and triggers a rolling restart of the sandbox pod (unless `--no-restart` is passed). +Feishu credentials are separated from Telegram/plugin credentials from initial creation onward and stored in a dedicated immutable Secret referenced by `credentialSecretRef`. For ordinary credentials, the command updates the conventional Secret and triggers a rolling restart unless `--no-restart` is passed. Feishu rotation is stricter: the CLI requires App ID and App Secret together, rejects mixing Feishu and ordinary updates in one command, creates an immutable versioned Secret containing only those two keys, and updates `credentialSecretRef`. The controller claims the new App ID before rolling the Pod and deletes obsolete managed revisions afterward. `--no-restart` is rejected for Feishu. --- @@ -243,6 +276,11 @@ No manual configuration files needed — everything is driven by environment var | Slack `invalid_auth` | Token revoked or wrong workspace | Reinstall the Slack app and use the new `xoxb-` token | | Discord bot offline | Missing `MESSAGE_CONTENT` intent | Enable it in Discord Developer Portal → Bot → Privileged Gateway Intents | | WhatsApp QR not appearing | Console output buffered | Check gateway logs: `kubectl logs -c openclaw` | +| Feishu `ChannelReady=False/Connecting` | Runtime is starting or the WebSocket is reconnecting | Wait for the retry loop; if it persists, inspect runtime logs and learned egress | +| Feishu `ChannelReady=False/ConnectionFailed` | Adapter startup, authentication, dependency, or egress failed | Check runtime logs, then review `kars egress --learned` and approve only the exact Feishu/Lark hosts | +| Feishu `AppAlreadyClaimed` | The App ID is already active in another sandbox | Give each sandbox a dedicated app, or remove or rotate the existing channel first | +| Feishu DMs return a pairing code | Safe DM default is active | Approve with the runtime pairing command; use persistent workspace storage to retain approval | +| Feishu group messages are ignored | Chat ID is absent from the group allowlist or the bot was not directly mentioned | Add the `oc_...` ID and mention the bot; `@all` does not count | | Channel traffic blocked | Domain not on egress allowlist | Run `kars egress --learned` and approve channel API domains | ### Plugin Issues diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 7aa320598..dcd4595f8 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -423,11 +423,19 @@ kars add [options] | `--no-governance` | — | Disable AGT governance | | `--trust-threshold ` | `500` | AGT trust threshold (0–1000) | | `--policy-profile ` | `default` | AGT policy profile name | -| `--channels ` | — | Channels: `telegram,slack,discord,whatsapp` | +| `--channels ` | — | Channels: `telegram,slack,discord,whatsapp,feishu` | | `--telegram-token ` | — | Telegram bot token | | `--telegram-allow-from ` | — | Allowed Telegram user IDs (comma-separated) | | `--slack-token ` | — | Slack bot OAuth token | | `--discord-token ` | — | Discord bot token | +| `--feishu-app-id ` | — | Feishu/Lark App ID; required with `--channels feishu` | +| `--feishu-app-secret ` | — | Feishu/Lark App Secret; required with `--channels feishu` | +| `--feishu-domain ` | `feishu` | API domain: `feishu` or `lark` | +| `--feishu-dm-policy ` | `pairing` | Direct-message policy: `pairing`, `allowlist`, or `disabled` | +| `--feishu-allow-from ` | — | Allowed DM user open IDs (`ou_...`, comma-separated); required for `allowlist` | +| `--feishu-group-policy ` | `allowlist` | Group policy: `allowlist` or `disabled` | +| `--feishu-group-allow-from ` | — | Allowed group chat IDs (`oc_...`, comma-separated) | +| `--feishu-require-mention ` | `true` | Require a direct bot mention in admitted groups | | `--skills ` | — | Skills: `browser,github,summarize,weather` | | `--brave-api-key ` | — | Brave Search API key | | `--tavily-api-key ` | — | Tavily search API key | @@ -455,6 +463,15 @@ kars add hermes-support --runtime hermes \ --channels telegram --telegram-token "$TOKEN" \ --workspace-storage 20Gi +# Add a WebSocket-only Feishu agent with safe DM/group defaults +kars add teaching-agent --runtime openclaw \ + --channels feishu \ + --feishu-app-id "$FEISHU_APP_ID" \ + --feishu-app-secret "$FEISHU_APP_SECRET" \ + --feishu-group-allow-from oc_teaching \ + --workspace-storage 10Gi \ + --learn-egress + # Add an OpenClaw agent with a retained 20Gi workspace and bootstrap files kars add teaching-agent --workspace-storage 20Gi \ --workspace-storage-class managed-csi \ @@ -475,8 +492,9 @@ kars add reviewer --dry-run Workspace PVC flags apply to every wired Kubernetes runtime. The controller mounts the claim at `/sandbox`; each runtime/application must store recoverable state there. `--workspace-bootstrap` remains OpenClaw-only. Channel flags are -supported by the OpenClaw and Hermes adapters; other runtimes must configure -their own channel integration. +supported by the OpenClaw and Hermes adapters. Feishu is rejected for every +other runtime in v1. Feishu uses outbound WebSocket only and does not create a +public ingress. The initial OpenClaw sandbox created by `kars up` supports the same generated or existing workspace claim options: @@ -1128,13 +1146,15 @@ kars credentials [subcommand] [arguments] [options] | `--telegram-allow-from ` | — | Allowed Telegram user IDs (comma-separated) | | `--slack-token ` | — | New Slack bot token | | `--discord-token ` | — | New Discord bot token | +| `--feishu-app-id ` | — | New Feishu/Lark App ID | +| `--feishu-app-secret ` | — | New Feishu/Lark App Secret | | `--brave-api-key ` | — | New Brave Search API key | | `--tavily-api-key ` | — | New Tavily API key | | `--exa-api-key ` | — | New Exa API key | | `--firecrawl-api-key ` | — | New Firecrawl API key | | `--perplexity-api-key ` | — | New Perplexity API key | | `--openai-api-key ` | — | New OpenAI API key | -| `--no-restart` | — | Update secret without restarting the pod | +| `--no-restart` | — | Update an ordinary credential without restarting the pod. Rejected for Feishu rotation. | **Examples:** ```bash @@ -1150,6 +1170,15 @@ kars credentials list # Update a running sandbox's Telegram token and restart kars credentials update my-agent --telegram-token 999999:NEW-TOKEN +# Rotate Feishu credentials and restart the target sandbox +kars credentials update teaching-agent \ + --feishu-app-id "$NEW_APP_ID" \ + --feishu-app-secret "$NEW_APP_SECRET" + +# Feishu uses a versioned Secret and controller-owned rollout; +# --no-restart is intentionally unsupported for this operation. +# Rotate Telegram or plugin credentials in a separate command. + # Update without restarting the pod kars credentials update my-agent --brave-api-key $KEY --no-restart ``` diff --git a/docs/hermes-plugin.md b/docs/hermes-plugin.md index 0b142e2eb..f468209c2 100644 --- a/docs/hermes-plugin.md +++ b/docs/hermes-plugin.md @@ -95,7 +95,7 @@ Each is replaced by its kars equivalent (`foundry_*` via MCP or `http_fetch`) th --- -## Channels (4 first-class adapters today) +## Channels (5 first-class adapters today) Hermes ships 20+ channel adapters; kars wires the four production-grade ones via CLI flag → env var → `entrypoint.sh` → `hermes config set channels.*` flow: @@ -105,8 +105,9 @@ Hermes ships 20+ channel adapters; kars wires the four production-grade ones via | **Slack** | `SLACK_BOT_TOKEN` | `channels.slack.{token,enabled}` | | **Discord** | `DISCORD_BOT_TOKEN` | `channels.discord.{token,enabled}` | | **WhatsApp** | `WHATSAPP_TOKEN` | `channels.whatsapp.{token,enabled}` | +| **Feishu/Lark** | `FEISHU_APP_ID`, `FEISHU_APP_SECRET` | `platforms.feishu.extra` policy + native WebSocket adapter | -Credentials live in a Kubernetes secret named `-credentials` in namespace `kars-`, mounted via `envFrom: { secretRef: { optional: true } }` so a Hermes pod starts even before the secret is created. Add or rotate tokens with: +Telegram, Slack, Discord, and plugin credentials use `-credentials` in namespace `kars-`. Feishu App credentials use a dedicated immutable, versioned Secret selected by `spec.channels[].credentialSecretRef` and are injected through explicit key refs. Add or rotate credentials with: ```bash kars credentials update my-hermes-agent --telegram-token @@ -115,6 +116,10 @@ kubectl rollout restart deployment/my-hermes-agent -n kars-my-hermes-agent When no channels are configured the entrypoint logs `No channels — starting hermes gateway in idle daemon mode` and serves only mesh / spawn / hook traffic — perfect for sub-agents that talk only to other agents. +Feishu is WebSocket-only in the Kars v1 contract. The image pins `hermes-agent[feishu]==0.16.0`, translates DM pairing/allowlist/disabled policy, generates per-chat `group_rules` for allowed `oc_...` IDs, disables all unlisted groups, and enforces `require_mention` before model dispatch. A pod-local readiness marker is created only after the pinned `lark-oapi==1.5.3` SDK completes `websockets.connect()` and is removed by the SDK disconnect path; the controller maps that probe to `ChannelReady`. + +Use one Feishu App per sandbox. The controller atomically claims a SHA-256 fingerprint of the App ID and rejects a second owner without exposing the App ID or App Secret in status. Pairing approvals are runtime state under `/sandbox`; use a workspace PVC when approvals must survive Pod replacement. + --- ## Plugins (5 tool providers wired via env vars) diff --git a/docs/operations/image-versioning.md b/docs/operations/image-versioning.md index ed9b7706c..01e025a37 100644 --- a/docs/operations/image-versioning.md +++ b/docs/operations/image-versioning.md @@ -20,6 +20,24 @@ override env var on the controller (e.g. `OPENAI_AGENTS_RUNTIME_IMAGE`, `PYDANTIC_AI_RUNTIME_IMAGE`, `INFERENCE_ROUTER_IMAGE`, `SANDBOX_IMAGE`). +## Runtime dependency coupling + +The OpenClaw sandbox base pins `openclaw@2026.5.27` and +`@openclaw/feishu@2026.5.27` to the same exact version. Its build verifies that +the `feishu` plugin is discoverable in an immutable external-plugin stage; the +entrypoint copies that registered stage into the writable OpenClaw state before +generating channel config. Upgrade the host and plugin together. + +The Hermes runtime pins `hermes-agent[feishu]==0.16.0`, including +`lark-oapi==1.5.3`. Kars applies a source-anchored compatibility patch for typed +DM/group admission and SDK-handshake channel readiness; the image build fails if +either dependency changes those source anchors. Upgrade Hermes only after updating the patch and rerunning +`sandbox-images/hermes/testM_feishu_channel.sh` against the new wheel. + +These package pins are independent of the container tag channel. A floating +`:latest` image still contains exact runtime package versions from its source +revision. + ## Recommended channels per environment | Environment | Controller / router | Sandbox / runtimes | Why | diff --git a/docs/plans/2026-08-09-feishu-channel-contract-design.md b/docs/plans/2026-08-09-feishu-channel-contract-design.md new file mode 100644 index 000000000..eac093c05 --- /dev/null +++ b/docs/plans/2026-08-09-feishu-channel-contract-design.md @@ -0,0 +1,706 @@ +# Feishu Channel Contract Design + +**Status:** Approved design + +**Date:** 2026-08-09 + +**Scope:** A typed, runtime-aware channel contract with first-class Feishu support for OpenClaw and Hermes. Feishu uses outbound WebSocket long connections only. Other runtimes must report unsupported capability rather than silently ignore channel configuration. + +## 1. Problem + +Kars currently exposes four messaging channels through CLI flags and per-sandbox credentials: Telegram, Slack, Discord, and WhatsApp. The platform path is: + +```text +CLI flag -> -credentials Secret -> envFrom -> runtime entrypoint -> native channel config +``` + +This works, but the non-sensitive channel policy is implicit inside entrypoint shell code. The `KarsSandbox` does not describe which channel is expected, which runtime supports it, which groups are allowed, or whether the channel is ready. Unsupported runtimes can only reject broad OpenClaw-only CLI flags; there is no reusable channel capability contract. + +Feishu exposes this gap: + +- OpenClaw 2026.5.27 supports Feishu through the separately published `@openclaw/feishu@2026.5.27` plugin. The package is not currently installed in the Kars image. +- Hermes Agent 0.16.0 includes a native Feishu adapter under `gateway/platforms/feishu.py`, but its optional `feishu` dependencies are not currently installed in the Kars Hermes image and Kars does not translate Feishu credentials/configuration. +- The remaining shipping runtimes do not run an IM channel gateway and must not pretend to support Feishu. +- Multiple sandboxes using the same Feishu App credentials would create competing WebSocket consumers with undefined ownership. + +## 2. Decisions + +The first version uses the following locked decisions: + +1. **Typed channel declaration:** non-sensitive policy lives in `KarsSandbox.spec.channels[]`. +2. **Secret separation:** Feishu App ID and App Secret live in a dedicated immutable, versioned Secret, never in the CR or ConfigMap. +3. **WebSocket only:** both OpenClaw and Hermes establish outbound Feishu long connections. No webhook, public ingress, verification token, or encrypt key is supported in v1. +4. **Safe access defaults:** direct messages use `Pairing`; groups use `Allowlist`; group messages require a direct bot mention by default. +5. **One App per sandbox:** one Feishu App credential pair binds to one `KarsSandbox`. One bot may serve multiple users and multiple allowlisted groups. +6. **Two first-class adapters:** OpenClaw and Hermes support Feishu. Other runtimes fail closed with `ChannelReady=False/UnsupportedByRuntime`. +7. **No wake-from-zero:** the channel runs inside the runtime Pod. A suspended sandbox has no WebSocket consumer and cannot wake from an incoming Feishu message. +8. **No implicit egress widening:** Feishu hosts go through the existing Learn -> approve -> Strict workflow. + +## 3. Goals + +1. Provide one operator-facing Feishu configuration for OpenClaw and Hermes. +2. Keep credentials isolated per sandbox and out of declarative policy objects. +3. Support direct messages and multiple group chats through one Feishu App. +4. Make unsupported runtime/channel combinations visible at admission or status. +5. Preserve the existing credential rotation workflow. +6. Report whether the declared channel was translated and whether the runtime established it successfully. +7. Keep the design reusable for later DingTalk, WeCom, and other channel adapters. +8. Preserve backward compatibility for existing environment-only Telegram, Slack, Discord, and WhatsApp configurations. + +## 4. Non-goals + +The first version does not: + +- support Feishu webhook transport; +- expose a public channel ingress; +- wake a suspended sandbox from a message; +- allow multiple sandboxes to share one Feishu App; +- add a central Channel Gateway or message queue; +- implement Feishu for LangGraph, OpenAI Agents, Microsoft Agent Framework, Anthropic, PydanticAI, or BYO; +- add per-group overrides beyond one global allowlist and `requireMention` setting; +- support dynamic per-user agent creation; +- configure Feishu Docs, Drive, Wiki, Bitable, Calendar, or other workplace tools beyond the messaging channel; +- automatically grant or inspect Feishu tenant permissions; +- automatically approve egress domains; +- guarantee message delivery while the Pod is unavailable. + +## 5. Architecture + +```text +KarsSandbox.spec.channels[] + type: Feishu + policy only + | + +------------------------------+ + | | + v v +immutable Feishu Secret Controller capability check +FEISHU_APP_ID OpenClaw/Hermes -> supported +FEISHU_APP_SECRET others -> UnsupportedByRuntime + | | + +---------------+--------------+ + v + runtime container env + | + +------------+-------------+ + | | + v v + OpenClaw adapter Hermes adapter + @openclaw/feishu package native gateway/platforms/feishu.py + openclaw.json translation Hermes config/env translation + | | + +------------+-------------+ + v + outbound WebSocket via + Kars transparent egress proxy + | + v + Feishu Open Platform +``` + +The Controller remains responsible for runtime capability and status. It does not parse Feishu messages, hold Feishu credentials, or proxy plaintext channel traffic. + +## 6. Proposed API + +### 6.1 KarsSandbox channel declaration + +Add an optional channel list: + +```yaml +apiVersion: kars.azure.com/v1alpha1 +kind: KarsSandbox +metadata: + name: teaching-agent + namespace: kars-system +spec: + runtime: + kind: OpenClaw + openclaw: {} + + channels: + - type: Feishu + credentialSecretRef: + name: teaching-agent-credentials + feishu: + domain: Feishu + connectionMode: WebSocket + directMessages: + policy: Pairing + allowFrom: [] + groups: + policy: Allowlist + allowFrom: + - oc_teaching_group + - oc_admin_group + requireMention: true +``` + +`credentialSecretRef` is optional for hand-authored backward-compatible CRs. The CLI always creates a dedicated immutable Feishu Secret and writes its name here so ordinary channel/plugin rotation cannot mutate Feishu credentials in place. + +### 6.2 Rust shape + +```rust +pub struct KarsSandboxSpec { + // existing fields... + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub channels: Vec, +} + +pub struct ChannelSpec { + pub type_: ChannelType, + pub credential_secret_ref: Option, + pub feishu: Option, +} + +pub enum ChannelType { + Feishu, +} + +pub struct FeishuChannelSpec { + pub domain: FeishuDomain, + pub connection_mode: FeishuConnectionMode, + pub direct_messages: DirectMessagePolicy, + pub groups: GroupPolicy, +} + +pub enum FeishuDomain { + Feishu, + Lark, +} + +pub enum FeishuConnectionMode { + WebSocket, +} + +pub struct DirectMessagePolicy { + pub policy: DirectMessageAccess, + pub allow_from: Vec, +} + +pub enum DirectMessageAccess { + Pairing, + Allowlist, + Disabled, +} + +pub struct GroupPolicy { + pub policy: GroupAccess, + pub allow_from: Vec, + pub require_mention: bool, +} + +pub enum GroupAccess { + Allowlist, + Disabled, +} +``` + +Defaults: + +| Field | Default | +|---|---| +| `domain` | `Feishu` | +| `connectionMode` | `WebSocket` | +| `directMessages.policy` | `Pairing` | +| `directMessages.allowFrom` | `[]` | +| `groups.policy` | `Allowlist` | +| `groups.allowFrom` | `[]` | +| `groups.requireMention` | `true` | + +An empty group allowlist means no group is admitted. It does not mean all groups. + +### 6.3 Validation + +Helm CEL and Controller defense-in-depth must enforce: + +1. `type: Feishu` iff the `feishu` block is present. +2. `channels[]` contains at most one `Feishu` entry. +3. `connectionMode` only accepts `WebSocket` in v1. +4. `domain` accepts `Feishu` or `Lark`. +5. `directMessages.allowFrom` contains Feishu user Open IDs (`ou_...`) only. +6. `groups.allowFrom` contains Feishu chat IDs (`oc_...`) only. +7. `directMessages.policy=Allowlist` requires at least one user ID. +8. `groups.policy=Allowlist` with an empty list is valid but admits no groups. +9. `OpenClaw` and `Hermes` accept Feishu. +10. All other runtime kinds reject or degrade before Pod readiness. +11. The referenced Secret name is same-namespace in the generated runtime namespace; cross-namespace Secret references are impossible. +12. Feishu credentials cannot appear inline in the CR. + +## 7. Secret contract + +The credential Secret contains only sensitive Feishu application credentials: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: teaching-agent-credentials + namespace: kars-teaching-agent +type: Opaque +stringData: + FEISHU_APP_ID: cli_xxx + FEISHU_APP_SECRET: redacted +``` + +Required keys: + +| Key | Required | Purpose | +|---|---|---| +| `FEISHU_APP_ID` | yes | Feishu/Lark self-built application ID | +| `FEISHU_APP_SECRET` | yes | Application secret | + +Non-sensitive policy is compiled from the CR into Controller-owned runtime environment/config values: + +| Value | Source | +|---|---| +| `FEISHU_DOMAIN` | `channels[].feishu.domain` | +| `FEISHU_CONNECTION_MODE` | locked to `websocket` | +| `FEISHU_DM_POLICY` | `directMessages.policy` | +| `FEISHU_ALLOW_FROM` | `directMessages.allowFrom` | +| `FEISHU_GROUP_POLICY` | `groups.policy` | +| `FEISHU_GROUP_ALLOW_FROM` | `groups.allowFrom` | +| `FEISHU_REQUIRE_MENTION` | `groups.requireMention` | + +Credentials are injected only into the runtime container. The inference router and init containers must not receive Feishu App credentials. + +## 8. CLI contract + +### 8.1 Create a sandbox + +```bash +kars add teaching-agent \ + --runtime openclaw \ + --channels feishu \ + --feishu-app-id "$FEISHU_APP_ID" \ + --feishu-app-secret "$FEISHU_APP_SECRET" \ + --feishu-group-allow-from "oc_teaching_group,oc_admin_group" \ + --learn-egress +``` + +Hermes uses the same flags: + +```bash +kars add teaching-hermes \ + --runtime hermes \ + --channels feishu \ + --feishu-app-id "$FEISHU_APP_ID" \ + --feishu-app-secret "$FEISHU_APP_SECRET" \ + --feishu-group-allow-from "oc_teaching_group" \ + --learn-egress +``` + +New flags: + +```text +--feishu-app-id +--feishu-app-secret +--feishu-domain default feishu +--feishu-dm-policy +--feishu-allow-from +--feishu-group-policy +--feishu-group-allow-from +--feishu-require-mention / --no-feishu-require-mention +``` + +CLI rules: + +- `--channels feishu` requires App ID and App Secret from a flag, local credential store, or environment. +- Feishu flags are valid only with `--runtime openclaw` or `--runtime hermes`. +- Unsupported runtime combinations exit non-zero before writing any resource. +- The CLI writes credentials to the Secret and policy to the KarsSandbox CR. +- The CLI must not print App Secret in dry-run, logs, summary, or errors. + +### 8.2 Rotate credentials + +```bash +kars credentials update teaching-agent \ + --feishu-app-id "$NEW_APP_ID" \ + --feishu-app-secret "$NEW_APP_SECRET" +``` + +The credentials command rotates App ID and App Secret together into a new immutable Secret marked `staged`, then changes only the Feishu `credentialSecretRef` with a JSON Patch guarded by the CR's `resourceVersion`. This prevents rotation from overwriting a concurrent channel-policy edit. Once the referenced Secret is observed, the Controller marks it `adopted`, acquires App ownership before changing the Pod template, and deletes only obsolete adopted revisions after the new runtime connection is ready. A steady-state reconcile never garbage-collects an unadopted staged revision during the create-then-patch window. Feishu rotation rejects `--no-restart` and cannot be mixed with ordinary credential updates. + +## 9. OpenClaw adapter + +### 9.1 Image packaging + +The Kars OpenClaw image must install the exact plugin version matching the pinned OpenClaw host: + +```text +openclaw@2026.5.27 +@openclaw/feishu@2026.5.27 +``` + +The plugin declares: + +```text +@larksuiteoapi/node-sdk = 1.65.0 +typebox = 1.1.38 +zod = 4.4.3 +peer openclaw >= 2026.5.27 +``` + +The build must fail if the Feishu plugin cannot be installed or discovered. Runtime installation from npm is forbidden. + +The image build verifies: + +```bash +openclaw plugins list +``` + +and asserts plugin ID `feishu` exists. + +The image installs Feishu into an immutable external-plugin stage, copies that stage into runtime state at startup, and preserves the existing minimal bundled-plugin optimization. Build-time discovery must fail closed. + +### 9.2 Runtime translation + +The entrypoint converts the platform contract into `openclaw.json`: + +```json +{ + "channels": { + "feishu": { + "appId": "${FEISHU_APP_ID}", + "appSecret": "${FEISHU_APP_SECRET}", + "domain": "feishu", + "connectionMode": "websocket", + "dmPolicy": "pairing", + "allowFrom": [], + "groupPolicy": "allowlist", + "groupAllowFrom": ["oc_teaching_group"], + "requireMention": true + } + }, + "plugins": { + "allow": ["kars", "feishu"], + "entries": { + "kars": { "enabled": true }, + "feishu": { "enabled": true } + } + } +} +``` + +The entrypoint must fail loud if exactly one of App ID/App Secret is present or if the plugin is missing. It must not silently drop the channel. + +## 10. Hermes adapter + +### 10.1 Image packaging + +Hermes Agent 0.16.0 includes the Feishu platform adapter but declares its libraries as optional dependencies: + +```text +lark-oapi == 1.5.3 +qrcode == 7.4.2 +``` + +The Kars Hermes image must install the pinned Feishu extra or equivalent exact dependencies: + +```text +hermes-agent[feishu] == 0.16.0 +``` + +This replaces the assumption that installing `hermes-agent==0.16.0` alone makes Feishu operational. + +### 10.2 Runtime translation + +Hermes 0.16.0 reads these environment variables natively: + +```text +FEISHU_APP_ID +FEISHU_APP_SECRET +FEISHU_DOMAIN +FEISHU_CONNECTION_MODE +``` + +The Kars entrypoint also writes the non-sensitive policy through `hermes config set` or the Hermes YAML configuration using the exact keys supported by the pinned package. The implementation must inspect and test the pinned adapter before choosing key names for: + +```text +direct-message policy +allowed users +group policy +allowed groups +require mention +``` + +The spec does not guess those keys. If Hermes lacks a native setting for one platform policy, the Kars Hermes adapter must enforce it before dispatching the message to the model or mark the feature unsupported. It may not silently weaken Pairing, group Allowlist, or mention gating. + +Hermes startup fails loud if credentials are partial or its Feishu optional dependencies are unavailable. + +## 11. Runtime capability contract + +Add a platform-owned capability table: + +| Runtime | Feishu | Reason | +|---|---|---| +| OpenClaw | Supported | Pinned `@openclaw/feishu` plugin | +| Hermes | Supported | Native Feishu adapter + pinned optional dependencies | +| OpenAIAgents | Unsupported | No channel gateway daemon | +| MicrosoftAgentFramework | Unsupported | No Kars channel adapter | +| LangGraph Python/TypeScript | Unsupported | No Kars channel adapter | +| Anthropic | Unsupported | No Kars channel adapter | +| PydanticAi | Unsupported | No Kars channel adapter | +| BYO | Unsupported in v1 | No image capability declaration mechanism yet | +| SemanticKernel | Runtime adapter itself deferred | N/A | + +Capability validation exists in two layers: + +1. CLI rejects unsupported combinations before applying resources. +2. Controller validates the CR defensively and sets `ChannelReady=False/UnsupportedByRuntime` without creating a ready runtime Pod. + +A future BYO contract version may declare channel capabilities, but v1 does not trust arbitrary images to claim Feishu support. + +## 12. Group chat model + +One Feishu App may serve multiple groups through one sandbox: + +```text +Feishu App teaching-bot + -> KarsSandbox teaching-agent + -> group oc_teaching_a + -> group oc_teaching_b + -> group oc_admin +``` + +The v1 behavior is: + +1. The group `chat_id` must appear in `groups.allowFrom`. +2. The message must directly mention the bot when `requireMention=true`. +3. `@all` does not count as a direct bot mention. +4. Users in an admitted group are accepted according to the runtime adapter's group policy. Per-user group allowlists are out of scope. +5. Messages from unknown groups are dropped before an LLM call and recorded as a policy denial without logging message content. +6. One bot may participate in many groups, but the same App credentials must not be active in multiple sandboxes. + +## 13. One-App-one-Sandbox invariant + +Kars cannot prove globally that an external secret value is unique without reading and indexing secrets across namespaces. V1 uses layered enforcement: + +1. CLI local configuration warns or rejects when the same App ID is already associated with another known sandbox in the current cluster. +2. The Controller records a SHA-256 fingerprint of App ID only, never App Secret, in an internal ownership annotation or index ConfigMap. +3. Conflicting claims set `ChannelReady=False/AppAlreadyClaimed` and prevent the second runtime from starting its Feishu channel. +4. Secret values and fingerprints never appear in CR status, events, metrics, or user-visible logs. +5. Deleting a sandbox releases its ownership record through finalizer cleanup. + +If cluster-wide secret indexing is considered too invasive during implementation review, the minimum acceptable v1 behavior is an explicit warning plus documentation. Silent multi-consumer use is not acceptable. + +## 14. Status + +Add a `ChannelReady` condition for every declared channel. + +| Status | Reason | Meaning | +|---|---|---| +| `True` | `Configured` | Policy translated, credentials complete, runtime adapter reported connected | +| `False` | `CredentialsMissing` | App ID or App Secret is absent | +| `False` | `CredentialsPartial` | Exactly one required credential is present | +| `False` | `UnsupportedByRuntime` | Runtime has no Feishu adapter | +| `False` | `Connecting` | Configuration is valid; WebSocket has not connected yet | +| `False` | `ConnectionFailed` | Runtime failed to start or maintain the adapter; runtime logs distinguish plugin, dependency, authentication, and egress causes without exposing credentials | +| `False` | `AppAlreadyClaimed` | Another sandbox owns the App ID fingerprint | +| `False` | `PolicyInvalid` | IDs or access policy failed validation | +| `False` | `Suspended` | Runtime is intentionally scaled to zero; no channel consumer exists | + +`Ready=True` requires every declared channel to be `ChannelReady=True`, except when the sandbox is explicitly suspended. A channel failure must not be hidden behind a healthy inference router. + +The runtime reports channel state to a localhost Router/Controller-visible endpoint or writes a non-sensitive readiness artifact. The Controller must not infer `Configured` solely from Pod readiness or Secret existence. + +## 15. Egress + +Feishu WebSocket and REST calls run under UID 1000 and use the router's explicit HTTP CONNECT proxy. The pinned plugin receives a source-anchored Axios/WebSocket agent patch because the upstream REST bootstrap otherwise emits absolute-form HTTPS requests that the forward proxy cannot tunnel safely. + +V1 does not auto-add Feishu domains. Operators use: + +```bash +kars add teaching-agent ... --learn-egress +kars egress teaching-agent --learned +kars egress teaching-agent --pending +kars egress teaching-agent --approve +kars egress teaching-agent --enforce +``` + +Requirements: + +- blocklist enforcement remains active in Learn mode; +- the channel reconnects after the proxy's tunnel lifetime/idle limits; +- Strict mode must deny unapproved Feishu/Lark hosts; +- approval is based on observed exact hosts or reviewed parent domains; +- `domain=Feishu` and `domain=Lark` are tested separately; +- logs and learned-host records never contain App credentials or message content. + +## 16. Persistence and lifecycle + +Channel credentials are already persistent Kubernetes Secret data. Channel session state and pairing state are runtime filesystem state: + +| State | Location | Persistence requirement | +|---|---|---| +| App ID / App Secret | Kubernetes Secret | Survives Pod recreation | +| Typed access policy | KarsSandbox CR | Survives Pod recreation | +| Generated runtime config | Runtime workspace | Regenerated from CR + Secret on boot | +| Pairing approvals | Runtime-specific `/sandbox` path | Requires `spec.storage.workspace` PVC to survive Pod recreation | +| Conversation history | Runtime-specific `/sandbox` path | Requires PVC or external store | +| WebSocket connection | Process memory | Re-established after restart | +| Dedup cursor/state | Runtime-specific state | Requires PVC if the runtime stores it under `/sandbox` | + +For production Feishu agents, CLI should recommend `--workspace-storage`. It must not falsely imply that configuring a channel automatically persists pairing or conversation state. + +When `spec.suspended=true`: + +- `ChannelReady=False/Suspended`; +- the WebSocket disconnects; +- Feishu messages do not wake the sandbox; +- delivery behavior while offline follows Feishu platform semantics and is not guaranteed by Kars. + +## 17. Security + +1. App Secret is never serialized into CRs, ConfigMaps, status, events, metrics, command summaries, dry-run output, or audit messages. +2. Only the runtime container receives Feishu credentials. +3. The inference router, egress-guard, workspace bootstrap, and other init containers do not receive the Secret. +4. Partial credentials fail closed. +5. DM Pairing and group Allowlist are the defaults. +6. Group mention gating occurs before an LLM call. +7. Runtime adapters may not weaken typed platform policy. +8. Secret rotation restarts only the target sandbox. +9. Runtime logs redact App ID, App Secret, authorization headers, event bodies, and message attachments. +10. Channel health exposes only state and error categories. +11. Webhook verification/encryption fields are rejected in v1 rather than ignored. +12. The channel process remains subject to seccomp, read-only rootfs, UID 1000, egress allowlist, and token budgets for model calls. + +## 18. Error handling + +| Failure | Behavior | +|---|---| +| App credentials missing | Do not start channel; `CredentialsMissing` | +| One credential missing | Do not start channel; `CredentialsPartial` | +| OpenClaw plugin absent | Fail runtime channel bootstrap; `PluginMissing` | +| Hermes Feishu extra absent | Fail runtime channel bootstrap; `AdapterDependencyMissing` | +| Invalid user/group ID | Admission or Controller `PolicyInvalid` | +| Unsupported runtime | No ready Pod; `UnsupportedByRuntime` | +| WebSocket authentication failure | Retry with bounded backoff; `ConnectionFailed` | +| WebSocket transient disconnect | Reconnect with jitter; `Connecting` during recovery | +| Egress denied | Remain disconnected; expose host-only egress denial and `ConnectionFailed` | +| Duplicate event | Drop through runtime dedup state; no second LLM call | +| Secret rotated | Restart target Pod; reconnect with new credentials | +| App ID already owned | Prevent second channel consumer; `AppAlreadyClaimed` | + +Retries must be bounded and jittered. Logs must not include message content or credentials. + +## 19. Testing + +### 19.1 CRD and capability tests + +- Feishu default values serialize in camelCase/PascalCase correctly. +- Exactly one Feishu entry is allowed. +- Webhook or unknown connection modes are rejected. +- Invalid `ou_` and `oc_` identifiers are rejected. +- OpenClaw and Hermes combinations are accepted. +- Other runtime combinations produce `UnsupportedByRuntime`. +- Inline credential fields are absent from schema. +- Helm and Rust-generated schema validation remains in parity. + +### 19.2 CLI tests + +- `kars add` creates CR policy and Secret credentials separately. +- Dry-run never emits App Secret. +- Missing/partial credentials fail before resource writes. +- Unsupported runtime combinations exit non-zero. +- `credentials update` merges and rotates both credential keys. +- Summaries show channel type/policy without secret values. +- Existing Telegram/Slack/Discord/WhatsApp flows remain unchanged. + +### 19.3 OpenClaw image and entrypoint tests + +- Image contains the exact pinned `@openclaw/feishu` plugin. +- Plugin discovery succeeds from the immutable external plugin stage while the bundled tree remains pruned. +- Complete credentials generate `channels.feishu` and plugin allow/entry blocks. +- Partial credentials fail loud. +- Pairing, group allowlist, and mention policy translate exactly. +- No secret appears in generated logs. +- WebSocket reconnect works through the Kars transparent proxy. + +### 19.4 Hermes image and entrypoint tests + +- Image contains `lark-oapi==1.5.3` and `qrcode==7.4.2` or equivalent pinned Feishu extra. +- Native adapter imports successfully. +- Credentials and domain/connection mode reach the adapter. +- Typed DM/group policy is enforced without weakening. +- Partial credentials or missing dependencies fail loud. +- Existing channel adapters remain functional. + +### 19.5 End-to-end tests + +Run separate OpenClaw and Hermes sandboxes against dedicated test Feishu Apps: + +1. WebSocket connects and `ChannelReady=True`. +2. Allowed DM enters Pairing flow and approved user can chat. +3. Unknown DM cannot trigger an LLM call before pairing. +4. Allowed group with direct bot mention receives a response. +5. Allowed group without mention receives no response. +6. Unknown group receives no response. +7. `@all` alone does not trigger a response. +8. Text, image, and file message behavior is verified at the declared support level. +9. Pod restart reconnects automatically. +10. Credential rotation reconnects only the target sandbox. +11. Strict egress denies unapproved hosts; approved hosts restore connection. +12. Two sandboxes attempting one App ID produce `AppAlreadyClaimed` or an explicit supported warning path. +13. Suspended sandbox reports `ChannelReady=False/Suspended` and does not claim message wake-up support. +14. PVC-backed sandbox preserves pairing/dedup state across Pod recreation when the runtime stores that state under `/sandbox`. + +Tests use dedicated non-production Feishu tenants/apps and never record credential values or message bodies in fixtures. + +## 20. Documentation updates + +Implementation must update: + +- `docs/channels-plugins.md` with Feishu setup, permissions, group IDs, pairing, egress, and rotation; +- `docs/cli-reference.md` with the new flags; +- `docs/runtimes.md` and `docs/hermes-plugin.md` with runtime capability differences; +- `docs/security.md` with channel credential and message trust boundaries; +- `docs/api/crd-reference.md` and `docs/api/conditions.md` with typed channels and `ChannelReady`; +- image versioning documentation with the OpenClaw plugin and Hermes extra pins; +- troubleshooting guidance for WebSocket auth, group mention, allowlist, reconnect, and duplicate App ownership. + +## 21. Rollout + +1. Land schema and capability validation behind a disabled-by-default feature gate if CRD evolution requires staged rollout. +2. Build and scan OpenClaw/Hermes images with the pinned Feishu dependencies. +3. Run unit and image-shape tests. +4. Validate one dedicated OpenClaw test App in Learn mode. +5. Validate one dedicated Hermes test App in Learn mode. +6. Approve reviewed egress hosts and repeat in Strict mode. +7. Enable Feishu CLI flags for production use. +8. Monitor connection failures, reconnect counts, egress denials, and duplicate event drops without message-content labels. + +Rollback removes the channel declaration and credentials from the selected sandbox, restarts only that Deployment, and leaves the rest of the runtime unaffected. + +## 22. Acceptance criteria + +The feature is complete when: + +1. The same typed Feishu policy works for OpenClaw and Hermes. +2. App credentials exist only in the per-sandbox Secret. +3. Both runtimes use outbound WebSocket transport and require no public ingress. +4. OpenClaw image contains the exact compatible Feishu plugin. +5. Hermes image contains the exact compatible Feishu optional dependencies. +6. DM defaults to Pairing. +7. Groups default to Allowlist with direct mention required. +8. One App can serve multiple allowlisted groups in one sandbox. +9. Unsupported runtime combinations fail visibly before readiness. +10. Missing or partial credentials fail closed. +11. `ChannelReady` reflects connection state rather than only configuration presence. +12. Egress remains operator-approved and no channel host is silently allowlisted. +13. Secret rotation restarts only the target sandbox and does not leak credentials. +14. Existing Discord, Telegram, Slack, and WhatsApp behavior does not regress. +15. No wake-from-zero capability is claimed. +16. OpenClaw and Hermes Feishu E2E scenarios pass with dedicated test Apps. + +## 23. Future extensions + +- central Channel Gateway for one App routing to multiple sandboxes; +- message queue and wake-from-zero; +- webhook transport with public ingress, signature verification, encryption, and replay protection; +- DingTalk and WeCom adapters using the same typed contract; +- BYO runtime channel capability declarations in a future contract version; +- per-group user policies and per-group mention overrides; +- dynamic per-user agent creation; +- channel-level rate limits, quotas, and delivery SLOs; +- centralized deduplication and dead-letter handling; +- richer Feishu workplace tools under separately governed tool capabilities. diff --git a/docs/plans/2026-08-09-feishu-channel-contract-implementation.md b/docs/plans/2026-08-09-feishu-channel-contract-implementation.md new file mode 100644 index 000000000..d0d1b5ccc --- /dev/null +++ b/docs/plans/2026-08-09-feishu-channel-contract-implementation.md @@ -0,0 +1,119 @@ +# Feishu Channel Contract Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Add typed Feishu channel policy and per-sandbox credentials with first-class OpenClaw and Hermes WebSocket adapters, while rejecting unsupported runtimes. + +**Architecture:** `KarsSandbox.spec.channels[]` is the source of truth for non-sensitive channel policy; a dedicated immutable, versioned Secret selected by `credentialSecretRef` holds App ID/App Secret. The controller validates runtime capability, App ownership, and Secret identity, then injects policy plus explicit credential key refs only into the runtime container. OpenClaw and Hermes entrypoints translate the common contract into their native configuration, while image builds pin their runtime-specific Feishu dependencies. + +**Tech Stack:** Rust 2024/kube-rs/schemars, Kubernetes CRD CEL/Secret/env, TypeScript Commander/Vitest, Bash entrypoints, OpenClaw 2026.5.27, Hermes Agent 0.16.0, Helm. + +--- + +### Task 1: Typed channel CRD and capability validation + +**Files:** +- Modify: `controller/src/crd.rs` +- Modify: `controller/src/crd_validations.rs` +- Modify: `controller/src/reconciler/runtime.rs` +- Modify: `deploy/helm/kars/templates/crd.yaml` +- Test: `controller/src/crd.rs` +- Test: `controller/src/reconciler/runtime.rs` +- Test: `controller/src/helm_drift.rs` + +1. Write failing serialization/default tests for `spec.channels[].type=Feishu` and safe policy defaults. +2. Write failing runtime capability tests accepting OpenClaw/Hermes and rejecting all other runtime kinds. +3. Implement typed Rust structs/enums and defensive capability validation. +4. Add matching Helm schema/CEL and Rust-generated CRD validation parity. +5. Run focused CRD/runtime/drift tests. + +### Task 2: Controller policy translation and secret isolation + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Modify: `controller/src/reconciler/tests.rs` +- Modify: `controller/src/status/conditions.rs` +- Modify: `controller/src/status/mod.rs` + +1. Write failing tests for Feishu policy env generation and runtime-container-only injection. +2. Write failing tests for unsupported runtime and missing/partial credential status outcomes. +3. Implement policy env compilation (`FEISHU_DOMAIN`, WebSocket mode, DM/group policy, ID lists, mention requirement). +4. Validate the credential Secret shape without copying secret values into status/logs. +5. Add `ChannelReady` vocabulary and fail-closed status conditions. +6. Verify router/init containers do not receive Feishu credentials or policy env. + +### Task 3: CLI create and credential rotation + +**Files:** +- Modify: `cli/src/commands/add.ts` +- Modify: `cli/src/commands/add.test.ts` +- Modify: `cli/src/commands/credentials.ts` +- Modify: `cli/src/config.ts` +- Test: `cli/src/commands/add.test.ts` +- Test: `cli/src/config.test.ts` + +1. Write failing tests for Feishu flags, CR policy output, Secret mapping, and unsupported runtimes. +2. Add App ID/App Secret and policy flags to `kars add`. +3. Keep App credentials out of KarsSandbox/dry-run output and put them in a dedicated immutable Feishu Secret. +4. Extend local secret lookup and `kars credentials update` rotation with staged/adopted Secret revisions and a `resourceVersion`-guarded JSON Patch that changes only `credentialSecretRef`. +5. Run add/config/credentials tests, typecheck, and build. + +### Task 4: OpenClaw image and adapter + +**Files:** +- Modify: `sandbox-images/openclaw/Dockerfile.base` +- Modify: `sandbox-images/openclaw/Dockerfile` +- Modify: `sandbox-images/openclaw/entrypoint.sh` +- Create: `sandbox-images/openclaw/testM_feishu_channel.sh` + +1. Write a failing shell test for complete credentials, partial credentials, policy translation, and plugin enablement. +2. Install `@openclaw/feishu` at the exact OpenClaw version during image build and make discovery fail closed. +3. Preserve the Feishu plugin in the pruned runtime plugin layout or install it in the immutable external plugin directory. +4. Generate `channels.feishu`, `plugins.allow`, and `plugins.entries` from the common env contract. +5. Fail loud on partial credentials or missing plugin. +6. Run shell syntax and behavior tests. + +### Task 5: Hermes image and adapter + +**Files:** +- Modify: `sandbox-images/hermes/Dockerfile` +- Modify: `sandbox-images/hermes/entrypoint.sh` +- Create: `sandbox-images/hermes/testM_feishu_channel.sh` + +1. Write a failing shell/image-shape test for Feishu dependencies and env/config translation. +2. Install the pinned `hermes-agent[feishu]==0.16.0` dependency set. +3. Translate common credentials and WebSocket/domain policy to Hermes native env/config. +4. Map or enforce DM Pairing, group Allowlist, allowed IDs, and mention gating without weakening policy. +5. Fail loud on partial credentials or dependency import failure. +6. Run shell syntax, behavior, and Hermes runtime tests. + +### Task 6: Channel readiness and App ownership + +**Files:** +- Modify: `controller/src/reconciler/mod.rs` +- Modify: `controller/src/reconciler/tests.rs` +- Modify: `controller/src/status/conditions.rs` +- Modify: `controller/src/status/mod.rs` + +1. Write failing tests for `ChannelReady` Configured/Connecting/Failed/Suspended and App fingerprint conflicts. +2. Implement a non-secret App ID fingerprint ownership record and finalizer cleanup, or explicitly narrow v1 to warning-only if cluster-wide ownership cannot be made race-free. +3. Consume a non-sensitive runtime readiness signal instead of treating Secret existence as connection success. +4. Gate overall `Ready=True` on declared channel readiness. +5. Verify credentials and message content never appear in status/events/metrics. + +### Task 7: Documentation and regression validation + +**Files:** +- Modify: `docs/channels-plugins.md` +- Modify: `docs/cli-reference.md` +- Modify: `docs/runtimes.md` +- Modify: `docs/hermes-plugin.md` +- Modify: `docs/security.md` +- Modify: `docs/api/crd-reference.md` +- Modify: `docs/api/conditions.md` +- Modify: `docs/operations/image-versioning.md` + +1. Document Feishu Open Platform setup, WebSocket mode, permissions, IDs, pairing, group allowlist, egress, persistence, and rotation. +2. Document runtime capability differences and unsupported combinations. +3. Run Controller full tests, Clippy, Rust formatting, CLI tests/typecheck/lint/build, both shell tests, Dockerfile/image-shape checks, CRD server dry-run, and diff validation. +4. Request independent code review focused on credential leakage, policy weakening, plugin discovery, runtime capability, and ChannelReady truthfulness. diff --git a/docs/runtimes.md b/docs/runtimes.md index 0b39fbc0e..0b9619c4b 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -65,7 +65,7 @@ If your runtime SDK reads its model endpoint from one of the well-known env vars ### `OpenClaw` -Default. Uses the [OpenClaw](https://openclaw.ai) public plugin API + `tools.deny` config. **No OpenClaw source is modified, patched, or vendored.** Any upstream OpenClaw release is drop-in compatible. See **[Upstream alignment](upstream-alignment.md)** for the contract details. +Default. Uses the [OpenClaw](https://openclaw.ai) public plugin API + `tools.deny` config. The image pins `openclaw@2026.5.27` and the matching `@openclaw/feishu@2026.5.27`; image builds fail if the Feishu plugin is not discoverable. OpenClaw core source is not modified. See **[Upstream alignment](upstream-alignment.md)** for the contract details. The OpenClaw adapter ships two multi-agent helpers on top of the platform mesh: @@ -74,7 +74,7 @@ The OpenClaw adapter ships two multi-agent helpers on top of the platform mesh: ### `Hermes` -[Hermes Agent](https://github.com/NousResearch/hermes-agent) (Nous Research, MIT). Python 3.11+ agent harness pinned to v0.15.2 by default. Ships **20+ messaging channels** (Telegram, Slack, Discord, WhatsApp, …), **18+ inference providers**, **70+ built-in tools**, and a native MCP client out-of-the-box. The kars Hermes plugin wires Hermes into AGT governance, the kars mesh (Python AGT MeshClient via `runtimes/agt-mesh-python/`), Foundry data-plane tools, and the same CRD-driven `agentCode` mounting story as OpenClaw. +[Hermes Agent](https://github.com/NousResearch/hermes-agent) (Nous Research, MIT). Python 3.11+ agent harness pinned to v0.16.0 by default. The image installs the exact `hermes-agent[feishu]==0.16.0` extra and applies a source-anchored compatibility patch for the Kars DM/group admission contract; a changed upstream source shape fails the image build. Hermes ships **20+ messaging channels**, **18+ inference providers**, **70+ built-in tools**, and a native MCP client out-of-the-box. The kars Hermes plugin wires Hermes into AGT governance, the kars mesh (Python AGT MeshClient via `runtimes/agt-mesh-python/`), Foundry data-plane tools, and the same CRD-driven `agentCode` mounting story as OpenClaw. What makes Hermes a useful counterpart to OpenClaw: @@ -82,6 +82,8 @@ What makes Hermes a useful counterpart to OpenClaw: - **MCP client built-in.** Hermes' native `mcp_servers` config lets the agent reach the kars platform MCP server (Foundry tools) without writing the bridge yourself. - **Bidi mesh peer.** Hermes participates in the AGT mesh identically to OpenClaw — `OpenClaw → Hermes`, `Hermes → OpenClaw`, and `Hermes → Hermes` are all proven end-to-end on AKS (see [`tests/e2e/interop/hermes_openclaw_bidi.sh`](../tests/e2e/interop/hermes_openclaw_bidi.sh) and [`tests/e2e/interop/aks_full_suite.sh`](../tests/e2e/interop/aks_full_suite.sh)). +OpenClaw and Hermes are the only v1 runtimes that accept `spec.channels[].type=Feishu`. Both use outbound WebSocket transport, DM pairing by default, group chat-ID allowlisting, and mention gating. Other runtime kinds fail capability validation before a runtime Pod is made ready. + Full operator-facing reference: **[Hermes plugin](hermes-plugin.md)**. ### `OpenAIAgents` diff --git a/docs/security.md b/docs/security.md index bd616427b..7f66ef122 100644 --- a/docs/security.md +++ b/docs/security.md @@ -15,6 +15,10 @@ For threat-model walkthroughs, see **[STRIDE](security/stride.md)** and the **[R 3. **Inter-agent messages are E2E encrypted with forward secrecy.** Compromise of the AgentMesh relay does not expose any past or future message content. 4. **Every external call is audited in a tamper-evident chain.** Each audit record carries a SHA-256 hash of the previous record, so any deletion or modification — including by the cluster operator — breaks the chain and is detectable on replay. (We do not yet sign the chain head with a separate key; that is on the roadmap. The integrity property today is *detection*, not *non-repudiation*.) +Channel credentials are a separate boundary from Azure credentials. A messaging runtime must receive its bot token or App Secret to authenticate to that platform. Ordinary channel/plugin values use the per-sandbox `-credentials` Secret. Feishu App credentials use a dedicated immutable, versioned Secret referenced by `spec.channels[].credentialSecretRef`. Only the UID 1000 runtime container receives these values; the inference-router and init containers do not. Feishu policy is non-sensitive and lives in `KarsSandbox.spec.channels[]`. + +Feishu defaults fail closed: DM pairing, group chat-ID allowlist, direct mention required, WebSocket-only transport, and one App ID per sandbox. The controller indexes only a SHA-256 App ID fingerprint in an internal ownership ConfigMap and never writes the App ID, App Secret, fingerprint, or message content to CR status. `ChannelReady=True` requires a runtime-specific connection signal, not Secret presence or generic Pod readiness. + Everything below explains how those four guarantees are enforced and where the seams are. > **What is not yet enforced in this release.** Trying to be explicit so reviewers do not have to hunt: diff --git a/sandbox-images/hermes/Dockerfile b/sandbox-images/hermes/Dockerfile index d07c5f758..fd875deec 100644 --- a/sandbox-images/hermes/Dockerfile +++ b/sandbox-images/hermes/Dockerfile @@ -110,7 +110,16 @@ RUN if ls /tmp/agt-wheels/*.whl >/dev/null 2>&1; then \ # bumping to a newer Hermes should also re-verify the kars runtime # contract is still honored (entrypoint env shape + plugin context API). ARG HERMES_VERSION=0.16.0 -RUN pip install --no-cache-dir "hermes-agent==${HERMES_VERSION}" +RUN pip install --no-cache-dir "hermes-agent[feishu]==${HERMES_VERSION}" + +# Hermes 0.16.0's Feishu adapter supports chat-ID group rules and the gateway +# has a DM pairing store, but the adapter does not expose dm_policy or declare +# that it owns admission. Apply a source-anchored compatibility patch; it fails +# the build if the pinned upstream source shape changes. +COPY sandbox-images/hermes/patch-hermes-feishu-policy.py /tmp/patch-hermes-feishu-policy.py +RUN python3 /tmp/patch-hermes-feishu-policy.py && \ + python3 -c "import lark_oapi, qrcode; from gateway.platforms.feishu import FeishuAdapter; assert FeishuAdapter.enforces_own_access_policy" && \ + rm /tmp/patch-hermes-feishu-policy.py # ---- Channel adapter libraries ----------------------------------------- # Hermes auto-detects channels (Telegram / Slack / Discord) from env @@ -187,7 +196,8 @@ COPY --chown=1000:1000 sandbox-images/hermes/default-agent/ /opt/kars-default-ag # ---- Entrypoint --------------------------------------------------------- COPY sandbox-images/hermes/entrypoint.sh /usr/local/bin/kars-hermes-entrypoint.sh -RUN chmod 0755 /usr/local/bin/kars-hermes-entrypoint.sh +COPY sandbox-images/hermes/kars-channel-feishu-ready /usr/local/bin/kars-channel-feishu-ready +RUN chmod 0755 /usr/local/bin/kars-hermes-entrypoint.sh /usr/local/bin/kars-channel-feishu-ready # ---- Security: drop to UID 1000 (matches AKS pod spec) ------------------ # In docker dev mode, the entrypoint may run as root briefly to set up diff --git a/sandbox-images/hermes/entrypoint.sh b/sandbox-images/hermes/entrypoint.sh index 98a3e028d..3ffa903be 100644 --- a/sandbox-images/hermes/entrypoint.sh +++ b/sandbox-images/hermes/entrypoint.sh @@ -24,7 +24,73 @@ # 7. Set up iptables egress guard in docker dev (k8s has init container) # 8. Drop to UID 1000 if started as root (docker dev only) and exec hermes +clear_feishu_readiness() { + rm -f "${KARS_FEISHU_READY_PATH:-/tmp/kars-channel-feishu-ready}" +} + +validate_feishu_channel() { + if [ -z "${FEISHU_CONNECTION_MODE:-}" ]; then + return 0 + fi + if [ -z "${FEISHU_APP_ID:-}" ] && [ -z "${FEISHU_APP_SECRET:-}" ]; then + echo "[kars-hermes] ERROR: Feishu requires both FEISHU_APP_ID and FEISHU_APP_SECRET" >&2 + return 1 + fi + if [ -z "${FEISHU_APP_ID:-}" ] || [ -z "${FEISHU_APP_SECRET:-}" ]; then + echo "[kars-hermes] ERROR: Feishu requires both FEISHU_APP_ID and FEISHU_APP_SECRET" >&2 + return 1 + fi + if [ "$FEISHU_CONNECTION_MODE" != "websocket" ]; then + echo "[kars-hermes] ERROR: Feishu only supports websocket connection mode" >&2 + return 1 + fi + case "${FEISHU_DM_POLICY:-pairing}" in pairing|allowlist|disabled) ;; *) return 1 ;; esac + case "${FEISHU_GROUP_POLICY:-allowlist}" in allowlist|disabled) ;; *) return 1 ;; esac + case "${FEISHU_REQUIRE_MENTION:-true}" in true|false) ;; *) return 1 ;; esac +} + +render_feishu_platform_config() { + validate_feishu_channel + [ -z "${FEISHU_CONNECTION_MODE:-}" ] && return 0 + + local dm_allow_json group_id + dm_allow_json=$(python3 -c 'import json, os; print(json.dumps([v.strip() for v in os.getenv("FEISHU_ALLOW_FROM", "").split(",") if v.strip()], separators=(",", ":")))') + cat <&2 + return 1 + ;; + esac + cat < "$MCP_FRAGMENT" # Merge into config.yaml. The two blocks the entrypoint owns -# (`plugins:` and `mcp_servers:`) are stripped from any existing +# (`plugins:`, `mcp_servers:`, `model:`, and `platforms:`) are stripped from any existing # config and replaced with freshly-generated versions. Everything # else the user may have written via `hermes config set ` # is preserved across pod restarts. @@ -296,7 +366,7 @@ fragment = open(fragment_path).read() # blocks (the three sections the entrypoint owns) so re-runs are # idempotent. Everything else the user wrote via `hermes config set # ` is preserved across pod restarts. -top_key = re.compile(r"^(plugins|mcp_servers|model):", re.M) +top_key = re.compile(r"^(plugins|mcp_servers|model|platforms):", re.M) out, idx, prev = io.StringIO(), 0, 0 while True: m = top_key.search(src, idx) @@ -331,7 +401,7 @@ rm -f "$MCP_FRAGMENT" # Channels Hermes supports natively that map from kars envs: # telegram, slack, discord, whatsapp, signal, matrix, email # Channels kars doesn't expose creds for yet: -# mattermost, dingtalk, feishu, wecom, weixin, bluebubbles, +# mattermost, dingtalk, wecom, weixin, bluebubbles, # qqbot, homeassistant, webhook, api_server, yuanbao, sms # Operators wanting those can `hermes config set` manually post-boot # from their own creds. diff --git a/sandbox-images/hermes/kars-channel-feishu-ready b/sandbox-images/hermes/kars-channel-feishu-ready new file mode 100644 index 000000000..4efda2caa --- /dev/null +++ b/sandbox-images/hermes/kars-channel-feishu-ready @@ -0,0 +1,7 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail + +test -f "${KARS_FEISHU_READY_PATH:-/tmp/kars-channel-feishu-ready}" diff --git a/sandbox-images/hermes/patch-hermes-feishu-policy.py b/sandbox-images/hermes/patch-hermes-feishu-policy.py new file mode 100644 index 000000000..b2f694bc0 --- /dev/null +++ b/sandbox-images/hermes/patch-hermes-feishu-policy.py @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Patch Hermes 0.16.0 Feishu admission to honor the Kars channel contract.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +def replace_once(source: str, old: str, new: str) -> str: + if source.count(old) != 1: + raise RuntimeError(f"expected one Hermes Feishu source anchor, found {source.count(old)}") + return source.replace(old, new) + + +def main() -> None: + spec = importlib.util.find_spec("gateway.platforms.feishu") + if spec is None or spec.origin is None: + raise RuntimeError("Hermes Feishu adapter is not installed") + + gateway_spec = importlib.util.find_spec("gateway.run") + if gateway_spec is None or gateway_spec.origin is None: + raise RuntimeError("Hermes gateway runner is not installed") + gateway_source = Path(gateway_spec.origin).read_text() + pairing_contract = ( + "def _adapter_dm_policy", + "pairing_store.is_approved", + 'self._adapter_dm_policy(source.platform) == "pairing"', + "self.pairing_store.generate_code", + ) + if any(anchor not in gateway_source for anchor in pairing_contract): + raise RuntimeError("Hermes gateway pairing contract is incompatible") + + path = Path(spec.origin) + source = path.read_text() + source = replace_once( + source, + 'class FeishuAdapter(BasePlatformAdapter):\n """Feishu/Lark bot adapter."""\n', + 'class FeishuAdapter(BasePlatformAdapter):\n' + ' """Feishu/Lark bot adapter."""\n\n' + ' enforces_own_access_policy = True\n' + ' _kars_ready_path = Path("/tmp/kars-channel-feishu-ready")\n', + ) + source = replace_once( + source, + " group_policy: str\n allowed_group_users: frozenset[str]\n", + " dm_policy: str\n dm_allow_from: frozenset[str]\n" + " group_policy: str\n allowed_group_users: frozenset[str]\n", + ) + source = replace_once( + source, + ' group_policy=os.getenv("FEISHU_GROUP_POLICY", "allowlist").strip().lower(),\n', + ' dm_policy=str(extra.get("dm_policy") or os.getenv("FEISHU_DM_POLICY", "pairing")).strip().lower(),\n' + ' dm_allow_from=frozenset(str(item).strip() for item in extra.get("dm_allow_from", []) if str(item).strip()),\n' + ' group_policy=os.getenv("FEISHU_GROUP_POLICY", "allowlist").strip().lower(),\n', + ) + source = replace_once( + source, + " self._group_policy = settings.group_policy\n", + " self._dm_policy = settings.dm_policy\n" + " self._dm_allow_from = set(settings.dm_allow_from)\n" + " self._group_policy = settings.group_policy\n", + ) + source = replace_once( + source, + " if not is_group:\n return None\n", + " if not is_group:\n" + " if self._dm_policy == \"disabled\":\n" + " return \"dm_policy_rejected\"\n" + " if self._dm_policy == \"allowlist\":\n" + " if not sender_ids or not (sender_ids & self._dm_allow_from):\n" + " return \"dm_policy_rejected\"\n" + " if self._dm_policy not in {\"pairing\", \"allowlist\", \"disabled\"}:\n" + " return \"dm_policy_rejected\"\n" + " return None\n", + ) + source = replace_once( + source, + " async def disconnect(self) -> None:\n" + " \"\"\"Disconnect from Feishu/Lark.\"\"\"\n" + " self._running = False\n", + " async def disconnect(self) -> None:\n" + " \"\"\"Disconnect from Feishu/Lark.\"\"\"\n" + " self._running = False\n" + " self._kars_ready_path.unlink(missing_ok=True)\n", + ) + source = replace_once( + source, + " except Exception as exc:\n await self._release_app_lock()\n", + " except Exception as exc:\n" + " self._kars_ready_path.unlink(missing_ok=True)\n" + " await self._release_app_lock()\n", + ) + source = replace_once( + source, + " except Exception as exc:\n" + " self._running = False\n" + " self._disable_websocket_auto_reconnect()\n", + " except Exception as exc:\n" + " self._running = False\n" + " self._kars_ready_path.unlink(missing_ok=True)\n" + " self._disable_websocket_auto_reconnect()\n", + ) + path.write_text(source) + + sdk_spec = importlib.util.find_spec("lark_oapi") + if sdk_spec is None or sdk_spec.submodule_search_locations is None: + raise RuntimeError("lark-oapi WebSocket client is not installed") + + sdk_root = next(iter(sdk_spec.submodule_search_locations), None) + if sdk_root is None: + raise RuntimeError("lark-oapi package directory is unavailable") + sdk_path = Path(sdk_root) / "ws" / "client.py" + sdk_source = sdk_path.read_text() + sdk_source = replace_once( + sdk_source, + "import time\nfrom urllib.parse import urlparse, parse_qs\n", + "import time\nfrom pathlib import Path\nfrom urllib.parse import urlparse, parse_qs\n\n" + '_KARS_READY_PATH = Path("/tmp/kars-channel-feishu-ready")\n', + ) + sdk_source = replace_once( + sdk_source, + " self._conn = conn\n self._conn_url = conn_url\n", + " self._conn = conn\n" + " _KARS_READY_PATH.touch(mode=0o600, exist_ok=True)\n" + " self._conn_url = conn_url\n", + ) + sdk_source = replace_once( + sdk_source, + " finally:\n self._conn = None\n self._conn_url = \"\"\n", + " finally:\n" + " _KARS_READY_PATH.unlink(missing_ok=True)\n" + " self._conn = None\n" + " self._conn_url = \"\"\n", + ) + sdk_path.write_text(sdk_source) + + +if __name__ == "__main__": + main() diff --git a/sandbox-images/hermes/testM_feishu_channel.sh b/sandbox-images/hermes/testM_feishu_channel.sh new file mode 100644 index 000000000..a5de1fa12 --- /dev/null +++ b/sandbox-images/hermes/testM_feishu_channel.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +DOCKERFILE="$SCRIPT_DIR/Dockerfile" + +grep -Fq '"hermes-agent[feishu]==${HERMES_VERSION}"' "$DOCKERFILE" +grep -Fq 'patch-hermes-feishu-policy.py' "$DOCKERFILE" +grep -Fq 'import lark_oapi, qrcode' "$DOCKERFILE" +grep -Fq 'kars-channel-feishu-ready' "$DOCKERFILE" +grep -Fq '"ws" / "client.py"' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq '_KARS_READY_PATH.touch' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq '_adapter_dm_policy' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq 'pairing_store.is_approved' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" + +READY_MARKER=$(mktemp) +rm -f "$READY_MARKER" +KARS_FEISHU_READY_PATH="$READY_MARKER" bash "$SCRIPT_DIR/kars-channel-feishu-ready" 2>/dev/null && { + echo "expected Hermes readiness to fail without its marker" >&2 + exit 1 +} +touch "$READY_MARKER" +KARS_FEISHU_READY_PATH="$READY_MARKER" bash "$SCRIPT_DIR/kars-channel-feishu-ready" +rm -f "$READY_MARKER" + +# shellcheck source=entrypoint.sh +source "$SCRIPT_DIR/entrypoint.sh" + +READY_MARKER=$(mktemp) +KARS_FEISHU_READY_PATH="$READY_MARKER" clear_feishu_readiness +if [ -e "$READY_MARKER" ]; then + echo "expected Hermes entrypoint startup to clear a stale readiness marker" >&2 + exit 1 +fi + +reset_config() { + unset FEISHU_APP_ID FEISHU_APP_SECRET FEISHU_DOMAIN FEISHU_CONNECTION_MODE + unset FEISHU_DM_POLICY FEISHU_ALLOW_FROM FEISHU_GROUP_POLICY + unset FEISHU_GROUP_ALLOW_FROM FEISHU_REQUIRE_MENTION +} + +reset_config +export FEISHU_APP_ID='cli_test' +export FEISHU_APP_SECRET='secret' +export FEISHU_DOMAIN='feishu' +export FEISHU_CONNECTION_MODE='websocket' +export FEISHU_DM_POLICY='pairing' +export FEISHU_GROUP_POLICY='allowlist' +export FEISHU_GROUP_ALLOW_FROM='oc_group1,oc_group2' +export FEISHU_REQUIRE_MENTION='true' +CONFIG=$(render_feishu_platform_config) +printf '%s\n' "$CONFIG" | grep -Fq 'dm_policy: "pairing"' +printf '%s\n' "$CONFIG" | grep -Fq 'default_group_policy: "disabled"' +printf '%s\n' "$CONFIG" | grep -Fq '"oc_group1":' +printf '%s\n' "$CONFIG" | grep -Fq '"oc_group2":' +printf '%s\n' "$CONFIG" | grep -Fq 'require_mention: true' + +reset_config +export FEISHU_APP_ID='cli_test' +export FEISHU_APP_SECRET='secret' +export FEISHU_CONNECTION_MODE='websocket' +export FEISHU_DM_POLICY='allowlist' +export FEISHU_ALLOW_FROM='ou_user1,ou_user2' +CONFIG=$(render_feishu_platform_config) +printf '%s\n' "$CONFIG" | grep -Fq 'dm_allow_from: ["ou_user1","ou_user2"]' + +reset_config +export FEISHU_APP_SECRET='secret' +export FEISHU_CONNECTION_MODE='websocket' +if (validate_feishu_channel >/dev/null 2>&1); then + echo "expected partial Feishu credentials to fail" >&2 + exit 1 +fi + +reset_config +export FEISHU_APP_ID='cli_stale' +export FEISHU_APP_SECRET='stale-secret' +if [ -n "$(render_feishu_platform_config)" ]; then + echo "expected stale Feishu credentials without typed policy to stay disabled" >&2 + exit 1 +fi + +reset_config +export FEISHU_APP_ID='cli_test' +export FEISHU_APP_SECRET='secret' +export FEISHU_CONNECTION_MODE='webhook' +if (validate_feishu_channel >/dev/null 2>&1); then + echo "expected webhook mode to fail" >&2 + exit 1 +fi + +echo "Hermes Feishu channel tests passed" \ No newline at end of file diff --git a/sandbox-images/openclaw/Dockerfile b/sandbox-images/openclaw/Dockerfile index 962b9c7c6..d4b668732 100644 --- a/sandbox-images/openclaw/Dockerfile +++ b/sandbox-images/openclaw/Dockerfile @@ -49,6 +49,15 @@ RUN node -e "const f='./mesh-plugin/package.json'; const p=require(f); delete p. FROM ${SANDBOX_BASE_IMAGE} +# Keep the overlay compatible with base images built before Feishu CONNECT +# support was added. The patch is source-anchored and idempotent. +COPY sandbox-images/openclaw/patch-feishu-proxy.cjs /tmp/patch-feishu-proxy.cjs +RUN chmod -R u+w /opt/openclaw-feishu-stage && \ + node /tmp/patch-feishu-proxy.cjs /opt/openclaw-feishu-stage && \ + chmod -R a-w /opt/openclaw-feishu-stage && \ + chmod -R a+rX /opt/openclaw-feishu-stage && \ + rm /tmp/patch-feishu-proxy.cjs + # Proxy bootstrap — preloaded by NODE_OPTIONS to set undici global dispatcher COPY sandbox-images/openclaw/proxy-bootstrap.js /usr/local/lib/proxy-bootstrap.js @@ -176,7 +185,9 @@ RUN mkdir -p /etc/kars/policies /etc/kars/blocklist && \ # Copy entrypoint that auto-configures OpenClaw from mounted secrets COPY sandbox-images/openclaw/entrypoint.sh /usr/local/bin/entrypoint.sh COPY sandbox-images/openclaw/workspace-bootstrap.sh /usr/local/bin/workspace-bootstrap.sh -RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/workspace-bootstrap.sh +COPY sandbox-images/openclaw/kars-channel-feishu-ready /usr/local/bin/kars-channel-feishu-ready +RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/workspace-bootstrap.sh \ + /usr/local/bin/kars-channel-feishu-ready # Labels LABEL org.opencontainers.image.title="kars OpenClaw Sandbox" \ diff --git a/sandbox-images/openclaw/Dockerfile.base b/sandbox-images/openclaw/Dockerfile.base index 304176e23..4a7e694cb 100644 --- a/sandbox-images/openclaw/Dockerfile.base +++ b/sandbox-images/openclaw/Dockerfile.base @@ -15,7 +15,7 @@ FROM ${AZURELINUX_BASE} AS builder # Detect arch for Node.js download (x64 or arm64) ARG TARGETARCH RUN NODEJS_ARCH=$(case "${TARGETARCH:-$(uname -m)}" in arm64|aarch64) echo "arm64";; *) echo "x64";; esac) && \ - tdnf install -y tar gzip ca-certificates curl git && tdnf clean all && \ + tdnf install -y tar gzip ca-certificates curl git jq && tdnf clean all && \ curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors --connect-timeout 15 "https://nodejs.org/dist/v22.22.3/node-v22.22.3-linux-${NODEJS_ARCH}.tar.gz" -o /tmp/node.tar.gz && \ tar xzf /tmp/node.tar.gz -C /usr/local --strip-components=1 && \ rm /tmp/node.tar.gz @@ -26,7 +26,22 @@ RUN NODEJS_ARCH=$(case "${TARGETARCH:-$(uname -m)}" in arm64|aarch64) echo "arm6 # or change the value below to force a fresh pull. ARG OPENCLAW_VERSION=2026.5.27 ARG OPENCLAW_CACHE_BUST=0 -RUN echo "openclaw cache-bust: ${OPENCLAW_CACHE_BUST}" && npm install -g openclaw@${OPENCLAW_VERSION} +COPY sandbox-images/openclaw/patch-feishu-proxy.cjs /usr/local/bin/patch-feishu-proxy.cjs +RUN echo "openclaw cache-bust: ${OPENCLAW_CACHE_BUST}" && \ + npm install -g openclaw@${OPENCLAW_VERSION} && \ + mkdir -p /opt/openclaw-feishu-stage && \ + OPENCLAW_STATE_DIR=/opt/openclaw-feishu-stage \ + OPENCLAW_CONFIG_PATH=/opt/openclaw-feishu-stage/openclaw.json \ + openclaw plugins install "@openclaw/feishu@${OPENCLAW_VERSION}" --pin && \ + node /usr/local/bin/patch-feishu-proxy.cjs /opt/openclaw-feishu-stage && \ + grep -Rqs 'httpsAgent: agent' /opt/openclaw-feishu-stage/npm/node_modules/@openclaw/feishu/dist/client-*.js && \ + grep -Rqs 'r.httpsAgent = agent' /opt/openclaw-feishu-stage/npm/node_modules/@openclaw/feishu/dist/client-*.js && \ + grep -Rqs 'error.config.data = "\[redacted\]"' /opt/openclaw-feishu-stage/npm/node_modules/@openclaw/feishu/dist/client-*.js && \ + OPENCLAW_STATE_DIR=/opt/openclaw-feishu-stage \ + OPENCLAW_CONFIG_PATH=/opt/openclaw-feishu-stage/openclaw.json \ + openclaw plugins list --json | \ + jq -e 'any(.plugins[]?; .id == "feishu" and .status == "loaded")' >/dev/null && \ + chmod -R a-w /opt/openclaw-feishu-stage && chmod -R a+rX /opt/openclaw-feishu-stage # OpenClaw ≥2026.4 hoists channel bundle chunks to dist/ but npm installs don't # stage extension deps into the top-level node_modules/ (openclaw/openclaw#62749). @@ -275,6 +290,7 @@ COPY --from=builder /root/.openclaw/workspace/skills/ /opt/clawhub-skills/ # the deps without invoking npm. Read-only + world-readable; agent UID 1000 can read # but not modify, satisfying the agent self-modification prevention rule. COPY --from=builder /opt/openclaw-stage /opt/openclaw-stage +COPY --from=builder /opt/openclaw-feishu-stage /opt/openclaw-feishu-stage RUN chmod -R a+rX /opt/openclaw-stage # Symlink Control UI assets (the gateway expects dist/control-ui/) diff --git a/sandbox-images/openclaw/entrypoint.sh b/sandbox-images/openclaw/entrypoint.sh index 78c1088e4..c862f1dcc 100644 --- a/sandbox-images/openclaw/entrypoint.sh +++ b/sandbox-images/openclaw/entrypoint.sh @@ -10,6 +10,75 @@ # UID 1001 (router) — inference router, can reach internet # UID 1000 (sandbox) — agent processes, restricted to localhost + DNS +append_feishu_channel_config() { + if [ -z "${FEISHU_CONNECTION_MODE:-}" ]; then + return 0 + fi + if [ -z "${FEISHU_APP_ID:-}" ] && [ -z "${FEISHU_APP_SECRET:-}" ]; then + echo "[kars] FATAL: Feishu requires both FEISHU_APP_ID and FEISHU_APP_SECRET" >&2 + return 1 + fi + if [ -z "${FEISHU_APP_ID:-}" ] || [ -z "${FEISHU_APP_SECRET:-}" ]; then + echo "[kars] FATAL: Feishu requires both FEISHU_APP_ID and FEISHU_APP_SECRET" >&2 + return 1 + fi + if [ "$FEISHU_CONNECTION_MODE" != "websocket" ]; then + echo "[kars] FATAL: Feishu only supports websocket connection mode" >&2 + return 1 + fi + local plugin_stage="${KARS_FEISHU_PLUGIN_STAGE:-/opt/openclaw-feishu-stage}" + local openclaw_state="${OPENCLAW_DIR:-/sandbox/.openclaw}" + if [ ! -f "$plugin_stage/plugins/installs.json" ] || \ + [ ! -d "$plugin_stage/npm/node_modules/@openclaw/feishu" ]; then + echo "[kars] FATAL: pinned OpenClaw Feishu plugin is missing" >&2 + return 1 + fi + mkdir -p "$openclaw_state/plugins" + rm -rf "$openclaw_state/npm" + cp -r "$plugin_stage/npm" "$openclaw_state/npm" + cp "$plugin_stage/plugins/installs.json" "$openclaw_state/plugins/installs.json" + + local allow_from group_allow_from require_mention feishu_config separator + allow_from=$(jq -cn --arg value "${FEISHU_ALLOW_FROM:-}" \ + '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))') + group_allow_from=$(jq -cn --arg value "${FEISHU_GROUP_ALLOW_FROM:-}" \ + '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))') + case "${FEISHU_REQUIRE_MENTION:-true}" in + true) require_mention=true ;; + false) require_mention=false ;; + *) + echo "[kars] FATAL: FEISHU_REQUIRE_MENTION must be true or false" >&2 + return 1 + ;; + esac + + feishu_config=$(jq -cn \ + --arg app_id "$FEISHU_APP_ID" \ + --arg app_secret "$FEISHU_APP_SECRET" \ + --arg domain "${FEISHU_DOMAIN:-feishu}" \ + --arg connection_mode "${FEISHU_CONNECTION_MODE:-websocket}" \ + --arg dm_policy "${FEISHU_DM_POLICY:-pairing}" \ + --argjson allow_from "$allow_from" \ + --arg group_policy "${FEISHU_GROUP_POLICY:-allowlist}" \ + --argjson group_allow_from "$group_allow_from" \ + --argjson require_mention "$require_mention" \ + '{appId: $app_id, appSecret: $app_secret, domain: $domain, + connectionMode: $connection_mode, dmPolicy: $dm_policy, + allowFrom: $allow_from, groupPolicy: $group_policy, + groupAllowFrom: $group_allow_from, requireMention: $require_mention}') + + separator="" + [ -n "${CHANNELS_CONFIG:-}" ] && separator=", " + CHANNELS_CONFIG="${CHANNELS_CONFIG:-}${separator}\"feishu\": ${feishu_config}" + PLUGINS_LIST="${PLUGINS_LIST}, \"feishu\"" + [ -n "${PLUGINS_ENTRIES:-}" ] && PLUGINS_ENTRIES="${PLUGINS_ENTRIES}, " + PLUGINS_ENTRIES="${PLUGINS_ENTRIES}\"feishu\": { \"enabled\": true }" +} + +if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then + return 0 +fi + set -e # Make pre-staged OpenClaw bundled-runtime-deps discoverable at runtime. @@ -966,6 +1035,8 @@ AUTHPROFEOF PLUGINS_ENTRIES="${PLUGINS_ENTRIES}\"discord\": { \"enabled\": true }" fi + append_feishu_channel_config + # Default: no channels configured if [ -z "${CHANNELS_CONFIG}" ]; then CHANNELS_CONFIG="\"_placeholder\": false" diff --git a/sandbox-images/openclaw/kars-channel-feishu-ready b/sandbox-images/openclaw/kars-channel-feishu-ready new file mode 100644 index 000000000..585e4c760 --- /dev/null +++ b/sandbox-images/openclaw/kars-channel-feishu-ready @@ -0,0 +1,16 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail + +status=$(openclaw channels status --channel feishu --json 2>/dev/null) +printf '%s' "$status" | jq -e ' + .gatewayReachable == true and + (.configOnly // false) != true and + any(.channelAccounts.feishu[]?; + .enabled == true and + .configured == true and + .running == true and + (.lastError // null) == null) +' >/dev/null diff --git a/sandbox-images/openclaw/patch-feishu-proxy.cjs b/sandbox-images/openclaw/patch-feishu-proxy.cjs new file mode 100644 index 000000000..07c66f298 --- /dev/null +++ b/sandbox-images/openclaw/patch-feishu-proxy.cjs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +function sanitizeFeishuAxiosError(error) { + const sanitized = new Error('Feishu request failed'); + sanitized.name = 'FeishuRequestError'; + sanitized.isAxiosError = true; + if (typeof error?.code === 'string' && /^[A-Z0-9_-]{1,64}$/.test(error.code)) { + sanitized.code = error.code; + } + if (Number.isInteger(error?.response?.status) && error.response.status >= 100 && error.response.status <= 599) { + sanitized.status = error.response.status; + } + return sanitized; +} + +function main() { +const stageDir = process.argv[2] || '/opt/openclaw-feishu-stage'; +const distDir = path.join(stageDir, 'npm', 'node_modules', '@openclaw', 'feishu', 'dist'); +const candidates = fs.readdirSync(distDir).filter((name) => /^client-.*\.js$/.test(name)); +if (candidates.length !== 1) { + throw new Error(`expected one Feishu client bundle, found ${candidates.length}`); +} + +const bundle = path.join(distDir, candidates[0]); +let source = fs.readFileSync(bundle, 'utf8'); +const oldBlock = `\tconst agent = await getWsProxyAgent(); +\treturn new feishuClientSdk.WSClient({ +\t\tappId, +\t\tappSecret, +\t\tdomain: resolveDomain(domain), +\t\t...callbacks,`; +const newBlock = `\tconst agent = await getWsProxyAgent(); +\tconst httpInstance = agent ? { +\t\trequest: (opts) => feishuClientSdk.defaultHttpInstance.request({ +\t\t\t...opts, +\t\t\tproxy: false, +\t\t\thttpAgent: agent, +\t\t\thttpsAgent: agent +\t\t}) +\t} : feishuClientSdk.defaultHttpInstance; +\treturn new feishuClientSdk.WSClient({ +\t\tappId, +\t\tappSecret, +\t\tdomain: resolveDomain(domain), +\t\thttpInstance, +\t\t...callbacks,`; + +const occurrences = source.split(oldBlock).length - 1; +if (occurrences === 1) { + source = source.replace(oldBlock, newBlock); +} else if (!(occurrences === 0 && source.includes('httpsAgent: agent'))) { + throw new Error(`expected one Feishu WS client source anchor, found ${occurrences}`); +} + +const oldInterceptor = `\t\tinst.interceptors.request.use((req) => { + const r = req; + if (r.headers) r.headers["User-Agent"] = getFeishuUserAgent(); + return req; + });`; +const newInterceptor = `${sanitizeFeishuAxiosError.toString()} + inst.interceptors.request.use(async (req) => { + const r = req; + if (r.headers) r.headers["User-Agent"] = getFeishuUserAgent(); + const agent = await getWsProxyAgent(); + if (agent) { + r.proxy = false; + r.httpAgent = agent; + r.httpsAgent = agent; + } + return req; + }); + inst.interceptors.response?.use(undefined, (error) => { + return Promise.reject(sanitizeFeishuAxiosError(error)); + });`; +const interceptorOccurrences = source.split(oldInterceptor).length - 1; +if (interceptorOccurrences === 1) { + source = source.replace(oldInterceptor, newInterceptor); +} else if (!(interceptorOccurrences === 0 && source.includes('sanitizeFeishuAxiosError(error)'))) { + throw new Error(`expected one Feishu Axios interceptor anchor, found ${interceptorOccurrences}`); +} +if (!source.includes('sanitizeFeishuAxiosError(error)')) { + throw new Error('Feishu Axios error redaction patch missing'); +} +fs.writeFileSync(bundle, source); +console.log(`Patched Feishu Axios proxy transport: ${bundle}`); +} + +if (require.main === module) { + main(); +} + +module.exports = { sanitizeFeishuAxiosError }; diff --git a/sandbox-images/openclaw/testM_feishu_channel.sh b/sandbox-images/openclaw/testM_feishu_channel.sh new file mode 100644 index 000000000..61255e6e6 --- /dev/null +++ b/sandbox-images/openclaw/testM_feishu_channel.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BASE_DOCKERFILE="$SCRIPT_DIR/Dockerfile.base" +RUNTIME_DOCKERFILE="$SCRIPT_DIR/Dockerfile" +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' EXIT + +grep -Fq 'ARG OPENCLAW_VERSION=2026.5.27' "$BASE_DOCKERFILE" +grep -Fq 'plugins install "@openclaw/feishu@${OPENCLAW_VERSION}" --pin' "$BASE_DOCKERFILE" +grep -Fq 'tdnf install -y tar gzip ca-certificates curl git jq' "$BASE_DOCKERFILE" +grep -Fq 'COPY --from=builder /opt/openclaw-feishu-stage /opt/openclaw-feishu-stage' "$BASE_DOCKERFILE" +grep -Fq 'patch-feishu-proxy.cjs' "$BASE_DOCKERFILE" +grep -Fq 'patch-feishu-proxy.cjs' "$RUNTIME_DOCKERFILE" +grep -Fq 'r.httpsAgent = agent' "$SCRIPT_DIR/patch-feishu-proxy.cjs" +grep -Fq 'sanitizeFeishuAxiosError' "$SCRIPT_DIR/patch-feishu-proxy.cjs" +grep -Fq '/opt/openclaw-feishu-stage' "$SCRIPT_DIR/entrypoint.sh" +grep -Fq 'openclaw plugins list' "$BASE_DOCKERFILE" +grep -Fq 'kars-channel-feishu-ready' "$RUNTIME_DOCKERFILE" + +mkdir -p "$TEST_ROOT/bin" +cat > "$TEST_ROOT/bin/openclaw" <<'EOF' +#!/bin/bash +printf '%s' "$OPENCLAW_TEST_STATUS" +EOF +chmod +x "$TEST_ROOT/bin/openclaw" +export PATH="$TEST_ROOT/bin:$PATH" +export OPENCLAW_TEST_STATUS='{"gatewayReachable":true,"channelAccounts":{"feishu":[{"enabled":true,"configured":true,"running":true,"lastError":null}]}}' +bash "$SCRIPT_DIR/kars-channel-feishu-ready" +export OPENCLAW_TEST_STATUS='{"channelAccounts":{"feishu":[{"enabled":true,"configured":true,"running":true,"lastError":null}]}}' +if bash "$SCRIPT_DIR/kars-channel-feishu-ready"; then + echo "expected missing gateway reachability to fail closed" >&2 + exit 1 +fi +export OPENCLAW_TEST_STATUS='{"gatewayReachable":false,"configOnly":true,"configuredChannels":["feishu"]}' +if bash "$SCRIPT_DIR/kars-channel-feishu-ready"; then + echo "expected config-only OpenClaw status to be unready" >&2 + exit 1 +fi + +node - "$SCRIPT_DIR/patch-feishu-proxy.cjs" <<'EOF' +const { sanitizeFeishuAxiosError } = require(process.argv[2]); +const sentinels = ["secret-value", "Bearer credential", "cli_private", "event-body"]; +const error = new Error(sentinels[0]); +error.code = "ECONNRESET"; +error.config = { + url: `https://example.invalid/${sentinels[2]}`, + headers: { Authorization: sentinels[1] }, + data: sentinels[3], +}; +error.response = { status: 403, data: sentinels[3], headers: { cookie: sentinels[0] } }; +error.cause = new Error(sentinels[0]); +const serialized = JSON.stringify(sanitizeFeishuAxiosError(error)) + + String(sanitizeFeishuAxiosError(error).stack); +if (sentinels.some((sentinel) => serialized.includes(sentinel))) { + throw new Error("sanitized Feishu Axios error retained sensitive fields"); +} +EOF +export OPENCLAW_TEST_STATUS='{"gatewayReachable":true,"channelAccounts":{"feishu":[{"enabled":true,"configured":true,"running":true,"lastError":"connection failed"}]}}' +if bash "$SCRIPT_DIR/kars-channel-feishu-ready"; then + echo "expected an OpenClaw account with lastError to be unready" >&2 + exit 1 +fi + +# shellcheck source=entrypoint.sh +source "$SCRIPT_DIR/entrypoint.sh" + +mkdir -p "$TEST_ROOT/plugin-stage/npm/node_modules/@openclaw/feishu" +mkdir -p "$TEST_ROOT/plugin-stage/plugins" +printf '{"plugins":{"feishu":{"source":"npm"}}}' > "$TEST_ROOT/plugin-stage/plugins/installs.json" +export KARS_FEISHU_PLUGIN_STAGE="$TEST_ROOT/plugin-stage" +export OPENCLAW_DIR="$TEST_ROOT/openclaw-state" + +reset_config() { + PLUGINS_LIST='"kars"' + PLUGINS_ENTRIES='"kars": { "enabled": true }' + CHANNELS_CONFIG="" + unset FEISHU_APP_ID FEISHU_APP_SECRET FEISHU_DOMAIN FEISHU_CONNECTION_MODE + unset FEISHU_DM_POLICY FEISHU_ALLOW_FROM FEISHU_GROUP_POLICY + unset FEISHU_GROUP_ALLOW_FROM FEISHU_REQUIRE_MENTION +} + +reset_config +export FEISHU_APP_ID='cli_test' +export FEISHU_APP_SECRET='secret-with-"quote' +export FEISHU_DOMAIN='feishu' +export FEISHU_CONNECTION_MODE='websocket' +export FEISHU_DM_POLICY='pairing' +export FEISHU_ALLOW_FROM='ou_user1,ou_user2' +export FEISHU_GROUP_POLICY='allowlist' +export FEISHU_GROUP_ALLOW_FROM='oc_group1,oc_group2' +export FEISHU_REQUIRE_MENTION='true' +append_feishu_channel_config + +CONFIG=$(printf '{"plugins":{"allow":[%s],"entries":{%s}},"channels":{%s}}' \ + "$PLUGINS_LIST" "$PLUGINS_ENTRIES" "$CHANNELS_CONFIG") +printf '%s' "$CONFIG" | jq -e ' + .plugins.allow == ["kars", "feishu"] and + .plugins.entries.feishu.enabled == true and + .channels.feishu.appId == "cli_test" and + .channels.feishu.appSecret == "secret-with-\"quote" and + .channels.feishu.domain == "feishu" and + .channels.feishu.connectionMode == "websocket" and + .channels.feishu.dmPolicy == "pairing" and + .channels.feishu.allowFrom == ["ou_user1", "ou_user2"] and + .channels.feishu.groupPolicy == "allowlist" and + .channels.feishu.groupAllowFrom == ["oc_group1", "oc_group2"] and + .channels.feishu.requireMention == true +' >/dev/null + +reset_config +export FEISHU_APP_ID='cli_partial' +export FEISHU_CONNECTION_MODE='websocket' +if (append_feishu_channel_config >/dev/null 2>&1); then + echo "expected partial Feishu credentials to fail" >&2 + exit 1 +fi + +reset_config +export FEISHU_APP_ID='cli_stale' +export FEISHU_APP_SECRET='stale-secret' +append_feishu_channel_config +if [ -n "$CHANNELS_CONFIG" ]; then + echo "expected stale Feishu credentials without typed policy to stay disabled" >&2 + exit 1 +fi + +reset_config +export FEISHU_APP_ID='cli_test' +export FEISHU_APP_SECRET='secret' +export FEISHU_CONNECTION_MODE='websocket' +export KARS_FEISHU_PLUGIN_STAGE="$TEST_ROOT/missing-plugins" +if (append_feishu_channel_config >/dev/null 2>&1); then + echo "expected a missing Feishu plugin to fail" >&2 + exit 1 +fi + +echo "OpenClaw Feishu channel tests passed" \ No newline at end of file From 0b0ea84b8d8d5b7d5d9767e39a99857060890d91 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 10 Aug 2026 14:00:09 +0000 Subject: [PATCH 5/5] fix(hermes): tunnel Feishu WebSocket through proxy --- .../hermes/patch-hermes-feishu-policy.py | 91 ++++++++++++++++++- sandbox-images/hermes/testM_feishu_channel.sh | 87 ++++++++++++++++++ 2 files changed, 176 insertions(+), 2 deletions(-) diff --git a/sandbox-images/hermes/patch-hermes-feishu-policy.py b/sandbox-images/hermes/patch-hermes-feishu-policy.py index b2f694bc0..7f68d671b 100644 --- a/sandbox-images/hermes/patch-hermes-feishu-policy.py +++ b/sandbox-images/hermes/patch-hermes-feishu-policy.py @@ -6,7 +6,11 @@ from __future__ import annotations import importlib.util +import inspect +import os +import socket from pathlib import Path +from urllib.parse import urlparse def replace_once(source: str, old: str, new: str) -> str: @@ -15,6 +19,62 @@ def replace_once(source: str, old: str, new: str) -> str: return source.replace(old, new) +def _open_kars_proxy_tunnel(target_url: str): + target = urlparse(target_url) + if target.scheme.lower() != "wss": + raise RuntimeError("Feishu WebSocket target must use wss") + if target.username or target.password: + raise RuntimeError("Feishu WebSocket target userinfo is unsupported") + if not target.hostname: + raise RuntimeError("Feishu WebSocket target has no hostname") + + target_host = target.hostname.encode("idna").decode("ascii") + target_port = target.port or 443 + authority = f"{target_host}:{target_port}" + request = ( + f"CONNECT {authority} HTTP/1.1\r\n" + f"Host: {authority}\r\n" + "Proxy-Connection: Keep-Alive\r\n\r\n" + ).encode("ascii") + + proxy_url = os.getenv("HTTPS_PROXY") or os.getenv("https_proxy") + if not proxy_url: + return None + proxy = urlparse(proxy_url) + if proxy.scheme.lower() != "http" or not proxy.hostname: + raise RuntimeError("Feishu WebSocket requires an HTTP CONNECT proxy") + if proxy.username or proxy.password: + raise RuntimeError("authenticated Feishu proxy URLs are unsupported") + proxy_host = proxy.hostname.encode("idna").decode("ascii") + + proxy_socket = socket.create_connection((proxy_host, proxy.port or 80), timeout=10) + try: + proxy_socket.sendall(request) + response = bytearray() + while b"\r\n\r\n" not in response: + chunk = proxy_socket.recv(4096) + if not chunk: + raise RuntimeError("Feishu proxy closed during CONNECT") + response.extend(chunk) + if len(response) > 65536: + raise RuntimeError("Feishu proxy CONNECT response is too large") + status_line = bytes(response).split(b"\r\n", 1)[0] + status_parts = status_line.split(b" ", 2) + if ( + len(status_parts) < 2 + or status_parts[0] not in {b"HTTP/1.0", b"HTTP/1.1"} + or status_parts[1] != b"200" + ): + raise RuntimeError( + f"Feishu proxy CONNECT failed: {status_line.decode('ascii', 'replace')}" + ) + proxy_socket.settimeout(None) + return proxy_socket + except Exception: + proxy_socket.close() + raise + + def main() -> None: spec = importlib.util.find_spec("gateway.platforms.feishu") if spec is None or spec.origin is None: @@ -117,8 +177,35 @@ def main() -> None: sdk_source = replace_once( sdk_source, "import time\nfrom urllib.parse import urlparse, parse_qs\n", - "import time\nfrom pathlib import Path\nfrom urllib.parse import urlparse, parse_qs\n\n" - '_KARS_READY_PATH = Path("/tmp/kars-channel-feishu-ready")\n', + "import time\n" + "import os\n" + "import socket\n" + "from pathlib import Path\n" + "from urllib.parse import urlparse, parse_qs\n\n" + '_KARS_READY_PATH = Path("/tmp/kars-channel-feishu-ready")\n\n' + + inspect.getsource(_open_kars_proxy_tunnel) + + "\n", + ) + sdk_source = replace_once( + sdk_source, + " conn = await websockets.connect(conn_url)\n", + " proxy_socket = await asyncio.to_thread(_open_kars_proxy_tunnel, conn_url)\n" + " try:\n" + " conn = await websockets.connect(conn_url, sock=proxy_socket) if proxy_socket else await websockets.connect(conn_url)\n" + " except Exception:\n" + " if proxy_socket is not None:\n" + " proxy_socket.close()\n" + " raise\n", + ) + sdk_source = replace_once( + sdk_source, + ' logger.info(self._fmt_log("connected to {}", conn_url))\n', + ' logger.info(self._fmt_log("connected to Feishu WebSocket host {}", u.hostname or "unknown"))\n', + ) + sdk_source = replace_once( + sdk_source, + ' logger.info(self._fmt_log("disconnected to {}", self._conn_url))\n', + ' logger.info(self._fmt_log("disconnected from Feishu WebSocket host {}", urlparse(self._conn_url).hostname or "unknown"))\n', ) sdk_source = replace_once( sdk_source, diff --git a/sandbox-images/hermes/testM_feishu_channel.sh b/sandbox-images/hermes/testM_feishu_channel.sh index a5de1fa12..55d95a36c 100644 --- a/sandbox-images/hermes/testM_feishu_channel.sh +++ b/sandbox-images/hermes/testM_feishu_channel.sh @@ -15,6 +15,93 @@ grep -Fq '"ws" / "client.py"' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" grep -Fq '_KARS_READY_PATH.touch' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" grep -Fq '_adapter_dm_policy' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" grep -Fq 'pairing_store.is_approved' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq '_open_kars_proxy_tunnel' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq 'sock=proxy_socket' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq 'connected to Feishu WebSocket host' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" +grep -Fq 'disconnected from Feishu WebSocket host' "$SCRIPT_DIR/patch-hermes-feishu-policy.py" + +PATCH_PATH="$SCRIPT_DIR/patch-hermes-feishu-policy.py" python3 - <<'PY' +import importlib.util +import os +import socket +import threading + +spec = importlib.util.spec_from_file_location("kars_hermes_patch", os.environ["PATCH_PATH"]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def run_proxy(status_line): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + received = [] + closed = [] + + def serve(): + connection, _ = listener.accept() + request = bytearray() + while b"\r\n\r\n" not in request: + request.extend(connection.recv(4096)) + received.append(bytes(request)) + connection.sendall(status_line + b"\r\n\r\n") + closed.append(connection.recv(1) == b"") + connection.close() + + thread = threading.Thread(target=serve) + thread.start() + os.environ["HTTPS_PROXY"] = f"http://127.0.0.1:{port}" + return listener, thread, received, closed + + +listener, thread, received, _ = run_proxy(b"HTTP/1.1 200 Connection Established") +tunnel = module._open_kars_proxy_tunnel("wss://open.feishu.cn/callback/ws") +assert received[0] == ( + b"CONNECT open.feishu.cn:443 HTTP/1.1\r\n" + b"Host: open.feishu.cn:443\r\n" + b"Proxy-Connection: Keep-Alive\r\n\r\n" +) +tunnel.close() +thread.join(timeout=2) +listener.close() +assert not thread.is_alive() + +for malformed in (b"HTTP/1.1 2000 Invalid", b"HTTP/1.1 200OK"): + listener, thread, _, closed = run_proxy(malformed) + try: + module._open_kars_proxy_tunnel("wss://open.feishu.cn/callback/ws") + raise AssertionError("malformed CONNECT status was accepted") + except RuntimeError as error: + assert "CONNECT failed" in str(error) + thread.join(timeout=2) + listener.close() + assert closed == [True] + assert not thread.is_alive() + +os.environ["HTTPS_PROXY"] = "http://127.0.0.1:1" +for invalid_target in ( + "ws://open.feishu.cn/callback/ws", + "wss://user:password@open.feishu.cn/callback/ws", +): + try: + module._open_kars_proxy_tunnel(invalid_target) + raise AssertionError("invalid WebSocket target was accepted") + except RuntimeError: + pass + +os.environ.pop("HTTPS_PROXY", None) +os.environ.pop("https_proxy", None) +for invalid_target in ( + "ws://open.feishu.cn/callback/ws", + "wss://user:password@open.feishu.cn/callback/ws", +): + try: + module._open_kars_proxy_tunnel(invalid_target) + raise AssertionError("invalid direct WebSocket target was accepted") + except RuntimeError: + pass +PY READY_MARKER=$(mktemp) rm -f "$READY_MARKER"