Skip to content

feat(controller): add persistent sandbox workspaces #494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
掌柜的 (nikawang) wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(controller): add persistent sandbox workspaces #494

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90d208f
feat(controller): add persistent sandbox workspaces
nikawang Aug 8, 2026
d3d45c8
docs: align workspace spec with implementation
nikawang Aug 9, 2026
92bb3be
feat(runtime): generalize persistent workspace support
nikawang Aug 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 127 additions & 8 deletions cli/src/commands/add.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
// Licensed under the MIT License.

import { describe, it, expect } from "vitest";
import { validateRuntimeSpecificAddFlags } from "./add.js";
import { buildRuntimeBlock, type RuntimeKind } from "../runtime.js";
import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js";

/**
* Tests for the `add` command's sandbox manifest generation logic.
Expand Down Expand Up @@ -36,6 +39,13 @@ interface AddOptions {
openaiApiKey?: string;
learnEgress: boolean;
skills?: string;
workspaceStorage?: string;
workspaceStorageClass?: string;
workspaceExistingClaim?: string;
workspaceRetainPolicy?: "Retain" | "Delete";
workspaceBootstrap?: string;
workspaceOverwrite?: "IfMissing" | "Always";
runtimeKind?: RuntimeKind;
}

function defaultOptions(overrides: Partial<AddOptions> = {}): AddOptions {
Expand All @@ -54,19 +64,21 @@ function defaultOptions(overrides: Partial<AddOptions> = {}): AddOptions {

/** Build the KarsSandbox manifest object (mirrors add.ts action logic). */
function buildSandboxManifest(name: string, options: AddOptions) {
const runtimeKind = options.runtimeKind ?? "OpenClaw";
const sandbox: Record<string, unknown> = {
apiVersion: "kars.azure.com/v1alpha1",
kind: "KarsSandbox",
metadata: { name, namespace: "kars-system" },
spec: {
runtime: {
kind: "OpenClaw",
openclaw: {
version: "2026.3.13",
...(options.image ? { image: options.image } : {}),
config: { agent: { model: `azure/${options.model}` } },
},
},
runtime: buildRuntimeBlock({
kind: runtimeKind,
openclawVersion: "2026.3.13",
model: options.model,
image: options.image,
byoImage: runtimeKind === "BYO" ? "example.invalid/byo:latest" : undefined,
byoContractVersion: "v1",
mafLanguage: "python",
}),
sandbox: {
isolation: options.isolation,
seccompProfile: options.isolation === "standard" ? "RuntimeDefault" : "kars-strict",
Expand Down Expand Up @@ -113,6 +125,19 @@ function buildSandboxManifest(name: string, options: AddOptions) {
np.egressMode = "Learn";
}

const storage = buildWorkspaceStorageSpec(options);
if (storage) {
(sandbox.spec as Record<string, unknown>).storage = storage;
}

if (options.workspaceBootstrap) {
const runtime = (sandbox.spec as any).runtime;
runtime.openclaw.workspace = {
bootstrapConfigMapRef: { name: options.workspaceBootstrap },
overwritePolicy: options.workspaceOverwrite ?? "IfMissing",
};
}

return sandbox;
}

Expand Down Expand Up @@ -180,6 +205,47 @@ describe("KarsSandbox manifest generation", () => {
});
});

it("configures a dynamically provisioned workspace PVC", () => {
const manifest = buildSandboxManifest(
"my-agent",
defaultOptions({
workspaceStorage: "20Gi",
workspaceStorageClass: "managed-csi",
workspaceRetainPolicy: "Retain",
}),
);
expect((manifest.spec as any).storage.workspace).toEqual({
size: "20Gi",
storageClassName: "managed-csi",
accessModes: ["ReadWriteOnce"],
retainPolicy: "Retain",
});
});

it("references an existing workspace claim without dynamic fields", () => {
const manifest = buildSandboxManifest(
"my-agent",
defaultOptions({ workspaceExistingClaim: "restored-workspace" }),
);
expect((manifest.spec as any).storage.workspace).toEqual({
existingClaim: "restored-workspace",
});
});

it("configures OpenClaw workspace bootstrap", () => {
const manifest = buildSandboxManifest(
"my-agent",
defaultOptions({
workspaceBootstrap: "my-agent-workspace",
workspaceOverwrite: "Always",
}),
);
expect((manifest.spec as any).runtime.openclaw.workspace).toEqual({
bootstrapConfigMapRef: { name: "my-agent-workspace" },
overwritePolicy: "Always",
});
});

it("uses default model gpt-4.1 with azure/ prefix", () => {
const manifest = buildSandboxManifest("a", defaultOptions());
const spec = manifest.spec as any;
Expand Down Expand Up @@ -308,6 +374,59 @@ describe("KarsSandbox manifest generation", () => {
expect(spec.sandbox.allowPrivilegeEscalation).toBe(false);
expect(spec.sandbox.writablePaths).toEqual(["/sandbox", "/tmp"]);
});

it.each([
"OpenClaw",
"Hermes",
"OpenAIAgents",
"MicrosoftAgentFramework",
"LangGraph",
"Anthropic",
"PydanticAi",
"BYO",
] satisfies RuntimeKind[])("configures workspace storage for %s", (runtimeKind) => {
const manifest = buildSandboxManifest(
"persistent-agent",
defaultOptions({ runtimeKind, workspaceStorage: "10Gi" }),
);
const spec = manifest.spec as any;
expect(spec.runtime.kind).toBe(runtimeKind);
expect(spec.storage.workspace).toEqual({
size: "10Gi",
accessModes: ["ReadWriteOnce"],
retainPolicy: "Retain",
});
});
});

describe("runtime-specific add flag validation", () => {
it("allows channels for Hermes", () => {
expect(
validateRuntimeSpecificAddFlags("Hermes", {
channels: "telegram",
telegramToken: "test-token",
}),
).toEqual([]);
});

it("rejects channels for runtimes without channel adapters", () => {
expect(
validateRuntimeSpecificAddFlags("LangGraph", { channels: "telegram" }),
).toEqual([
"--channels is only valid with --runtime openclaw or --runtime hermes.",
]);
});

it("keeps skills, plugins, and image overrides OpenClaw-only", () => {
expect(
validateRuntimeSpecificAddFlags("Hermes", {
skills: "browser",
image: "example.invalid/custom:latest",
}),
).toEqual([
"--skills, --image are only valid with --runtime openclaw.",
]);
});
});

describe("channel and plugin secret generation", () => {
Expand Down
144 changes: 121 additions & 23 deletions cli/src/commands/add.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,56 @@ import {
inferenceRefName,
toolPolicyRefName,
} from "../refs.js";
import type { RuntimeKind } from "../runtime.js";
import { buildWorkspaceStorageSpec } from "../lib/workspace-storage.js";

export function validateRuntimeSpecificAddFlags(
runtimeKind: RuntimeKind,
options: Record<string, unknown>,
): string[] {
const errors: string[] = [];
const channelFlags: Array<[string, unknown]> = [
["--channels", options.channels],
["--telegram-token", options.telegramToken],
["--telegram-allow-from", options.telegramAllowFrom],
["--slack-token", options.slackToken],
["--discord-token", options.discordToken],
];
if (runtimeKind !== "OpenClaw" && runtimeKind !== "Hermes") {
const used = channelFlags
.filter(([, value]) => value !== undefined && value !== "" && value !== false)
.map(([flag]) => flag);
if (used.length > 0) {
errors.push(
`${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` +
`--runtime openclaw or --runtime hermes.`,
);
}
}

const openClawOnlyFlags: Array<[string, unknown]> = [
["--skills", options.skills],
["--brave-api-key", options.braveApiKey],
["--tavily-api-key", options.tavilyApiKey],
["--exa-api-key", options.exaApiKey],
["--firecrawl-api-key", options.firecrawlApiKey],
["--perplexity-api-key", options.perplexityApiKey],
["--openai-api-key", options.openaiApiKey],
["--image", options.image],
];
if (runtimeKind !== "OpenClaw") {
const used = openClawOnlyFlags
.filter(([, value]) => value !== undefined && value !== "" && value !== false)
.map(([flag]) => flag);
if (used.length > 0) {
errors.push(
`${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with ` +
`--runtime openclaw.`,
);
}
}
return errors;
}

export function addCommand(): Command {
const cmd = new Command("add");
Expand All @@ -26,6 +76,14 @@ export function addCommand(): Command {
.option("--isolation <level>", "Isolation level: standard | enhanced | confidential", "enhanced")
.option("--image <image>", "Custom sandbox image (default: from Helm values; OpenClaw runtime only)")

// ── Workspace storage (all runtimes; bootstrap is OpenClaw-only) ───
.option("--workspace-storage <size>", "Create a persistent workspace PVC, e.g. 10Gi")
.option("--workspace-storage-class <name>", "StorageClass for a generated workspace PVC")
.option("--workspace-existing-claim <name>", "Use an existing PVC in the generated sandbox namespace")
.option("--workspace-retain-policy <policy>", "Generated PVC deletion policy: Retain | Delete", "Retain")
.option("--workspace-bootstrap <configmap>", "[OpenClaw only] ConfigMap containing workspace bootstrap files")
.option("--workspace-overwrite <policy>", "[OpenClaw only] Bootstrap policy: IfMissing | Always", "IfMissing")

// ── Inference budget (all runtimes) ────────────────────────────────
.option("--token-budget-daily <tokens>", "Daily token budget (0 = unlimited)", "0")
.option("--token-budget-per-request <tokens>", "Per-request token limit (0 = unlimited)", "0")
Expand Down Expand Up @@ -72,6 +130,7 @@ export function addCommand(): Command {
.addHelpText("after", `
Flag groups (see --help for details):
Core: --runtime, --model, --isolation, --image
Workspace: --workspace-storage, --workspace-existing-claim, --workspace-bootstrap
Inference budget: --token-budget-*
Governance / net: --governance, --trust-threshold, --policy-profile, --learn-egress
Foundry agent: --agent-instructions, --agent-tools
Expand All @@ -93,29 +152,10 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs.
// Validate runtime-specific flag combinations before doing any work.
// Reject incompatible flags up-front with a clear, actionable error
// — better than silently ignoring a user's intent.
const openClawOnlyFlags: Array<[string, unknown]> = [
["--channels", options.channels],
["--telegram-token", options.telegramToken],
["--telegram-allow-from", options.telegramAllowFrom],
["--slack-token", options.slackToken],
["--discord-token", options.discordToken],
["--skills", options.skills],
["--brave-api-key", options.braveApiKey],
["--tavily-api-key", options.tavilyApiKey],
["--exa-api-key", options.exaApiKey],
["--firecrawl-api-key", options.firecrawlApiKey],
["--perplexity-api-key", options.perplexityApiKey],
["--openai-api-key", options.openaiApiKey],
["--image", options.image],
];
if (runtimeKind !== "OpenClaw") {
const used = openClawOnlyFlags.filter(([, v]) => v !== undefined && v !== "" && v !== false).map(([f]) => f);
if (used.length > 0) {
console.error(chalk.red(`\n Error: ${used.join(", ")} ${used.length === 1 ? "is" : "are"} only valid with --runtime openclaw.`));
console.error(chalk.dim(` Channels, skills, and plugin API keys are OpenClaw-specific entrypoint features.`));
console.error(chalk.dim(` For ${options.runtime}, configure equivalents inside the agent's own code.\n`));
process.exit(1);
}
const runtimeFlagErrors = validateRuntimeSpecificAddFlags(runtimeKind, options);
if (runtimeFlagErrors.length > 0) {
console.error(chalk.red(`\n Error: ${runtimeFlagErrors.join(" ")}\n`));
process.exit(1);
}
if (runtimeKind !== "BYO" && (options.byoImage || (options.byoContractVersion && options.byoContractVersion !== "v1"))) {
console.error(chalk.red(`\n Error: --byo-image / --byo-contract-version are only valid with --runtime byo.\n`));
Expand All @@ -129,6 +169,40 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs.
console.error(chalk.red(`\n Error: --maf-language is only valid with --runtime microsoft-agent-framework.\n`));
process.exit(1);
}
if (options.workspaceStorage && options.workspaceExistingClaim) {
console.error(chalk.red(`\n Error: --workspace-storage and --workspace-existing-claim are mutually exclusive.\n`));
process.exit(1);
}
if (options.workspaceStorageClass && !options.workspaceStorage) {
console.error(chalk.red(`\n Error: --workspace-storage-class requires --workspace-storage <size>.\n`));
process.exit(1);
}
if (options.workspaceExistingClaim && options.workspaceRetainPolicy !== "Retain") {
console.error(chalk.red(`\n Error: --workspace-retain-policy applies only to generated PVCs; existing claims are always externally managed.\n`));
process.exit(1);
}
if (!(["Retain", "Delete"] as const).includes(options.workspaceRetainPolicy)) {
console.error(chalk.red(`\n Error: --workspace-retain-policy must be Retain or Delete.\n`));
process.exit(1);
}
if (runtimeKind !== "OpenClaw" && options.workspaceBootstrap) {
console.error(chalk.red(`\n Error: --workspace-bootstrap is only valid with --runtime openclaw.\n`));
process.exit(1);
}
if (!(["IfMissing", "Always"] as const).includes(options.workspaceOverwrite)) {
console.error(chalk.red(`\n Error: --workspace-overwrite must be IfMissing or Always.\n`));
process.exit(1);
}
if (!options.workspaceBootstrap && options.workspaceOverwrite !== "IfMissing") {
console.error(chalk.red(`\n Error: --workspace-overwrite requires --workspace-bootstrap <configmap>.\n`));
process.exit(1);
}
if (options.workspaceStorage && options.workspaceRetainPolicy === "Delete") {
console.log(chalk.yellow(" ⚠ Workspace retain policy is Delete: deleting the KarsSandbox will delete its PVC and data."));
}
if (!options.workspaceStorage && !options.workspaceExistingClaim) {
console.log(chalk.yellow(" ⚠ Workspace is ephemeral (emptyDir): Pod recreation or suspension deletes sessions and files."));
}

const runtimeBlock = buildRuntimeBlock({
kind: runtimeKind,
Expand All @@ -139,6 +213,13 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs.
byoContractVersion: options.byoContractVersion,
mafLanguage: options.mafLanguage as "python" | "dotnet",
});
if (options.workspaceBootstrap) {
const openclaw = runtimeBlock.openclaw as Record<string, unknown>;
openclaw.workspace = {
bootstrapConfigMapRef: { name: options.workspaceBootstrap },
overwritePolicy: options.workspaceOverwrite,
};
}

const sandbox: Record<string, unknown> = {
apiVersion: "kars.azure.com/v1alpha1",
Expand Down Expand Up @@ -174,6 +255,11 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs.
},
};

const storage = buildWorkspaceStorageSpec(options);
if (storage) {
(sandbox.spec as Record<string, unknown>).storage = storage;
}

// Add Foundry agent config if provided
if (options.agentInstructions || options.agentTools) {
const agentSpec: Record<string, unknown> = {};
Expand Down Expand Up @@ -591,6 +677,18 @@ generating per-sandbox AGT ToolPolicy / TrustGraph CRs.
console.log(chalk.dim(` Namespace: ${namespace}`));
console.log(chalk.dim(` Model: ${options.model}`));
console.log(chalk.dim(` Isolation: ${options.isolation}`));
if (options.workspaceExistingClaim) {
console.log(chalk.dim(` Workspace: existing PVC ${options.workspaceExistingClaim} (externally managed)`));
} else if (options.workspaceStorage) {
console.log(chalk.dim(
` Workspace: ${name}-workspace (${options.workspaceStorage}, ${options.workspaceStorageClass || "default StorageClass"}, ${options.workspaceRetainPolicy})`,
));
} else {
console.log(chalk.yellow(" Workspace: ephemeral emptyDir"));
}
if (options.workspaceBootstrap) {
console.log(chalk.dim(` Bootstrap: ${options.workspaceBootstrap} (${options.workspaceOverwrite})`));
}
if (options.channels) {
console.log(chalk.dim(` Channels: ${options.channels}`));
}
Expand Down
Loading