From d7f181e9dccb20485f6093e5cc0541c51650962d Mon Sep 17 00:00:00 2001 From: Nitin Khanna Date: Tue, 2 Jun 2026 07:10:09 -0700 Subject: [PATCH] fix(oss): finish playground copilot workflow --- bun.lock | 26 + .../editor-oss/src/agent/CommandsRegistry.ts | 58 ++ .../src/agent/handlers/AssetHandlers.test.ts | 104 ++ .../src/agent/handlers/AssetHandlers.ts | 231 +++++ .../src/agent/handlers/LambdaHandlers.test.ts | 103 ++ .../src/agent/handlers/LambdaHandlers.ts | 128 +++ .../src/agent/script-tool/aliases.ts | 20 + .../script-tool/commandContractMatrix.ts | 28 + .../editor-oss/src/ai/HttpAIBackend.test.ts | 16 + .../editor-oss/src/ai/HttpAIBackend.ts | 13 + .../context/OssAssetRegistryContext.test.tsx | 56 ++ .../src/context/OssAssetRegistryContext.tsx | 27 + .../packages/editor-oss/src/context/index.ts | 2 + .../src/copilot/DirectCopilotProvider.test.ts | 386 ++++++++ .../src/copilot/DirectCopilotProvider.ts | 558 ++++++++--- .../packages/editor-oss/src/copilot/index.ts | 13 + .../src/copilot/playgroundCopilotKeys.test.ts | 90 ++ .../src/copilot/playgroundCopilotKeys.ts | 143 ++- .../src/copilot/playgroundLLMClient.test.ts | 135 +++ .../src/copilot/playgroundLLMClient.ts | 119 +++ .../copilot/playgroundStemscriptKnowledge.ts | 25 + .../copilot/playgroundStemscriptPlan.test.ts | 124 +++ .../src/copilot/playgroundStemscriptPlan.ts | 201 ++++ .../AiCopilot/AiCopilot.sessionMode.test.tsx | 346 +++++++ .../assets/v2/AiCopilot/AiCopilot.styles.ts | 566 ++--------- .../editor/assets/v2/AiCopilot/AiCopilot.tsx | 900 +++--------------- .../assets/v2/AiCopilot/AiKeysModal.test.tsx | 62 ++ .../assets/v2/AiCopilot/AiKeysModal.tsx | 103 +- .../chatHistory/ChatHistory.styles.ts | 183 ---- .../v2/AiCopilot/chatHistory/ChatHistory.tsx | 174 ---- .../v2/AiCopilot/copilotWorkspaceEntry.ts | 6 +- .../dashboardCopilotBootstrap.test.ts | 2 +- .../assets/v2/AiCopilot/utils/prompt.ts | 154 +++ .../workflow/CopilotActivityFeed.tsx | 72 -- .../workflow/CopilotConfirmationCard.test.ts | 48 - .../workflow/CopilotConfirmationCard.tsx | 131 --- .../workflow/CopilotVersionTimeline.tsx | 335 ------- .../copilotVersionTimelineModel.test.ts | 153 --- .../workflow/copilotVersionTimelineModel.ts | 130 --- .../SceneListItem.playground.test.tsx | 125 +++ .../SceneList/SceneListItem.tsx | 21 +- .../BYOKKeysPanel/BYOKKeysPanel.test.tsx | 95 ++ .../BYOKKeysPanel/BYOKKeysPanel.tsx | 40 +- .../editor-oss/src/utils/CSPMetaTag.test.tsx | 21 + .../editor-oss/src/utils/CSPMetaTag.tsx | 11 + .../editor-oss/src/v2/pages/Create/Create.tsx | 3 +- .../adapters/remote-go/asset/index.test.ts | 56 +- .../src/adapters/remote-go/asset/index.ts | 38 +- .../remote-go/copilotHistory/index.test.ts | 48 + .../remote-go/copilotHistory/index.ts | 22 + .../remote-go/scene/thumbnail.test.ts | 33 + .../src/adapters/remote-go/scene/thumbnail.ts | 3 + .../src/adapters/remote-go/scene/v2.test.ts | 27 +- .../src/adapters/remote-go/scene/v2.ts | 14 + client/packages/shared/src/AppContainer.tsx | 3 + .../shared/src/PublicAppContainer.tsx | 41 +- .../shared/src/PublicAppContainerLite.tsx | 17 +- .../src/context/OssAssetRegistryContext.tsx | 1 + .../chatHistory/ChatHistory.styles.ts | 1 - .../v2/AiCopilot/chatHistory/ChatHistory.tsx | 1 - .../workflow/CopilotActivityFeed.tsx | 1 - .../workflow/CopilotConfirmationCard.tsx | 1 - .../workflow/CopilotVersionTimeline.tsx | 1 - .../workflow/copilotVersionTimelineModel.ts | 1 - docs/byok.md | 9 + ...5-20-playground-direct-model-generation.md | 6 +- ...-05-20-playground-mode-and-copilot-keys.md | 15 +- package.json | 4 + 68 files changed, 3881 insertions(+), 2749 deletions(-) create mode 100644 client/packages/editor-oss/src/agent/handlers/AssetHandlers.test.ts create mode 100644 client/packages/editor-oss/src/agent/handlers/AssetHandlers.ts create mode 100644 client/packages/editor-oss/src/agent/handlers/LambdaHandlers.test.ts create mode 100644 client/packages/editor-oss/src/agent/handlers/LambdaHandlers.ts create mode 100644 client/packages/editor-oss/src/context/OssAssetRegistryContext.test.tsx create mode 100644 client/packages/editor-oss/src/context/OssAssetRegistryContext.tsx create mode 100644 client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundLLMClient.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts create mode 100644 client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts create mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx create mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiKeysModal.test.tsx delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/chatHistory/ChatHistory.styles.ts delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/chatHistory/ChatHistory.tsx delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/CopilotActivityFeed.tsx delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/CopilotConfirmationCard.test.ts delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/CopilotConfirmationCard.tsx delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/CopilotVersionTimeline.tsx delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/copilotVersionTimelineModel.test.ts delete mode 100644 client/packages/editor-oss/src/editor/assets/v2/AiCopilot/workflow/copilotVersionTimelineModel.ts create mode 100644 client/packages/editor-oss/src/editor/assets/v2/CreateDashboard/SceneList/SceneListItem.playground.test.tsx create mode 100644 client/packages/editor-oss/src/editor/assets/v2/CreateDashboard/SettingsPage/BYOKKeysPanel/BYOKKeysPanel.test.tsx create mode 100644 client/packages/editor-oss/src/utils/CSPMetaTag.test.tsx create mode 100644 client/packages/network/src/adapters/remote-go/copilotHistory/index.test.ts create mode 100644 client/packages/network/src/adapters/remote-go/scene/thumbnail.test.ts create mode 100644 client/packages/shared/src/context/OssAssetRegistryContext.tsx delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/chatHistory/ChatHistory.styles.ts delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/chatHistory/ChatHistory.tsx delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/workflow/CopilotActivityFeed.tsx delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/workflow/CopilotConfirmationCard.tsx delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/workflow/CopilotVersionTimeline.tsx delete mode 100644 client/packages/shared/src/editor/assets/v2/AiCopilot/workflow/copilotVersionTimelineModel.ts diff --git a/bun.lock b/bun.lock index 5bfe2ec2..637769c7 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,9 @@ "name": "stemstudio", "dependencies": { "@agentclientprotocol/sdk": "^0.16.1", + "@ai-sdk/anthropic": "^3.0.58", + "@ai-sdk/google": "^3.0.43", + "@ai-sdk/openai": "^3.0.41", "@capacitor/app": "^8.1.0", "@capacitor/browser": "^8.0.3", "@capacitor/splash-screen": "^8.0.1", @@ -34,6 +37,7 @@ "@tweenjs/tween.js": "^25.0.0", "acorn": "^8.16.0", "acorn-walk": "^8.3.5", + "ai": "^6.0.116", "ammo-debug-drawer": "^1.0.1", "axios": "1.15.2", "boring-avatars": "^2.0.4", @@ -171,6 +175,18 @@ "@agentclientprotocol/sdk": ["@agentclientprotocol/sdk@0.16.1", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-1ad+Sc/0sCtZGHthxxvgEUo5Wsbw16I+aF+YwdiLnPwkZG8KAGUEAPK6LM6Pf69lCyJPt1Aomk1d+8oE3C4ZEw=="], + "@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.81", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-B1JDd9Ugq9R5AgIaW3674lhGCMMYJcPUxnrZh8fzbGojgg4QvHFRv6eZahGQAUsmGHbcf74G9bdSBDLWQGY2GA=="], + + "@ai-sdk/gateway": ["@ai-sdk/gateway@3.0.122", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-U1k2fk7cSH/tS5CZ3ujROiUCOLFwkzb792OqR/Org8Mfm27dKSIdRZG4ZuJUifT8alUWa61IoaRu4foXKlP5TQ=="], + + "@ai-sdk/google": ["@ai-sdk/google@3.0.80", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-5ORbm/yFUPO0MEvZsxBMN0cdKw2+lwU/wVn5KN3KF8Dmk1LughuDuUohMh/7iU/XFTiyB0OvmTW/tdV/J7O9zg=="], + + "@ai-sdk/openai": ["@ai-sdk/openai@3.0.67", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-oAiGC9eWG7IgtdsdS74bOCnAAHarAfTJhWN9x5INwnWPekL802AvF+0I5DvLzIF1MIRmNw4N8mPSL/GUVbX9Mw=="], + + "@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="], + + "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="], + "@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="], "@ampproject/remapping": ["@ampproject/remapping@2.3.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw=="], @@ -585,6 +601,8 @@ "@openapitools/openapi-generator-cli": ["@openapitools/openapi-generator-cli@2.32.0", "", { "dependencies": { "@inquirer/select": "1.3.3", "@nestjs/axios": "4.0.1", "@nestjs/common": "11.1.17", "@nestjs/core": "11.1.18", "@nuxtjs/opencollective": "0.3.2", "axios": "^1.15.0", "chalk": "4.1.2", "commander": "8.3.0", "compare-versions": "6.1.1", "concurrently": "9.2.1", "console.table": "0.10.0", "fs-extra": "11.3.4", "glob": "13.0.6", "proxy-agent": "6.5.0", "reflect-metadata": "0.2.2", "rxjs": "7.8.2", "tslib": "2.8.1" }, "bin": { "openapi-generator-cli": "main.js" } }, "sha512-9HZ3fp3cankdUC89UNsnW+HZFmRUadjjtqOvIIo6/D+bAVs+VJRqyhDy4rT4/cxqcLhXw40njs/vJLj21r60JA=="], + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + "@oxc-project/types": ["@oxc-project/types@0.115.0", "", {}, "sha512-4n91DKnebUS4yjUHl2g3/b2T+IUdCfmoZGhmwsovZCDaJSs+QkVAM+0AqqTxHSsHfeiMuueT75cZaZcT/m0pSw=="], "@pixiv/three-vrm": ["@pixiv/three-vrm@3.5.2", "", { "dependencies": { "@pixiv/three-vrm-core": "3.5.2", "@pixiv/three-vrm-materials-hdr-emissive-multiplier": "3.5.2", "@pixiv/three-vrm-materials-mtoon": "3.5.2", "@pixiv/three-vrm-materials-v0compat": "3.5.2", "@pixiv/three-vrm-node-constraint": "3.5.2", "@pixiv/three-vrm-springbone": "3.5.2" }, "peerDependencies": { "three": ">=0.137" } }, "sha512-YC3T8WZvb18n1uzYS60I9WCc2kOJxWZ/Q0RfklFAg0nD3Dz/kvvEoCYfZj5idnJqOtbLUXZGXZF1KCXKUB2qHg=="], @@ -1021,6 +1039,8 @@ "@use-gesture/react": ["@use-gesture/react@10.3.1", "", { "dependencies": { "@use-gesture/core": "10.3.1" }, "peerDependencies": { "react": ">= 16.8.0" } }, "sha512-Yy19y6O2GJq8f7CHf7L0nxL8bf4PZCPaVOCgJrusOeFHY1LvHgYXnmnXg6N5iwAnbgbZCDjo60SiM6IPJi9C5g=="], + "@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@6.0.1", "", { "dependencies": { "@rolldown/pluginutils": "1.0.0-rc.7" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler"] }, "sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ=="], "@vitest/expect": ["@vitest/expect@4.1.6", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.6", "@vitest/utils": "4.1.6", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-7EHDquPthALSV0jhhjgEW8FXaviMx7rSqu8W6oqCoAuOhKov814P99QDV1pxMA3QPv21YudvJngIhjrNI4opLg=="], @@ -1051,6 +1071,8 @@ "agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="], + "ai": ["ai@6.0.194", "", { "dependencies": { "@ai-sdk/gateway": "3.0.122", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27", "@opentelemetry/api": "^1.9.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-0MkYqrSZZuC1zTECppcaUT0i54aocXpYaUMVue3V8z/weBHCytfO5/CcwZCU80msZpfkbBUKYSSrkZFotEO5wQ=="], + "ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="], "ammo-debug-drawer": ["ammo-debug-drawer@1.0.1", "", { "peerDependencies": { "ammo.js": "*" } }, "sha512-oPvQh5CQKgFQpIuBMMKxT019aYoONOEnBLn91dAg3CrMWdSP/Om8rLWRX0VkAk45SEK/B0ybXB7nsjXsz8QEBw=="], @@ -1569,6 +1591,8 @@ "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], + "eventsource-parser": ["eventsource-parser@3.1.0", "", {}, "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg=="], + "evp_bytestokey": ["evp_bytestokey@1.0.3", "", { "dependencies": { "md5.js": "^1.3.4", "safe-buffer": "^5.1.1" } }, "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA=="], "execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], @@ -1941,6 +1965,8 @@ "json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="], + "json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="], + "json-schema-traverse": ["json-schema-traverse@0.4.1", "", {}, "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="], "json-stable-stringify": ["json-stable-stringify@1.3.0", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.4", "isarray": "^2.0.5", "jsonify": "^0.0.1", "object-keys": "^1.1.1" } }, "sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg=="], diff --git a/client/packages/editor-oss/src/agent/CommandsRegistry.ts b/client/packages/editor-oss/src/agent/CommandsRegistry.ts index 4b311148..2b578491 100644 --- a/client/packages/editor-oss/src/agent/CommandsRegistry.ts +++ b/client/packages/editor-oss/src/agent/CommandsRegistry.ts @@ -1,7 +1,9 @@ import AIWorldController from "../controls/AiWorldController/AiWorldController"; import EngineRuntime from "../EngineRuntime"; import global from "../global"; +import {AssetHandlers} from "./handlers/AssetHandlers"; import {BehaviorHandlers} from "./handlers/BehaviorHandlers"; +import {LambdaHandlers} from "./handlers/LambdaHandlers"; import {LightHandlers} from "./handlers/LightHandlers"; import {ObjectHandlers} from "./handlers/ObjectHandlers"; import {PhysicsHandlers} from "./handlers/PhysicsHandlers"; @@ -27,8 +29,10 @@ export class CommandsRegistry { private aiWorldController = AIWorldController.getInstance(this.engine); // Handler instances + private assetHandlers: AssetHandlers; private objectHandlers: ObjectHandlers; private behaviorHandlers: BehaviorHandlers; + private lambdaHandlers: LambdaHandlers; private prefabHandlers: PrefabHandlers; private physicsHandlers: PhysicsHandlers; private vfxHandlers: VFXHandlers; @@ -37,8 +41,10 @@ export class CommandsRegistry { private taskHandlers: TaskHandlers; constructor(options: CommandsRegistryOptions = {}) { + this.assetHandlers = new AssetHandlers(this.engine); this.objectHandlers = new ObjectHandlers(this.engine, this.aiWorldController); this.behaviorHandlers = new BehaviorHandlers(this.engine); + this.lambdaHandlers = new LambdaHandlers(this.engine); this.prefabHandlers = new PrefabHandlers(this.engine); this.physicsHandlers = new PhysicsHandlers(this.engine); this.vfxHandlers = new VFXHandlers(this.engine); @@ -331,6 +337,54 @@ export class CommandsRegistry { handler: () => Promise.resolve(this.objectHandlers.handleGetPlayer()), }); + this.registerCommand({ + name: SupportedCommands.ListSceneAssets, + description: + "List imported scene/stem assets such as models, behavior packs, lambda packs, script imports, files, media, VFX, and prefabs", + parameters: [ + { + name: "type", + type: "string", + description: + "Optional asset type filter: all, model/models, import/imports/script/scripts, file/files, behavior/behaviors, lambda/lambdas, pack/packs, media, image, audio, video, vfx, prefab/stem", + required: false, + }, + {name: "filter", type: "string", description: "Optional filter by id, name, description, tag, or type", required: false}, + {name: "limit", type: "number", description: "Maximum assets to return (default 80, max 200)", required: false}, + ], + handler: params => this.assetHandlers.handleListSceneAssets(params), + }); + + this.registerCommand({ + name: SupportedCommands.GetSceneAsset, + description: "Get compact metadata for one imported scene/stem asset by assetId, revisionId, or name", + parameters: [ + {name: "assetId", type: "string", description: "Asset id, revision id, or exact name", required: false}, + {name: "name", type: "string", description: "Asset name when assetId is omitted", required: false}, + {name: "type", type: "string", description: "Optional asset type filter", required: false}, + ], + handler: params => this.assetHandlers.handleGetSceneAsset(params), + }); + + this.registerCommand({ + name: SupportedCommands.ListLambdas, + description: "List all available lambdas with optional filtering", + parameters: [ + {name: "filter", type: "string", description: "Optional filter by id or name pattern", required: false}, + ], + handler: params => Promise.resolve(this.lambdaHandlers.handleListLambdas(params)), + }); + + this.registerCommand({ + name: SupportedCommands.GetLambda, + description: "Get detailed information about a specific lambda by ID, optionally including code when available", + parameters: [ + {name: "lambdaId", type: "string", description: "ID of the lambda", required: true}, + {name: "includeCode", type: "boolean", description: "Try to include lambda source code", required: false}, + ], + handler: params => this.lambdaHandlers.handleGetLambda(params), + }); + // ===== MATERIAL & TEXTURE COMMANDS ===== this.registerCommand({ name: SupportedCommands.SetMaterial, @@ -1408,6 +1462,10 @@ export enum SupportedCommands { GetBehaviorSettings = "get_behavior_settings", GetSelectedObject = "get_selected_object", GetPlayer = "get_player", + ListSceneAssets = "list_scene_assets", + GetSceneAsset = "get_scene_asset", + ListLambdas = "list_lambdas", + GetLambda = "get_lambda", SetMaterial = "set_material", SetTexture = "set_texture", SetExternalTexture = "set_external_texture", diff --git a/client/packages/editor-oss/src/agent/handlers/AssetHandlers.test.ts b/client/packages/editor-oss/src/agent/handlers/AssetHandlers.test.ts new file mode 100644 index 00000000..27b2ff1b --- /dev/null +++ b/client/packages/editor-oss/src/agent/handlers/AssetHandlers.test.ts @@ -0,0 +1,104 @@ +import {describe, expect, it, vi} from "vitest"; + +import {AssetHandlers} from "./AssetHandlers"; + +const createHandlers = () => { + const assetSource = { + kind: "scene", + id: "scene-1", + getAssets: vi.fn(async ({types}: {types?: string[]} = {}) => { + const assets = [ + { + id: "model-1", + name: "Kart", + type: "model", + description: "A drivable kart model", + tags: ["vehicle"], + contentType: "model/gltf-binary", + format: "glb", + headRevisionId: "model-rev", + thumbnailUrl: "data:image/png;base64,large", + }, + { + id: "script-1", + name: "math-helpers", + type: "script", + description: "Reusable movement helpers", + tags: ["import"], + headRevisionId: "script-rev", + }, + { + id: "file-1", + name: "level-data.json", + type: "file", + description: "Level data", + headRevisionId: "file-rev", + }, + { + id: "lambda-1", + name: "Patrol Brain", + type: "lambda", + description: "Patrol lambda pack", + headRevisionId: "lambda-rev", + }, + ]; + return { + assets: types?.length ? assets.filter(asset => types.includes(asset.type)) : assets, + }; + }), + }; + const engine = { + editor: {assetSource}, + } as any; + + return {assetSource, handlers: new AssetHandlers(engine)}; +}; + +describe("AssetHandlers", () => { + it("lists compact scene asset metadata by semantic type", async () => { + const {assetSource, handlers} = createHandlers(); + + const result = await handlers.handleListSceneAssets({type: "models"}); + + expect(assetSource.getAssets).toHaveBeenCalledWith({ + types: ["model"], + includeLatestRelease: true, + includeThumbnails: true, + }); + expect(result.status).toBe("success"); + expect(result.data).toEqual(expect.objectContaining({ + assetSource: {kind: "scene", id: "scene-1"}, + total: 1, + assets: [ + expect.objectContaining({ + id: "model-1", + name: "Kart", + type: "model", + hasThumbnail: true, + }), + ], + })); + expect(JSON.stringify(result.data)).not.toContain("data:image/png"); + }); + + it("gets imports, files, and lambda packs by name or id", async () => { + const {handlers} = createHandlers(); + + const importResult = await handlers.handleGetSceneAsset({name: "math-helpers", type: "imports"}); + const fileResult = await handlers.handleGetSceneAsset({assetId: "file-1", type: "files"}); + const lambdaResult = await handlers.handleGetSceneAsset({name: "Patrol Brain", type: "lambdas"}); + + expect(importResult.status).toBe("success"); + expect(importResult.data).toEqual(expect.objectContaining({ + asset: expect.objectContaining({id: "script-1", type: "script"}), + })); + expect(fileResult.status).toBe("success"); + expect(fileResult.data).toEqual(expect.objectContaining({ + asset: expect.objectContaining({id: "file-1", type: "file"}), + })); + expect(lambdaResult.status).toBe("success"); + expect(lambdaResult.data).toEqual(expect.objectContaining({ + asset: expect.objectContaining({id: "lambda-1", type: "lambda"}), + })); + }); +}); diff --git a/client/packages/editor-oss/src/agent/handlers/AssetHandlers.ts b/client/packages/editor-oss/src/agent/handlers/AssetHandlers.ts new file mode 100644 index 00000000..6091248c --- /dev/null +++ b/client/packages/editor-oss/src/agent/handlers/AssetHandlers.ts @@ -0,0 +1,231 @@ +import {AssetType, type Asset} from "@stem/network/api/asset"; + +import EngineRuntime from "../../EngineRuntime"; +import {CommandResult} from "../types/ACPTypes"; + +type SceneAssetType = typeof AssetType[keyof typeof AssetType]; + +type AssetSummary = { + id: string; + name: string; + type: string; + description?: string; + tags?: string[]; + contentType?: string; + format?: string; + revisionId?: string; + headRevisionId?: string; + latestReleaseRevisionId?: string; + hasThumbnail?: boolean; + createTime?: string; + updateTime?: string; +}; + +const ASSET_TYPE_ALIASES: Record = { + all: [], + any: [], + asset: [], + assets: [], + audio: [AssetType.Audio], + behavior: [AssetType.Behavior], + behaviors: [AssetType.Behavior], + file: [AssetType.File], + files: [AssetType.File], + image: [AssetType.Image], + images: [AssetType.Image], + import: [AssetType.Script], + imports: [AssetType.Script], + lambda: [AssetType.Lambda], + lambdas: [AssetType.Lambda], + media: [AssetType.Audio, AssetType.Image, AssetType.Video], + model: [AssetType.Model], + models: [AssetType.Model], + npc: [AssetType.Npc], + npcs: [AssetType.Npc], + pack: [AssetType.Behavior, AssetType.Lambda], + packs: [AssetType.Behavior, AssetType.Lambda], + prefab: [AssetType.Prefab], + prefabs: [AssetType.Prefab], + script: [AssetType.Script], + scripts: [AssetType.Script], + stem: [AssetType.Prefab], + stems: [AssetType.Prefab], + video: [AssetType.Video], + videos: [AssetType.Video], + vfx: [AssetType.Quarks], +}; + +export class AssetHandlers { + constructor(private engine: EngineRuntime) {} + + async handleListSceneAssets({ + filter, + limit, + type, + }: { + filter?: string; + limit?: number; + type?: string; + }): Promise { + try { + const assetSource = this.engine.editor?.assetSource; + if (!assetSource) { + return { + status: "failed", + message: "No active editing context (scene or stem) available", + data: {assets: []}, + }; + } + + const types = resolveAssetTypes(type); + const response = await assetSource.getAssets({ + types, + includeLatestRelease: true, + includeThumbnails: true, + }); + let assets = response?.assets ?? []; + + const normalizedFilter = filter?.trim().toLowerCase(); + if (normalizedFilter && normalizedFilter !== "*") { + assets = assets.filter(asset => { + const haystack = [ + asset.id, + asset.name, + asset.description, + asset.type, + ...(asset.tags ?? []), + ].filter(Boolean).join(" ").toLowerCase(); + return haystack.includes(normalizedFilter); + }); + } + + const normalizedLimit = clampLimit(limit); + const summarized = assets.slice(0, normalizedLimit).map(summarizeAsset); + + return { + status: "success", + message: `Retrieved ${summarized.length}/${assets.length} scene asset(s)`, + data: { + assetSource: {kind: assetSource.kind, id: assetSource.id}, + typeFilter: type || "all", + total: assets.length, + returned: summarized.length, + assets: summarized, + }, + }; + } catch (error) { + return { + status: "failed", + message: `Error listing scene assets: ${error instanceof Error ? error.message : String(error)}`, + data: {assets: []}, + }; + } + } + + async handleGetSceneAsset({ + assetId, + name, + type, + }: { + assetId?: string; + name?: string; + type?: string; + }): Promise { + try { + const target = assetId?.trim() || name?.trim(); + if (!target) { + return {status: "failed", message: "No assetId or name provided", data: null}; + } + + const assetSource = this.engine.editor?.assetSource; + if (!assetSource) { + return { + status: "failed", + message: "No active editing context (scene or stem) available", + data: null, + }; + } + + const types = resolveAssetTypes(type); + const response = await assetSource.getAssets({ + types, + includeLatestRelease: true, + includeThumbnails: true, + }); + const asset = findAsset(response?.assets ?? [], target); + + if (!asset) { + return { + status: "failed", + message: `Scene asset "${target}" not found`, + data: null, + }; + } + + return { + status: "success", + message: `Retrieved scene asset ${asset.name} (${asset.id})`, + data: { + assetSource: {kind: assetSource.kind, id: assetSource.id}, + asset: summarizeAsset(asset), + }, + }; + } catch (error) { + return { + status: "failed", + message: `Error getting scene asset: ${error instanceof Error ? error.message : String(error)}`, + data: null, + }; + } + } +} + +function resolveAssetTypes(type: string | undefined): SceneAssetType[] | undefined { + if (!type?.trim()) return undefined; + const types = type + .split(",") + .flatMap(part => ASSET_TYPE_ALIASES[part.trim().toLowerCase()] ?? []) + .filter((item, index, array) => array.indexOf(item) === index); + return types.length > 0 ? types : undefined; +} + +function clampLimit(limit: number | undefined): number { + if (!Number.isFinite(limit)) return 80; + return Math.max(1, Math.min(200, Math.floor(limit!))); +} + +function findAsset(assets: Asset[], target: string): Asset | undefined { + const normalized = target.toLowerCase(); + return assets.find(asset => + asset.id === target || + asset.headRevisionId === target || + asset.revisionId === target || + asset.name === target || + asset.name?.toLowerCase() === normalized); +} + +function summarizeAsset(asset: Asset): AssetSummary { + const summary: AssetSummary = { + id: asset.id, + name: asset.name, + type: asset.type, + revisionId: asset.revisionId, + headRevisionId: asset.headRevisionId, + latestReleaseRevisionId: asset.latestRelease?.revisionId, + hasThumbnail: Boolean(asset.thumbnailUrl), + }; + + if (asset.description) summary.description = compactText(asset.description); + if (asset.tags?.length) summary.tags = asset.tags.slice(0, 12); + if (asset.contentType) summary.contentType = asset.contentType; + if (asset.format) summary.format = asset.format; + if (asset.createTime) summary.createTime = asset.createTime; + if (asset.updateTime) summary.updateTime = asset.updateTime; + + return summary; +} + +function compactText(value: string, maxLength = 220): string { + const normalized = value.replace(/\s+/g, " ").trim(); + return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 3)}...` : normalized; +} diff --git a/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.test.ts b/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.test.ts new file mode 100644 index 00000000..2b522215 --- /dev/null +++ b/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.test.ts @@ -0,0 +1,103 @@ +import {Object3D, Scene} from "three"; +import {beforeEach, describe, expect, it, vi} from "vitest"; + +const mocks = vi.hoisted(() => ({ + getScriptRevisionData: vi.fn(), +})); + +vi.mock("@stem/network/api/script", () => ({ + getScriptRevisionData: mocks.getScriptRevisionData, +})); + +import {LambdaHandlers} from "./LambdaHandlers"; + +const lambdaConfig = { + id: "motion", + name: "Motion Lambda", + description: "Moves registered objects.", + version: "1.0.0", + main: "MotionLambda.ts", + attributes: { + speed: {name: "Speed", type: "number", default: 1}, + }, + componentSchema: { + enabled: {name: "Enabled", type: "boolean", default: true}, + }, +}; + +const createHandlers = () => { + const scene = new Scene(); + scene.userData.lambdaInstances = [ + {lambdaId: "motion", instanceId: "scene-motion", enabled: true, attributes: {speed: 2}}, + ]; + + const object = new Object3D(); + object.name = "Mover"; + object.userData.lambdaComponents = [ + { + lambdaId: "motion", + instanceId: "component-motion", + uuid: "component-uuid", + enabled: true, + componentData: {enabled: true}, + }, + ]; + scene.add(object); + + const registry = { + getAllConfigs: vi.fn(() => [lambdaConfig]), + getAssetMeta: vi.fn(() => ({assetId: "lambda-asset", revisionId: "lambda-rev"})), + getConfig: vi.fn((id: string) => id === "motion" ? lambdaConfig : null), + }; + + const engine = { + editor: {lambdaConfigRegistry: registry}, + scene, + } as any; + + return {handlers: new LambdaHandlers(engine), registry}; +}; + +describe("LambdaHandlers", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getScriptRevisionData.mockResolvedValue({code: "export default class MotionLambda {}"}); + }); + + it("lists registered lambda metadata", () => { + const {handlers} = createHandlers(); + + const result = handlers.handleListLambdas({filter: "mot"}); + + expect(result.status).toBe("success"); + expect(result.data).toEqual([ + expect.objectContaining({ + id: "motion", + attributes: ["speed"], + componentSchema: ["enabled"], + }), + ]); + }); + + it("gets lambda config, bindings, and code when available", async () => { + const {handlers} = createHandlers(); + + const result = await handlers.handleGetLambda({lambdaId: "motion", includeCode: true}); + + expect(result.status).toBe("success"); + expect(result.data).toEqual(expect.objectContaining({ + assetMeta: {assetId: "lambda-asset", revisionId: "lambda-rev"}, + code: "export default class MotionLambda {}", + componentBindings: [ + expect.objectContaining({ + objectName: "Mover", + component: expect.objectContaining({lambdaId: "motion"}), + }), + ], + sceneInstances: [ + expect.objectContaining({lambdaId: "motion", instanceId: "scene-motion"}), + ], + })); + expect(mocks.getScriptRevisionData).toHaveBeenCalledWith("lambda-asset", "lambda-rev"); + }); +}); diff --git a/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.ts b/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.ts new file mode 100644 index 00000000..73460bff --- /dev/null +++ b/client/packages/editor-oss/src/agent/handlers/LambdaHandlers.ts @@ -0,0 +1,128 @@ +import {getScriptRevisionData} from "@stem/network/api/script"; + +import EngineRuntime from "../../EngineRuntime"; +import type {LambdaComponentData, LambdaConfig, LambdaInstanceData} from "../../lambdas/Lambda"; +import {CommandResult} from "../types/ACPTypes"; + +type LambdaRegistryLike = { + getAllConfigs?: () => LambdaConfig[]; + getAssetMeta?: (lambdaId: string) => {assetId: string; revisionId: string} | null; + getConfig?: (lambdaId: string) => LambdaConfig | null; +}; + +export class LambdaHandlers { + constructor(private engine: EngineRuntime) {} + + handleListLambdas({filter}: {filter?: string}): CommandResult { + try { + const registry = this.getRegistry(); + if (!registry?.getAllConfigs) { + return {status: "failed", message: "Lambda registry not found", data: []}; + } + + const normalizedFilter = filter?.toLowerCase(); + const lambdas = registry.getAllConfigs().map(config => ({ + id: config.id, + name: config.name, + description: config.description, + version: config.version, + tags: config.tags, + attributes: Object.keys(config.attributes ?? {}), + componentSchema: Object.keys(config.componentSchema ?? {}), + })); + const filtered = normalizedFilter && normalizedFilter !== "*" + ? lambdas.filter(lambda => + lambda.id?.toLowerCase().includes(normalizedFilter) || + lambda.name?.toLowerCase().includes(normalizedFilter)) + : lambdas; + + return { + status: "success", + message: `Retrieved ${filtered.length} lambdas (metadata only)`, + data: filtered, + }; + } catch (error) { + return { + status: "failed", + message: `Error listing lambdas: ${error instanceof Error ? error.message : String(error)}`, + data: [], + }; + } + } + + async handleGetLambda({ + includeCode = false, + lambdaId, + }: { + includeCode?: boolean; + lambdaId: string; + }): Promise { + try { + const registry = this.getRegistry(); + if (!registry?.getConfig) { + return {status: "failed", message: "Lambda registry not found", data: null}; + } + + const config = registry.getConfig(lambdaId); + if (!config) { + return {status: "failed", message: `Lambda ${lambdaId} not found`, data: null}; + } + + const assetMeta = registry.getAssetMeta?.(lambdaId) ?? null; + const data: Record = { + assetMeta, + componentBindings: this.getObjectComponents(lambdaId), + config, + sceneInstances: this.getSceneInstances(lambdaId), + }; + + if (includeCode && assetMeta) { + try { + data.code = (await getScriptRevisionData(assetMeta.assetId, assetMeta.revisionId)).code; + } catch (error) { + data.codeError = error instanceof Error ? error.message : String(error); + } + } + + return { + status: "success", + message: `Retrieved lambda ${config.name} (${lambdaId}) successfully`, + data, + }; + } catch (error) { + return { + status: "failed", + message: `Error getting lambda: ${error instanceof Error ? error.message : String(error)}`, + data: null, + }; + } + } + + private getRegistry(): LambdaRegistryLike | undefined { + return this.engine.editor?.lambdaConfigRegistry as unknown as LambdaRegistryLike | undefined; + } + + private getSceneInstances(lambdaId: string): LambdaInstanceData[] { + const instances = this.engine.scene?.userData?.lambdaInstances; + if (!Array.isArray(instances)) return []; + return instances.filter(instance => instance?.lambdaId === lambdaId); + } + + private getObjectComponents(lambdaId: string): Array<{objectName: string; objectUuid: string; component: LambdaComponentData}> { + const components: Array<{objectName: string; objectUuid: string; component: LambdaComponentData}> = []; + this.engine.scene?.traverse(object => { + const lambdaComponents = object.userData?.lambdaComponents; + if (!Array.isArray(lambdaComponents)) return; + for (const component of lambdaComponents) { + if (component?.lambdaId === lambdaId) { + components.push({ + objectName: object.name || object.type, + objectUuid: object.uuid, + component, + }); + } + } + }); + return components; + } +} diff --git a/client/packages/editor-oss/src/agent/script-tool/aliases.ts b/client/packages/editor-oss/src/agent/script-tool/aliases.ts index 48515e7c..2f6c57e8 100644 --- a/client/packages/editor-oss/src/agent/script-tool/aliases.ts +++ b/client/packages/editor-oss/src/agent/script-tool/aliases.ts @@ -31,6 +31,10 @@ export const SUPPORTED_RAW_COMMANDS: string[] = [ "get_behavior_settings", "get_selected_object", "get_player", + "list_scene_assets", + "get_scene_asset", + "list_lambdas", + "get_lambda", "set_material", "set_texture", "set_external_texture", @@ -106,6 +110,14 @@ export const ALIAS_MAP: Record = { // Scene queries "list objects": {command: "get_scene_objects"}, + "list assets": {command: "list_scene_assets"}, + "list imports": {command: "list_scene_assets", staticParams: {type: "imports"}}, + "list files": {command: "list_scene_assets", staticParams: {type: "files"}}, + "list models": {command: "list_scene_assets", staticParams: {type: "models"}}, + "list media": {command: "list_scene_assets", staticParams: {type: "media"}}, + "list behavior packs": {command: "list_scene_assets", staticParams: {type: "behaviors"}}, + "list lambda packs": {command: "list_scene_assets", staticParams: {type: "lambdas"}}, + "list packs": {command: "list_scene_assets", staticParams: {type: "packs"}}, "list behaviors": {command: "list_behaviors"}, "list prefabs": {command: "list_prefabs"}, "get camera": {command: "get_camera_settings", targetParam: "target"}, @@ -124,6 +136,10 @@ export const ALIAS_MAP: Record = { "get render settings": {command: "get_scene_setting", staticParams: {category: "rendering"}}, "get physics engine": {command: "get_scene_setting", staticParams: {category: "physics"}}, "get scene settings": {command: "get_scene_setting"}, + "get asset": {command: "get_scene_asset", targetParam: "assetId"}, + "get import": {command: "get_scene_asset", targetParam: "name", staticParams: {type: "imports"}}, + "get file": {command: "get_scene_asset", targetParam: "name", staticParams: {type: "files"}}, + "get scene asset": {command: "get_scene_asset", targetParam: "assetId"}, "get compartments": {command: "get_scene_setting", staticParams: {category: "compartments"}}, "get project": {command: "get_scene_setting", staticParams: {category: "project"}}, "get material": {command: "get_material_settings", targetParam: "target"}, @@ -151,6 +167,10 @@ export const ALIAS_MAP: Record = { get: {command: "get_object", targetParam: "target"}, select: {command: "get_selected_object"}, player: {command: "get_player"}, + "list lambdas": {command: "list_lambdas"}, + "lambda list": {command: "list_lambdas"}, + "lambda get": {command: "get_lambda"}, + "get lambda": {command: "get_lambda", targetParam: "lambdaId"}, // Materials & textures material: {command: "set_material", targetParam: "target"}, diff --git a/client/packages/editor-oss/src/agent/script-tool/commandContractMatrix.ts b/client/packages/editor-oss/src/agent/script-tool/commandContractMatrix.ts index 91d4a264..ee8b4ed4 100644 --- a/client/packages/editor-oss/src/agent/script-tool/commandContractMatrix.ts +++ b/client/packages/editor-oss/src/agent/script-tool/commandContractMatrix.ts @@ -120,6 +120,20 @@ export const STEMSCRIPT_COMMAND_CONTRACTS: StemScriptCommandContract[] = [ verification: "read-only", sample: "get_player", }, + { + command: "list_scene_assets", + effect: "read-only", + area: "asset", + verification: "read-only", + sample: "list_scene_assets type=models filter=*", + }, + { + command: "get_scene_asset", + effect: "read-only", + area: "asset", + verification: "read-only", + sample: "get_scene_asset assetId=asset-model", + }, { command: "set_material", effect: "setter", @@ -155,6 +169,20 @@ export const STEMSCRIPT_COMMAND_CONTRACTS: StemScriptCommandContract[] = [ verification: "read-only", sample: "get_behavior behaviorId=contract.behavior", }, + { + command: "list_lambdas", + effect: "read-only", + area: "lambda", + verification: "read-only", + sample: "list_lambdas filter=*", + }, + { + command: "get_lambda", + effect: "read-only", + area: "lambda", + verification: "read-only", + sample: "get_lambda lambdaId=contract.lambda includeCode=true", + }, { command: "add_behavior", effect: "asset-import", diff --git a/client/packages/editor-oss/src/ai/HttpAIBackend.test.ts b/client/packages/editor-oss/src/ai/HttpAIBackend.test.ts index 06680f8f..5a742b6a 100644 --- a/client/packages/editor-oss/src/ai/HttpAIBackend.test.ts +++ b/client/packages/editor-oss/src/ai/HttpAIBackend.test.ts @@ -84,6 +84,22 @@ describe("HttpAIBackend.setProviderKey", () => { const ok = await backend.setProviderKey("anthropic", "sk-test"); expect(ok).toBe(false); }); + + it("clears the client key and posts an empty key to reset the server BYOK session", async () => { + await keyStore.set("anthropic", "sk-test"); + const backend = new HttpAIBackend({keyStore}); + + await backend.clearProviderKey("anthropic"); + + expect(await keyStore.get("anthropic")).toBeUndefined(); + expect(fetchSpy).toHaveBeenCalledWith( + `${ORIGIN}/api/AI/ConfigureKeys`, + expect.objectContaining({ + method: "POST", + body: JSON.stringify({provider: "anthropic", key: ""}), + }), + ); + }); }); describe("HttpAIBackend.request BYOK header forwarding", () => { diff --git a/client/packages/editor-oss/src/ai/HttpAIBackend.ts b/client/packages/editor-oss/src/ai/HttpAIBackend.ts index a50a945f..676e1c18 100644 --- a/client/packages/editor-oss/src/ai/HttpAIBackend.ts +++ b/client/packages/editor-oss/src/ai/HttpAIBackend.ts @@ -95,6 +95,19 @@ export class HttpAIBackend implements AIBackend { async clearProviderKey(provider: AIProvider): Promise { await this.options.keyStore?.delete(provider); this.capabilitiesCache = undefined; + try { + const url = this.resolveUrl(CONFIGURE_KEYS_PATH); + await fetch(url, { + method: "POST", + headers: {"Content-Type": "application/json"}, + credentials: "include", + body: JSON.stringify({provider, key: ""}), + }); + } catch { + // Local deletion is still authoritative for direct BYOK-header + // requests. The configure endpoint exists only in OSS, so keep + // clear idempotent when the server is unavailable or integrated. + } } private async dispatch(path: string, options: AIRequestOptions): Promise { diff --git a/client/packages/editor-oss/src/context/OssAssetRegistryContext.test.tsx b/client/packages/editor-oss/src/context/OssAssetRegistryContext.test.tsx new file mode 100644 index 00000000..e658e1fb --- /dev/null +++ b/client/packages/editor-oss/src/context/OssAssetRegistryContext.test.tsx @@ -0,0 +1,56 @@ +import {render} from "@testing-library/react"; +import {afterEach, describe, expect, it} from "vitest"; + +import { + AssetType, + getOssAssetRegistry, + lookupOssAsset, + registerOssAsset, + resetOssAssetRegistryForTests, + type OssAssetRegistry, +} from "@stem/network/api/asset"; +import {OssAssetRegistryProvider, useOssAssetRegistry} from "./OssAssetRegistryContext"; + +const RegistryProbe = ({onRegistry}: {onRegistry: (registry: OssAssetRegistry) => void}) => { + const registry = useOssAssetRegistry(); + onRegistry(registry); + return null; +}; + +afterEach(() => { + resetOssAssetRegistryForTests(); +}); + +describe("OssAssetRegistryProvider", () => { + it("keeps the same registry object across rerenders", () => { + const observed: OssAssetRegistry[] = []; + const {rerender} = render( + + observed.push(registry)} /> + , + ); + + const first = observed.at(-1); + expect(first).toBeDefined(); + + registerOssAsset({ + assetId: "asset-1", + revisionId: "revision-1", + type: AssetType.Model, + format: "glb", + name: "Stable model", + dataUrl: "data:model/gltf-binary;base64,AAAA", + projectId: "project-1", + }); + + rerender( + + observed.push(registry)} /> + , + ); + + expect(observed.at(-1)).toBe(first); + expect(getOssAssetRegistry()).toBe(first); + expect(lookupOssAsset("revision-1")?.assetId).toBe("asset-1"); + }); +}); diff --git a/client/packages/editor-oss/src/context/OssAssetRegistryContext.tsx b/client/packages/editor-oss/src/context/OssAssetRegistryContext.tsx new file mode 100644 index 00000000..ab53567e --- /dev/null +++ b/client/packages/editor-oss/src/context/OssAssetRegistryContext.tsx @@ -0,0 +1,27 @@ +import React, {useContext, useEffect, useState} from "react"; + +import { + getOssAssetRegistry, + setOssAssetRegistry, + type OssAssetRegistry, +} from "@stem/network/api/asset"; + +export const OssAssetRegistryContext = React.createContext(null); + +export const OssAssetRegistryProvider = ({children}: {children: React.ReactNode}) => { + const [registry] = useState(() => getOssAssetRegistry()); + + useEffect(() => { + setOssAssetRegistry(registry); + }, [registry]); + + return ( + + {children} + + ); +}; + +export const useOssAssetRegistry = (): OssAssetRegistry => { + return useContext(OssAssetRegistryContext) ?? getOssAssetRegistry(); +}; diff --git a/client/packages/editor-oss/src/context/index.ts b/client/packages/editor-oss/src/context/index.ts index 6a20f4ef..292bd9c3 100644 --- a/client/packages/editor-oss/src/context/index.ts +++ b/client/packages/editor-oss/src/context/index.ts @@ -12,6 +12,7 @@ import {LibrariesContext} from "./LibrariesContext"; import {LightingContext} from "./LightingContext"; import {ModelAnimationCombinerContext} from "./ModelAnimationCombinerContext"; import {ModelsTabContext} from "./ModelsTabContext"; +import {OssAssetRegistryContext} from "./OssAssetRegistryContext"; import {ProjectStateContext} from "./ProjectStateContext"; import {PublishingContext} from "./PublishingContext"; import {UIStateContext} from "./UIStateContext"; @@ -48,6 +49,7 @@ export const useAppGlobalContext = () => React.useContext(AppGlobalContext); export const useAssetsTabContext = () => React.useContext(AssetsTabContext); export const useModelsTabContext = () => React.useContext(ModelsTabContext); export const useLibrariesContext = () => React.useContext(LibrariesContext); +export const useOssAssetRegistryContext = () => React.useContext(OssAssetRegistryContext); // Split context hooks for optimized re-renders // Use these instead of useAppGlobalContext when you only need specific state export const useUIStateContext = () => React.useContext(UIStateContext); diff --git a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts new file mode 100644 index 00000000..30124712 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.test.ts @@ -0,0 +1,386 @@ +import {describe, expect, it, vi} from "vitest"; + +import global from "../global"; +import {DirectCopilotProvider} from "./DirectCopilotProvider"; +import type {CopilotChatKey} from "./playgroundCopilotKeys"; +import type {PlaygroundLLMClient} from "./playgroundLLMClient"; + +const openAIKey: CopilotChatKey = {provider: "openai", apiKey: "sk-test", model: "gpt-5.2-codex"}; +const anthropicKey: CopilotChatKey = { + provider: "anthropic", + apiKey: "sk-test", + model: "claude-sonnet-4-5-20250929", +}; + +const makeLLMClient = (...responses: Array>): PlaygroundLLMClient => ({ + generateText: vi.fn().mockImplementation(async () => { + const response = responses.shift(); + return JSON.stringify(response ?? {reply: "No changes.", stemscript: ""}); + }), +}); + +const makeExecutor = () => { + const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => ({ + success: true, + step: { + id: "step-1", + command, + parameters, + status: "completed", + }, + result: {message: "ok"}, + })); + + return { + executeCommand, + hasPendingInteractiveResults: () => false, + getPendingInteractiveResults: () => [], + handleUserSelectionResult: () => false, + on: vi.fn(), + }; +}; + +describe("DirectCopilotProvider", () => { + it("generates StemScript through the provider and executes it in the registry path", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Added a box.", + stemscript: "add box name=TestBox position=1,2,3 color=#ff0000", + }); + const events: string[] = []; + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + provider.on("commandWillExecute", event => events.push(`will:${event.data.command}`)); + provider.on("commandExecuted", event => events.push(`done:${event.data.command}`)); + + const response = await provider.prompt("make a red test box"); + const llmRequest = vi.mocked(llmClient.generateText).mock.calls[0]?.[0]; + + expect(llmClient.generateText).toHaveBeenCalledOnce(); + expect(llmRequest?.prompt).toContain("User request:"); + expect(llmRequest?.prompt).not.toContain("StemStudio playground knowledge base"); + expect(llmRequest?.systemPrompt).toContain("cached StemScript/API knowledge base"); + expect(llmRequest?.knowledgePrompt).toContain("StemStudio playground knowledge base"); + expect(llmRequest?.knowledgePrompt).toContain("1 unit = 1 meter"); + expect(llmRequest?.systemPrompt).toContain("complete playable changes"); + expect(llmRequest?.systemPrompt).toContain("Prefer existing built-in behavior components"); + expect(llmRequest?.knowledgePrompt).toContain("Behavior registry"); + expect(llmRequest?.knowledgePrompt).toContain("Built-in behavior catalog"); + expect(llmRequest?.knowledgePrompt).toContain("behaviorId=character"); + expect(llmRequest?.knowledgePrompt).toContain("behaviorId=consumable"); + expect(llmRequest?.knowledgePrompt).toContain("Import API reference"); + expect(llmRequest?.systemPrompt).toContain("set a project title"); + expect(llmRequest?.systemPrompt).toContain('project title "Arena Runner"'); + expect(llmRequest?.systemPrompt).toContain('description="Copilot generated for:'); + expect(llmRequest?.systemPrompt).toContain("inspected/reused assets"); + expect(llmRequest?.systemPrompt).toContain("inspectionStemscript"); + expect(llmRequest?.systemPrompt).toContain("lambda list"); + expect(llmRequest?.systemPrompt).toContain("list assets"); + expect(llmRequest?.systemPrompt).toContain("list imports"); + expect(llmRequest?.systemPrompt).toContain("list files"); + expect(llmRequest?.knowledgePrompt).toContain("Asset/import inspection"); + expect(llmRequest?.knowledgePrompt).toContain("Descriptions are searchable metadata"); + expect(llmRequest?.knowledgePrompt).toContain("list behavior packs"); + expect(llmRequest?.knowledgePrompt).toContain("list lambda packs"); + expect(llmRequest?.promptCacheKey).toBe("stemstudio-playground-copilot-v5"); + expect(llmRequest?.maxOutputTokens).toBe(4096); + expect(llmRequest?.key).toMatchObject({provider: "openai", model: "gpt-5.2-codex"}); + expect(executor.executeCommand).toHaveBeenCalledWith("create_primitive", expect.objectContaining({ + type: "box", + name: "TestBox", + })); + expect(events).toEqual(["will:create_primitive", "done:create_primitive"]); + expect(response).toContain("Added a box."); + expect(response).toContain("Applied 1/1 command"); + }); + + it("includes available behavior registry details in the dynamic provider prompt", async () => { + const previousApp = global.app; + global.app = { + editor: { + behaviorConfigRegistry: { + getAllConfigs: () => [ + { + id: "custom.doorController", + name: "Door Controller", + description: "Opens a door when a trigger activates it.", + isScript: true, + attributes: { + speed: {type: "number", default: 2}, + separator: {type: "separator"}, + }, + }, + ], + }, + lambdaConfigRegistry: { + getAllConfigs: () => [ + { + id: "custom.motionLambda", + name: "Motion Lambda", + description: "Moves registered objects.", + attributes: { + speed: {type: "number", default: 1}, + }, + componentSchema: { + enabled: {type: "boolean", default: true}, + }, + }, + ], + }, + }, + } as any; + + try { + const llmClient = makeLLMClient({reply: "No changes.", stemscript: ""}); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: makeExecutor, + }); + + await provider.prompt("what behavior can open a door?"); + const llmRequest = vi.mocked(llmClient.generateText).mock.calls[0]?.[0]; + + expect(llmRequest?.prompt).toContain("Available behavior registry JSON"); + expect(llmRequest?.prompt).toContain('"id": "custom.doorController"'); + expect(llmRequest?.prompt).toContain('"key": "speed"'); + expect(llmRequest?.prompt).toContain("Available lambda registry JSON"); + expect(llmRequest?.prompt).toContain('"id": "custom.motionLambda"'); + expect(llmRequest?.prompt).toContain('"componentSchema"'); + expect(llmRequest?.prompt).not.toContain("separator"); + } finally { + global.app = previousApp; + } + }); + + it("runs read-only inspection and replans before mutating the scene", async () => { + const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => { + const result = + command === "get_scene_objects" + ? {message: "Found objects", data: [{name: "Player", type: "Mesh"}]} + : command === "get_object" + ? {message: "Retrieved Player", data: {name: "Player", position: {x: 0, y: 1, z: 0}}} + : {message: "ok"}; + return { + success: true, + step: {id: "step-1", command, parameters, status: "completed"}, + result, + }; + }); + const executor = { + executeCommand, + hasPendingInteractiveResults: () => false, + getPendingInteractiveResults: () => [], + handleUserSelectionResult: () => false, + on: vi.fn(), + }; + const llmClient = makeLLMClient( + { + reply: "I need to inspect the player first.", + inspectionStemscript: "list objects filter=Player\nget Player", + stemscript: "", + }, + { + reply: "Moved the existing player.", + stemscript: "update Player position=1,1,0", + }, + ); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("move the player right"); + const secondRequest = vi.mocked(llmClient.generateText).mock.calls[1]?.[0]; + + expect(llmClient.generateText).toHaveBeenCalledTimes(2); + expect(executeCommand).toHaveBeenCalledWith("get_scene_objects", expect.objectContaining({filter: "Player"})); + expect(executeCommand).toHaveBeenCalledWith("get_object", expect.objectContaining({target: "Player"})); + expect(executeCommand).toHaveBeenCalledWith("modify_object", expect.objectContaining({target: "Player"})); + expect(secondRequest?.prompt).toContain("Inspection results JSON"); + expect(secondRequest?.prompt).toContain("Retrieved Player"); + expect(secondRequest?.prompt).toContain('"position": {'); + expect(secondRequest?.prompt).toContain('"x": 0'); + expect(response).toContain("Applied 1/1 command"); + }); + + it("lets inspection query imported models, imports, files, and behavior/lambda packs", async () => { + const executeCommand = vi.fn().mockImplementation(async (command: string, parameters: Record) => ({ + success: true, + step: {id: "step-1", command, parameters, status: "completed"}, + result: { + message: `ok ${command}`, + data: command === "list_scene_assets" + ? {assets: [{id: "asset-1", name: "Kart", type: parameters.type}]} + : {asset: {id: parameters.assetId ?? parameters.name, name: parameters.assetId ?? parameters.name}}, + }, + })); + const executor = { + executeCommand, + hasPendingInteractiveResults: () => false, + getPendingInteractiveResults: () => [], + handleUserSelectionResult: () => false, + on: vi.fn(), + }; + const llmClient = makeLLMClient( + { + reply: "I need to inspect imported assets.", + inspectionStemscript: [ + "list models", + "list imports", + "list files", + "list behavior packs", + "list lambda packs", + "get asset assetId=model-1", + ].join("\n"), + stemscript: "", + }, + { + reply: "Used the existing imported asset context.", + stemscript: "", + }, + ); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("use the existing imported kart model"); + const secondRequest = vi.mocked(llmClient.generateText).mock.calls[1]?.[0]; + + expect(executeCommand).toHaveBeenCalledWith("list_scene_assets", expect.objectContaining({type: "models"})); + expect(executeCommand).toHaveBeenCalledWith("list_scene_assets", expect.objectContaining({type: "imports"})); + expect(executeCommand).toHaveBeenCalledWith("list_scene_assets", expect.objectContaining({type: "files"})); + expect(executeCommand).toHaveBeenCalledWith("list_scene_assets", expect.objectContaining({type: "behaviors"})); + expect(executeCommand).toHaveBeenCalledWith("list_scene_assets", expect.objectContaining({type: "lambdas"})); + expect(executeCommand).toHaveBeenCalledWith("get_scene_asset", expect.objectContaining({assetId: "model-1"})); + expect(secondRequest?.prompt).toContain("Inspection results JSON"); + expect(secondRequest?.prompt).toContain("list_scene_assets"); + expect(response).toContain("Used the existing imported asset context."); + }); + + it("executes existing behavior attach and config commands", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Made the player controllable.", + stemscript: [ + 'behavior attach Player behaviorId=character config={isDefault:true,walkSpeed:3}', + 'behavior config Player behaviorId=character attributesData={runSpeed:8}', + ].join("\n"), + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("make player controllable"); + + expect(executor.executeCommand).toHaveBeenCalledWith("attach_behavior", expect.objectContaining({ + target: "Player", + behaviorId: "character", + })); + expect(executor.executeCommand).toHaveBeenCalledWith("set_behavior_config", expect.objectContaining({ + target: "Player", + behaviorId: "character", + })); + expect(response).toContain("Applied 2/2 command"); + }); + + it("executes game metadata commands when generating a playable game", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Created a playable arena game.", + stemscript: [ + 'project title "Crystal Dash"', + "game settings isGame=true lives=3 maxScore=5 showHUD=true", + ].join("\n"), + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => openAIKey, + createExecutor: () => executor, + }); + + const response = await provider.prompt("make a crystal collection game"); + + expect(executor.executeCommand).toHaveBeenCalledWith("set_project_title", expect.objectContaining({ + title: "Crystal Dash", + })); + expect(executor.executeCommand).toHaveBeenCalledWith("set_game_settings", expect.objectContaining({ + isGame: true, + lives: 3, + maxScore: 5, + showHUD: true, + })); + expect(response).toContain("Applied 2/2 command"); + }); + + it("passes Anthropic key and static knowledge to the LLM client", async () => { + const executor = makeExecutor(); + const llmClient = makeLLMClient({ + reply: "Added behavior.", + stemscript: + 'behavior add name="ScoreController" description="Copilot generated for: score over time" code="this.update = function(dt) {}"', + }); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => anthropicKey, + createExecutor: () => executor, + }); + + await provider.prompt("add a score controller behavior"); + const llmRequest = vi.mocked(llmClient.generateText).mock.calls[0]?.[0]; + + expect(llmRequest?.key).toMatchObject({provider: "anthropic", model: "claude-sonnet-4-5-20250929"}); + expect(llmRequest?.systemPrompt).toContain("StemStudio playground copilot"); + expect(llmRequest?.knowledgePrompt).toContain("StemStudio playground knowledge base"); + expect(llmRequest?.prompt).toContain("User request:"); + expect(executor.executeCommand).toHaveBeenCalledWith("add_behavior", expect.objectContaining({ + name: "ScoreController", + description: "Copilot generated for: score over time", + })); + }); + + it("does not call a provider without a BYOK chat key", async () => { + const llmClient = makeLLMClient(); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKey: async () => null, + createExecutor: makeExecutor, + }); + + const response = await provider.prompt("make a scene"); + + expect(llmClient.generateText).not.toHaveBeenCalled(); + expect(response).toContain("No AI provider key"); + }); + + it("asks for a model selection when multiple BYOK chat keys are configured", async () => { + const llmClient = makeLLMClient(); + const provider = new DirectCopilotProvider({ + llmClient, + resolveKeyChoice: async () => ({ + kind: "needs-selection", + keys: [ + openAIKey, + {provider: "gemini", apiKey: "gem-test", model: "gemini-2.5-flash"}, + ], + }), + createExecutor: makeExecutor, + }); + + const response = await provider.prompt("make a game"); + + expect(llmClient.generateText).not.toHaveBeenCalled(); + expect(response).toContain("Multiple AI provider keys"); + expect(response).toContain("openai: gpt-5.2-codex"); + expect(response).toContain("gemini: gemini-2.5-flash"); + }); +}); diff --git a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts index 165c0659..e3fde1ad 100644 --- a/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts +++ b/client/packages/editor-oss/src/copilot/DirectCopilotProvider.ts @@ -1,44 +1,152 @@ // DirectCopilotProvider — a browser-only copilot for the playground. // -// The integrated build talks to a hosted agent (the proprietary -// StudioACPClient) over Agent Client Protocol. The OSS playground has no such -// server, so this provider streams a plain conversation straight from the -// visitor's chosen AI provider (Anthropic or OpenAI-compatible) using a key -// they supply via the BYOK panel. Nothing is proxied through the Go AI server. -// -// Scope: this is a *conversational* copilot only. It does not perform scene -// mutations / tool calls — that is the integrated agent's job. The ACP methods -// the editor never drives for a plain chat (executeCommand, interactive -// results, permission requests) are intentionally inert. +// The integrated build talks to a hosted ACP agent. The OSS playground has no +// such server, so this provider calls the visitor's chosen LLM provider +// directly from the browser, asks for a constrained StemScript plan, then +// applies that script through the same CommandsRegistry used by the terminal. import type {RequestPermissionResponse} from "@agentclientprotocol/sdk"; -import type {CommandExecutionResult} from "../agent/CommandsExecutor"; -import type {ACPEvent, ACPEventType, InteractiveSelectionResolution} from "../agent/types/ACPTypes"; +import {CommandsExecutor, type CommandExecutionResult} from "../agent/CommandsExecutor"; +import {CommandsRegistry} from "../agent/CommandsRegistry"; +import {ScriptExecutor} from "../agent/script-tool/ScriptExecutor"; +import type {ACPEvent, ACPEventType, InteractiveResult, InteractiveSelectionResolution} from "../agent/types/ACPTypes"; import {ConnectionState} from "../agent/types/ACPTypes"; +import { + buildBehaviorRegistrySummary, + buildLambdaRegistrySummary, + buildStructuredSceneSummary, +} from "../editor/assets/v2/AiCopilot/utils/prompt"; import type {CopilotEventHandler, ICopilotProvider} from "./ICopilotProvider"; -import {resolveCopilotChatKey, type CopilotChatKey} from "./playgroundCopilotKeys"; - -const ANTHROPIC_URL = "https://api.anthropic.com/v1/messages"; -const ANTHROPIC_MODEL = "claude-3-5-sonnet-latest"; -const OPENAI_URL = "https://api.openai.com/v1/chat/completions"; -const OPENAI_MODEL = "gpt-4o-mini"; -const MAX_TOKENS = 1024; - -const SYSTEM_PROMPT = - "You are the StemStudio playground copilot, helping a visitor explore a " + - "browser-based 3D game editor. Answer questions about 3D scenes, game " + - "behaviors, and how to build with the editor. Be concise. You are running " + - "as a direct browser client, so you cannot modify the scene yourself — " + - "explain the steps the user should take instead."; +import { + resolveCopilotChatKeyChoice, + type CopilotChatKey, + type CopilotChatKeyChoice, +} from "./playgroundCopilotKeys"; +import { + createPlaygroundLLMClient, + PLAYGROUND_MAX_OUTPUT_TOKENS, + PLAYGROUND_PROMPT_CACHE_KEY, + type PlaygroundLLMClient, +} from "./playgroundLLMClient"; +import {PLAYGROUND_STEMSCRIPT_KNOWLEDGE} from "./playgroundStemscriptKnowledge"; +import { + parseProviderStemscriptPlan, + validateGeneratedStemscript, + validateInspectionStemscript, + type PlaygroundStemscriptPlan, +} from "./playgroundStemscriptPlan"; + +const MAX_INSPECTION_ROUNDS = 2; const NO_KEY_MESSAGE = "No AI provider key is configured. Click the **Keys** button above to add " + - "an Anthropic or OpenAI key — it is stored locally in this browser and " + - "never leaves your machine."; + "an Anthropic, OpenAI/Codex, or Gemini key. It is stored locally in this " + + "browser and used only for direct provider calls."; + +const MULTIPLE_KEYS_MESSAGE = + "Multiple AI provider keys are configured. Click the **Keys** button above " + + "and choose the copilot model to use before running this request."; + +const SYSTEM_PROMPT = ` +You are the StemStudio playground copilot. You run inside a browser-based 3D editor. + +Your job: +- Convert the user's request into live StemScript commands that create or edit the current scene. +- Use the cached StemScript/API knowledge base when choosing scale, physics, cameras, VFX, behaviors, game rules, and scene structure. +- Build complete playable changes, not static mockups. When a request implies gameplay, set a project title, attach/configure behaviors, physics, camera, game settings, triggers, feedback, and any needed custom controller behavior in the same script. +- Prefer existing built-in behavior components and behavior IDs from the available behavior registry before writing custom behavior code. +- Use the available lambda registry when debugging or extending ECS-style runtime systems. Query lambda metadata with lambda list/lambda get before assuming schema. +- Query imported scene assets before referencing models, behavior/lambda packs, script imports, generic files, media, VFX assets, or prefabs. Use list assets/list imports/list files/list models/list behavior packs/list lambda packs and get asset/get import/get file. Use names, descriptions, tags, and formats from those results to decide which existing asset can be reused. +- Prefer commands that can execute immediately in the browser. If the user asks for local file imports, explain the exact import StemScript they can run in the terminal instead of emitting direct import commands here. +- Return only JSON with this exact shape: + {"reply":"short user-facing summary","inspectionStemscript":"optional read-only query commands","stemscript":"multi-line mutation commands","notes":["optional note"]} + +When the user asks a question or does not want a mutation, set "stemscript" to "" and answer in "reply". +When you need more scene context before editing, set "inspectionStemscript" to read-only query commands and leave "stemscript" empty. The editor will run those queries and call you again with the results. + +Allowed live patterns: +- add group name="Arena" +- add box|sphere|cylinder|cone|plane|torus|torusKnot|triangle|capsule|icosahedron|octahedron|dodecahedron|ring name="Object" position=x,y,z size=x,y,z color=#rrggbb parent="Group" +- update "Object" position=x,y,z rotation=x,y,z scale=x,y,z color=#rrggbb tag=Tag +- material "Object" color=#rrggbb roughness=0.5 metalness=0.1 opacity=1 +- light "Directional" intensity=0.8 castShadow=true +- scene background type=Color color=#rrggbb +- scene lighting ambient={color:"#ffffff",intensity:0.5} +- scene fog type=linear color=#rrggbb near=20 far=80 +- render settings useShadows=true shadowMapType=2 +- physics enable "Object"; physics set "Object" config={shape:"box",mass:0,ctype:"Static"} +- camera "DefaultCamera" cameraType=THIRD_PERSON defaultDistance=6 +- project title "Arena Runner" +- game settings isGame=true lives=3 maxScore=10 showHUD=true +- vfx add name="Effect" position=x,y,z config={...} +- list objects filter=Player; get Player; get settings Player; get material Ground; get physics Player; get camera DefaultCamera; get game settings +- behavior list filter=character; behavior get behaviorId=character +- get behavior Player behaviorId=character +- lambda list filter=motion; lambda get lambdaId=motionController includeCode=true +- list assets type=models|imports|files|behaviors|lambdas|packs|media filter=* limit=80; get asset assetId=asset-id; list imports; list files; list models; get import "math-helpers"; get file "level-data.json" +- behavior attach Player behaviorId=character config={isDefault:true,walkSpeed:3,runSpeed:8,jumpHeight:1.2} +- behavior attach Pickup behaviorId=consumable config={pointAmount:1,disposable:true} +- behavior attach Door behaviorId=tween config={startOnTrigger:true,move:{x:0,y:3,z:0},speed:1,loopMode:"Once"} +- behavior attach TriggerZone behaviorId=trigger config={if_condition:[{conditionType:"player_touches"}],if_operator:"and",then_steps:[{thenType:"activate",delay:0}]} +- behavior add name="GameController" description="Copilot generated for: arena scoring loop; uses Player, Coin, and Goal objects" code="this.init = function(game) {...}" +- behavior update behaviorId=GameController description="Copilot revised for: faster pickups and win condition" code="this.init = function(game) {...}" +- behavior attach Player behaviorId=GameController config={speed:6} +- behavior config Player behaviorId=GameController attributesData={speed:8} +- behavior detach Target behaviorId=BehaviorId +- navmesh add target="Default Scene" autoGenerate=true +- waypoint path add name=PatrolPath loop=true; waypoint add path=PatrolPath position=0,0,0 order=0 + +Rules: +- Do not use exec, export, save, require, add_model_to_scene, search_external_assets, search_local_assets, get_library_asset, or generate_3d_model. +- Do not create files, folders, bundles, YAML files, or external asset dependencies. +- Behavior code is allowed when built-ins are insufficient. Before adding or updating custom behavior code, inspect existing behavior/lambda registries or packs when relevant; if a listed asset fits, reuse it. If you add or update custom behavior code, include a description summarizing the user request, runtime purpose, inspected/reused assets, and expected attachment target, then attach it to the right scene object in the same stemscript. +- Existing behavior IDs are exact and case-sensitive. Use behaviorId=character, behaviorId=trigger, etc.; do not invent behavior IDs when a listed behavior fits. +- Inspection commands must be read-only: list/get objects, settings, materials, physics, lights, camera, scene settings, behavior settings/code, VFX, prefabs, lambdas, and scene assets/imports/files/models. Never put mutating commands in "inspectionStemscript". +- Keep most plans between 5 and 40 commands. Name important objects and group related objects. +- Use "size" for primitive dimensions. Use "parent" to organize children. +- For floors and walls, mark static colliders with physics commands when relevant. +- Keep JSON valid. Do not wrap the JSON in markdown. +`.trim(); type ChatMessage = {role: "user" | "assistant"; content: string}; +type DirectExecutor = Pick< + CommandsExecutor, + | "executeCommand" + | "hasPendingInteractiveResults" + | "getPendingInteractiveResults" + | "handleUserSelectionResult" + | "on" +>; + +export interface DirectCopilotProviderOptions { + fetchImpl?: typeof fetch; + resolveKey?: () => Promise; + resolveKeyChoice?: () => Promise; + createExecutor?: () => DirectExecutor; + llmClient?: PlaygroundLLMClient; +} + +type CommandEventMeta = { + index?: number; + total?: number; +}; + +type InspectionRound = { + script: string; + results: InspectionCommandResult[]; +}; + +type InspectionCommandResult = { + lineNumber: number; + command: string; + success: boolean; + message?: string; + data?: unknown; + error?: string; +}; + export class DirectCopilotProvider implements ICopilotProvider { readonly isSuppressingSessionUpdates = false; @@ -48,6 +156,26 @@ export class DirectCopilotProvider implements ICopilotProvider { private history: ChatMessage[] = []; private readonly handlers = new Map>(); private abortController: AbortController | null = null; + private executor: DirectExecutor | null = null; + private readonly resolveKeyChoice: () => Promise; + private readonly createExecutor: () => DirectExecutor; + private readonly llmClient: PlaygroundLLMClient; + + constructor(options: DirectCopilotProviderOptions = {}) { + const fetchImpl = options.fetchImpl ?? fetch.bind(globalThis); + this.resolveKeyChoice = + options.resolveKeyChoice ?? + (options.resolveKey + ? async () => { + const key = await options.resolveKey!(); + return key ? {kind: "ready", key, keys: [key]} : {kind: "none", keys: []}; + } + : resolveCopilotChatKeyChoice); + this.createExecutor = + options.createExecutor ?? + (() => new CommandsExecutor(new CommandsRegistry({getSessionId: () => this.sessionId}))); + this.llmClient = options.llmClient ?? createPlaygroundLLMClient(fetchImpl); + } private emit(type: ACPEventType, data?: ACPEvent["data"]): void { const set = this.handlers.get(type); @@ -61,6 +189,16 @@ export class DirectCopilotProvider implements ICopilotProvider { } } + private getExecutor(): DirectExecutor { + if (!this.executor) { + this.executor = this.createExecutor(); + this.executor.on("interactiveResult", (interactive: InteractiveResult) => { + this.emit("interactiveResult", interactive); + }); + } + return this.executor; + } + on(eventType: ACPEventType, handler: CopilotEventHandler): void { let set = this.handlers.get(eventType); if (!set) { @@ -71,10 +209,6 @@ export class DirectCopilotProvider implements ICopilotProvider { } async connect(): Promise { - // The conversation is per-request HTTPS — there is no persistent - // transport to open. "Connected" simply means the panel is usable; - // key resolution happens lazily on each prompt so the visitor can add - // a key after the panel is already open. this.connected = true; this.connectionState = ConnectionState.CONNECTED; this.emit("connected"); @@ -116,8 +250,6 @@ export class DirectCopilotProvider implements ICopilotProvider { } async loadSession(sessionId: string): Promise { - // No server-side session store in the playground — adopt the id so - // history message ids stay stable, but there is nothing to replay. this.sessionId = sessionId; } @@ -129,165 +261,305 @@ export class DirectCopilotProvider implements ICopilotProvider { return this.sessionId; } - async prompt(promptText: string): Promise { + async prompt(promptText: string, context: Record = {}): Promise { this.emit("promptStarted", {prompt: promptText}); - const key = await resolveCopilotChatKey(); - if (!key) { - this.emit("agentMessage", {message: NO_KEY_MESSAGE}); + const keyChoice = await this.resolveKeyChoice(); + if (keyChoice.kind === "none") { + this.emit("agentMessage", {message: NO_KEY_MESSAGE, replayStartNewMessage: true}); this.emit("promptCompleted"); return NO_KEY_MESSAGE; } + if (keyChoice.kind === "needs-selection") { + const message = [ + MULTIPLE_KEYS_MESSAGE, + "", + "Available models:", + ...keyChoice.keys.map(key => `- ${key.provider}: ${key.model}`), + ].join("\n"); + this.emit("agentMessage", {message, replayStartNewMessage: true}); + this.emit("promptCompleted"); + return message; + } - this.history.push({role: "user", content: promptText}); + const {key} = keyChoice; const controller = new AbortController(); this.abortController = controller; - let answer = ""; try { - for await (const chunk of this.streamCompletion(key, controller.signal)) { - answer += chunk; - this.emit("agentMessage", {message: chunk}); + this.emit("agentThinking", {message: "Generating StemScript for the live scene..."}); + + let providerPrompt = this.buildProviderPrompt(promptText, context); + let rawPlan = await this.requestPlan(key, providerPrompt, controller.signal); + let plan = parseProviderStemscriptPlan(rawPlan); + const inspections: InspectionRound[] = []; + + for (let round = 0; round < MAX_INSPECTION_ROUNDS && plan.inspectionStemscript.trim(); round++) { + const validatedInspection = validateInspectionStemscript(plan.inspectionStemscript); + if (validatedInspection.executableCommands > 0) { + this.emit("toolCall", {toolCall: {title: "Inspect scene"}}); + const results = await this.executeInspectionStemscript(validatedInspection.script, controller.signal); + inspections.push({script: validatedInspection.script, results}); + } + + this.emit("agentThinking", {message: "Planning changes from scene inspection..."}); + providerPrompt = this.buildProviderPrompt(promptText, context, { + inspections, + previousPlan: plan, + }); + rawPlan = await this.requestPlan(key, providerPrompt, controller.signal); + plan = parseProviderStemscriptPlan(rawPlan); } + + let finalMessage = plan.reply || "Done."; + + if (plan.stemscript.trim()) { + const validated = validateGeneratedStemscript(plan.stemscript); + if (validated.executableCommands > 0) { + this.emit("toolCall", {toolCall: {title: "Apply StemScript commands"}}); + const execution = await this.executeStemscript(validated.script, controller.signal); + finalMessage = this.formatExecutionSummary(finalMessage, validated.script, execution); + } + } + + this.emit("agentMessage", {message: finalMessage, replayStartNewMessage: true}); + this.history.push({role: "user", content: promptText}); + this.history.push({role: "assistant", content: finalMessage}); + this.history = this.history.slice(-8); + return finalMessage; } catch (err) { const message = err instanceof DOMException && err.name === "AbortError" ? "(cancelled)" : `Copilot request failed: ${err instanceof Error ? err.message : String(err)}`; - this.emit("agentMessage", {message}); - answer = answer || message; + this.emit("agentMessage", {message, replayStartNewMessage: true}); + return message; } finally { this.abortController = null; + this.emit("promptCompleted"); } + } - if (answer) this.history.push({role: "assistant", content: answer}); - this.emit("promptCompleted"); - return answer; + private buildProviderPrompt( + promptText: string, + context: Record, + inspectionContext?: {inspections: InspectionRound[]; previousPlan: PlaygroundStemscriptPlan}, + ): string { + const sceneSummary = buildStructuredSceneSummary(); + const behaviorRegistry = buildBehaviorRegistrySummary(); + const lambdaRegistry = buildLambdaRegistrySummary(); + const recentHistory = this.history.slice(-6); + + return [ + "User request:", + promptText, + "", + "Current scene summary JSON:", + JSON.stringify(sceneSummary ?? {}, null, 2), + "", + behaviorRegistry.length > 0 ? "Available behavior registry JSON:" : "", + behaviorRegistry.length > 0 ? JSON.stringify(behaviorRegistry, null, 2) : "", + behaviorRegistry.length > 0 ? "Use these exact behaviorId values for existing behavior attachments." : "", + behaviorRegistry.length > 0 ? "" : "", + lambdaRegistry.length > 0 ? "Available lambda registry JSON:" : "", + lambdaRegistry.length > 0 ? JSON.stringify(lambdaRegistry, null, 2) : "", + lambdaRegistry.length > 0 ? "Use these exact lambdaId values for lambda inspection and references." : "", + lambdaRegistry.length > 0 ? "" : "", + inspectionContext ? "Previous provider plan JSON:" : "", + inspectionContext ? JSON.stringify({ + inspectionStemscript: inspectionContext.previousPlan.inspectionStemscript, + reply: inspectionContext.previousPlan.reply, + notes: inspectionContext.previousPlan.notes, + }, null, 2) : "", + inspectionContext ? "" : "", + inspectionContext ? "Inspection results JSON:" : "", + inspectionContext ? JSON.stringify(inspectionContext.inspections, null, 2) : "", + inspectionContext ? "" : "", + "Attached/request context JSON:", + JSON.stringify(context ?? {}, null, 2), + "", + recentHistory.length > 0 ? "Recent conversation JSON:" : "", + recentHistory.length > 0 ? JSON.stringify(recentHistory, null, 2) : "", + "", + inspectionContext + ? "Return final JSON only. You may request another inspectionStemscript only if the results are still insufficient; otherwise produce the mutation stemscript." + : "Return JSON only. Use inspectionStemscript for read-only scene queries before edits. Use an empty stemscript string if no scene change should be applied.", + ].filter(part => part !== "").join("\n"); } - private async *streamCompletion( + private async requestPlan( key: CopilotChatKey, + prompt: string, signal: AbortSignal, - ): AsyncGenerator { - const response = - key.provider === "anthropic" - ? await this.requestAnthropic(key.apiKey, signal) - : await this.requestOpenAI(key.apiKey, signal); - - if (!response.ok || !response.body) { - const body = await response.text().catch(() => ""); - throw new Error(`HTTP ${response.status} ${body.slice(0, 300)}`); - } + ): Promise { + return this.llmClient.generateText({ + key, + prompt, + signal, + systemPrompt: SYSTEM_PROMPT, + knowledgePrompt: PLAYGROUND_STEMSCRIPT_KNOWLEDGE, + promptCacheKey: PLAYGROUND_PROMPT_CACHE_KEY, + maxOutputTokens: PLAYGROUND_MAX_OUTPUT_TOKENS, + }); + } + + private async executeInspectionStemscript( + script: string, + signal: AbortSignal, + ): Promise { + const lines = ScriptExecutor.parseScript(script).filter(line => !line.isComment && !line.isEmpty && line.parsed); + const results: InspectionCommandResult[] = []; - for await (const payload of readSseData(response.body)) { - const text = - key.provider === "anthropic" - ? extractAnthropicDelta(payload) - : extractOpenAIDelta(payload); - if (text) yield text; + for (let i = 0; i < lines.length; i++) { + if (signal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + const line = lines[i]!; + const parsed = line.parsed!; + this.emit("toolCallUpdate", {line: parsed.raw, index: i, total: lines.length}); + + try { + const result = await this.executeRegistryCommand(parsed.command, parsed.params, { + index: i, + total: lines.length, + }); + results.push({ + lineNumber: line.lineNumber, + command: parsed.raw, + success: result.success, + message: stringifyForPrompt(result.result?.message, 800), + data: compactForPrompt(result.result?.data), + error: result.error, + }); + } catch (err) { + results.push({ + lineNumber: line.lineNumber, + command: parsed.raw, + success: false, + error: err instanceof Error ? err.message : String(err), + }); + } } + + return results; } - private requestAnthropic(apiKey: string, signal: AbortSignal): Promise { - return fetch(ANTHROPIC_URL, { - method: "POST", - signal, - headers: { - "content-type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - // Required for browser-origin requests to the Anthropic API. - "anthropic-dangerous-direct-browser-access": "true", + private async executeStemscript( + script: string, + signal: AbortSignal, + ): Promise extends Promise ? T : never> { + let currentIndex = 0; + let total = 0; + + return ScriptExecutor.execute( + script, + async (command, params) => { + if (signal.aborted) { + throw new DOMException("Aborted", "AbortError"); + } + const result = await this.executeRegistryCommand(command, params, { + index: currentIndex, + total, + }); + return { + success: result.success, + message: result.result?.message, + error: result.error, + }; }, - body: JSON.stringify({ - model: ANTHROPIC_MODEL, - max_tokens: MAX_TOKENS, - system: SYSTEM_PROMPT, - stream: true, - messages: this.history, - }), - }); + (current, nextTotal, line) => { + currentIndex = current - 1; + total = nextTotal; + this.emit("toolCallUpdate", {line, index: currentIndex, total}); + }, + ); } - private requestOpenAI(apiKey: string, signal: AbortSignal): Promise { - return fetch(OPENAI_URL, { - method: "POST", - signal, - headers: { - "content-type": "application/json", - authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model: OPENAI_MODEL, - stream: true, - messages: [{role: "system", content: SYSTEM_PROMPT}, ...this.history], - }), - }); + private async executeRegistryCommand( + command: string, + params: Record, + meta: CommandEventMeta = {}, + ): Promise { + this.emit("commandWillExecute", {command, parameters: params, ...meta}); + const result = await this.getExecutor().executeCommand(command, params); + if (result.success) { + this.emit("commandExecuted", {command, parameters: params, result: result.result, ...meta}); + } else { + this.emit("commandExecutionFailed", {command, parameters: params, error: result.error, ...meta}); + } + return result; } - // --- Inert ACP surface (plain chat has no tool/agent loop) ------------- + private formatExecutionSummary( + reply: string, + script: string, + execution: Awaited>, + ): string { + const failures = execution.results.filter(result => !result.success); + const lines = [ + reply, + "", + "```stemscript", + script, + "```", + "", + `Applied ${execution.successCount}/${execution.executedCommands} command(s).`, + ]; + + if (failures.length > 0) { + lines.push(""); + lines.push("Some commands failed:"); + for (const failure of failures.slice(0, 5)) { + lines.push(`- Line ${failure.lineNumber}: ${failure.error || "Unknown error"}`); + } + } - async executeCommand(): Promise { - throw new Error("DirectCopilotProvider does not support command execution."); + return lines.join("\n").trim(); + } + + async executeCommand(method: string, params: Record): Promise { + return this.executeRegistryCommand(method, params); } respondToPermissionRequest(_requestId: string, _response: RequestPermissionResponse): void { - // No tool-permission flow in plain chat. + // Direct browser plans do not request host permissions. } hasPendingInteractiveResults(): boolean { - return false; + return this.getExecutor().hasPendingInteractiveResults(); } - submitInteractiveSelectionResolution(_resolution: InteractiveSelectionResolution): boolean { - return false; + submitInteractiveSelectionResolution(resolution: InteractiveSelectionResolution): boolean { + return this.getExecutor().handleUserSelectionResult( + resolution.interactiveResult.id, + resolution.results, + ); } - checkPendingInteractiveResult(_id: string): boolean { - return false; + checkPendingInteractiveResult(id: string): boolean { + return this.getExecutor().getPendingInteractiveResults().some(result => result.id === id); } } -/** Yields the JSON payload of each `data:` line in an SSE stream. */ -async function* readSseData(body: ReadableStream): AsyncGenerator { - const reader = body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - try { - for (;;) { - const {value, done} = await reader.read(); - if (done) break; - buffer += decoder.decode(value, {stream: true}); - let newlineIndex: number; - while ((newlineIndex = buffer.indexOf("\n")) >= 0) { - const line = buffer.slice(0, newlineIndex).trim(); - buffer = buffer.slice(newlineIndex + 1); - if (!line.startsWith("data:")) continue; - const payload = line.slice(5).trim(); - if (!payload || payload === "[DONE]") continue; - try { - yield JSON.parse(payload); - } catch { - // Skip malformed SSE frames. - } - } - } - } finally { - reader.releaseLock(); - } +function stringifyForPrompt(value: unknown, maxChars: number): string | undefined { + if (value === undefined || value === null) return undefined; + const text = typeof value === "string" ? value : safeJsonStringify(value); + if (!text) return undefined; + return text.length > maxChars ? `${text.slice(0, maxChars - 24)}... [truncated ${text.length} chars]` : text; } -function extractAnthropicDelta(payload: unknown): string { - const event = payload as {type?: string; delta?: {type?: string; text?: string}}; - if (event?.type === "content_block_delta" && event.delta?.type === "text_delta") { - return event.delta.text ?? ""; - } - return ""; +function compactForPrompt(value: unknown, maxChars = 6000): unknown { + if (value === undefined || value === null) return undefined; + const text = safeJsonStringify(value); + if (!text || text.length <= maxChars) return value; + return `${text.slice(0, maxChars - 24)}... [truncated ${text.length} chars]`; } -function extractOpenAIDelta(payload: unknown): string { - const event = payload as {choices?: Array<{delta?: {content?: string}}>}; - const text = event?.choices?.[0]?.delta?.content; - return typeof text === "string" ? text : ""; +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value); + } catch { + return String(value); + } } diff --git a/client/packages/editor-oss/src/copilot/index.ts b/client/packages/editor-oss/src/copilot/index.ts index a3ac23f7..4cb218f0 100644 --- a/client/packages/editor-oss/src/copilot/index.ts +++ b/client/packages/editor-oss/src/copilot/index.ts @@ -7,7 +7,20 @@ export { export {DirectCopilotProvider} from "./DirectCopilotProvider"; export {registerPlaygroundCopilot} from "./registerPlaygroundCopilot"; export { + CHAT_PROVIDERS, + COPILOT_DEFAULT_MODELS, + COPILOT_KEYS_CHANGED_EVENT, + COPILOT_MODEL_OPTIONS, hasCopilotKeysSync, refreshCopilotKeysMarker, resolveCopilotChatKey, + resolveCopilotChatKeyChoice, + resolveCopilotChatKeys, + setCopilotModelSelection, + getCopilotModelSelectionSync, +} from "./playgroundCopilotKeys"; +export type { + CopilotChatKey, + CopilotChatKeyChoice, + CopilotChatProvider, } from "./playgroundCopilotKeys"; diff --git a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts new file mode 100644 index 00000000..7d9f474c --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.test.ts @@ -0,0 +1,90 @@ +import {beforeEach, describe, expect, it, vi} from "vitest"; + +const mocks = vi.hoisted(() => ({ + store: { + all: vi.fn(), + }, +})); + +vi.mock("../ai", () => ({ + getBYOKKeyStore: () => mocks.store, +})); + +import { + getCopilotModelSelectionSync, + hasCopilotKeysSync, + refreshCopilotKeysMarker, + resolveCopilotChatKeyChoice, + resolveCopilotChatKeys, + setCopilotModelSelection, +} from "./playgroundCopilotKeys"; + +describe("playgroundCopilotKeys", () => { + beforeEach(() => { + vi.clearAllMocks(); + window.localStorage.clear(); + }); + + it("resolves a single chat key with the provider default model", async () => { + mocks.store.all.mockResolvedValue({openai: "sk-openai"}); + + const keys = await resolveCopilotChatKeys(); + const choice = await resolveCopilotChatKeyChoice(); + + expect(keys).toEqual([{provider: "openai", apiKey: "sk-openai", model: "gpt-5.2-codex"}]); + expect(choice).toEqual({kind: "ready", key: keys[0], keys}); + }); + + it("requires an explicit model selection when multiple chat keys exist", async () => { + mocks.store.all.mockResolvedValue({ + anthropic: "sk-anthropic", + openai: "sk-openai", + }); + + const choice = await resolveCopilotChatKeyChoice(); + + expect(choice.kind).toBe("needs-selection"); + expect(choice.keys.map(key => key.provider)).toEqual(["anthropic", "openai"]); + }); + + it("uses the selected provider and per-provider model override", async () => { + mocks.store.all.mockResolvedValue({ + anthropic: "sk-anthropic", + openai: "sk-openai", + }); + + setCopilotModelSelection("openai", "gpt-5.1-codex"); + + const choice = await resolveCopilotChatKeyChoice(); + + expect(getCopilotModelSelectionSync()).toEqual({provider: "openai", model: "gpt-5.1-codex"}); + expect(choice.kind).toBe("ready"); + if (choice.kind === "ready") { + expect(choice.key).toEqual({provider: "openai", apiKey: "sk-openai", model: "gpt-5.1-codex"}); + } + }); + + it("marks the playground copilot ready when any chat key exists", async () => { + mocks.store.all.mockResolvedValue({ + anthropic: "sk-anthropic", + openai: "sk-openai", + }); + + const ready = await refreshCopilotKeysMarker(); + + expect(ready).toBe(true); + expect(hasCopilotKeysSync()).toBe(true); + }); + + it("clears the playground copilot ready marker when chat keys are wiped", async () => { + mocks.store.all.mockResolvedValueOnce({anthropic: "sk-anthropic"}); + await refreshCopilotKeysMarker(); + expect(hasCopilotKeysSync()).toBe(true); + + mocks.store.all.mockResolvedValueOnce({}); + const ready = await refreshCopilotKeysMarker(); + + expect(ready).toBe(false); + expect(hasCopilotKeysSync()).toBe(false); + }); +}); diff --git a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts index bea5a7e6..739ef2ec 100644 --- a/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts +++ b/client/packages/editor-oss/src/copilot/playgroundCopilotKeys.ts @@ -16,19 +16,52 @@ import {getBYOKKeyStore} from "../ai"; import type {AIProvider} from "../ai"; /** - * Providers that can back a conversational copilot. Ordered by preference — - * `resolveCopilotChatKey()` returns the first one with a usable key. + * Providers that can back the playground copilot. Ordered for stable UI and + * legacy fallback behavior. */ -const CHAT_PROVIDERS: ReadonlyArray> = [ +export type CopilotChatProvider = Extract; + +export const CHAT_PROVIDERS: ReadonlyArray = [ "anthropic", "openai", + "gemini", ]; const COPILOT_READY_MARKER = "stem.playground.copilotReady"; +const COPILOT_SELECTED_PROVIDER = "stem.playground.copilot.selectedProvider"; +export const COPILOT_KEYS_CHANGED_EVENT = "stem:playground-copilot-keys-changed"; + +export const COPILOT_DEFAULT_MODELS: Record = { + anthropic: "claude-sonnet-4-5-20250929", + openai: "gpt-5.2-codex", + gemini: "gemini-2.5-flash", +}; + +export const COPILOT_MODEL_OPTIONS: Record> = { + anthropic: [ + {label: "Claude Sonnet 4.5", model: "claude-sonnet-4-5-20250929"}, + {label: "Claude Sonnet 4", model: "claude-sonnet-4-20250514"}, + {label: "Claude Opus 4.5", model: "claude-opus-4-5-20251101"}, + {label: "Claude Haiku 4.5", model: "claude-haiku-4-5-20251001"}, + ], + openai: [ + {label: "GPT-5.2 Codex", model: "gpt-5.2-codex"}, + {label: "GPT-5.1 Codex", model: "gpt-5.1-codex"}, + {label: "GPT-5.2", model: "gpt-5.2"}, + {label: "GPT-5.2 Chat", model: "gpt-5.2-chat-latest"}, + ], + gemini: [ + {label: "Gemini 2.5 Flash", model: "gemini-2.5-flash"}, + {label: "Gemini 2.5 Pro", model: "gemini-2.5-pro"}, + {label: "Gemini Flash Latest", model: "gemini-flash-latest"}, + {label: "Gemini 3 Flash Preview", model: "gemini-3-flash-preview"}, + ], +}; export type CopilotChatKey = { - provider: "anthropic" | "openai"; + provider: CopilotChatProvider; apiKey: string; + model: string; }; function getLocalStorage(): Storage | undefined { @@ -52,6 +85,15 @@ export function hasCopilotKeysSync(): boolean { } } +function notifyKeysChanged(): void { + if (typeof window === "undefined") return; + try { + window.dispatchEvent(new Event(COPILOT_KEYS_CHANGED_EVENT)); + } catch { + // Ignore environments that do not support Event construction. + } +} + function writeMarker(ready: boolean): void { const storage = getLocalStorage(); if (!storage) return; @@ -63,24 +105,100 @@ function writeMarker(ready: boolean): void { } } +function modelStorageKey(provider: CopilotChatProvider): string { + return `stem.playground.copilot.${provider}Model`; +} + +function readProviderModel(provider: CopilotChatProvider): string { + const storage = getLocalStorage(); + if (!storage) return COPILOT_DEFAULT_MODELS[provider]; + try { + const override = storage.getItem(modelStorageKey(provider))?.trim(); + return override || COPILOT_DEFAULT_MODELS[provider]; + } catch { + return COPILOT_DEFAULT_MODELS[provider]; + } +} + +function readSelectedProvider(): CopilotChatProvider | null { + const storage = getLocalStorage(); + if (!storage) return null; + try { + const provider = storage.getItem(COPILOT_SELECTED_PROVIDER); + return CHAT_PROVIDERS.includes(provider as CopilotChatProvider) + ? provider as CopilotChatProvider + : null; + } catch { + return null; + } +} + +export function getCopilotModelSelectionSync(): {provider: CopilotChatProvider; model: string} | null { + const provider = readSelectedProvider(); + return provider ? {provider, model: readProviderModel(provider)} : null; +} + +export function setCopilotModelSelection(provider: CopilotChatProvider, model?: string): void { + const storage = getLocalStorage(); + if (!storage) return; + try { + storage.setItem(COPILOT_SELECTED_PROVIDER, provider); + if (model?.trim()) { + storage.setItem(modelStorageKey(provider), model.trim()); + } + } catch { + // Ignore storage failures. + } +} + +export type CopilotChatKeyChoice = + | {kind: "none"; keys: []} + | {kind: "ready"; key: CopilotChatKey; keys: CopilotChatKey[]} + | {kind: "needs-selection"; keys: CopilotChatKey[]}; + /** - * Resolve the chat key to use for direct provider calls. Returns `null` when - * no chat-capable key is configured (or the encrypted store is locked). + * Resolve every chat-capable BYOK key currently available to the direct + * playground copilot. */ -export async function resolveCopilotChatKey(): Promise { +export async function resolveCopilotChatKeys(): Promise { const store = getBYOKKeyStore(); - if (!store) return null; + if (!store) return []; let keys: Partial>; try { keys = await store.all(); } catch { - return null; + return []; } + const available: CopilotChatKey[] = []; for (const provider of CHAT_PROVIDERS) { const apiKey = keys[provider]?.trim(); - if (apiKey) return {provider, apiKey}; + if (apiKey) available.push({provider, apiKey, model: readProviderModel(provider)}); } - return null; + return available; +} + +export async function resolveCopilotChatKeyChoice(): Promise { + const keys = await resolveCopilotChatKeys(); + if (keys.length === 0) return {kind: "none", keys: []}; + if (keys.length === 1) return {kind: "ready", key: keys[0]!, keys}; + + const selectedProvider = readSelectedProvider(); + const selectedKey = selectedProvider + ? keys.find(key => key.provider === selectedProvider) + : undefined; + if (selectedKey) return {kind: "ready", key: selectedKey, keys}; + + return {kind: "needs-selection", keys}; +} + +/** + * Resolve the chat key to use for direct provider calls. Returns `null` when + * no chat-capable key is configured, the encrypted store is locked, or multiple + * chat keys exist without a chosen copilot model. + */ +export async function resolveCopilotChatKey(): Promise { + const choice = await resolveCopilotChatKeyChoice(); + return choice.kind === "ready" ? choice.key : null; } /** @@ -89,7 +207,8 @@ export async function resolveCopilotChatKey(): Promise { * mutation. */ export async function refreshCopilotKeysMarker(): Promise { - const ready = (await resolveCopilotChatKey()) !== null; + const ready = (await resolveCopilotChatKeys()).length > 0; writeMarker(ready); + notifyKeysChanged(); return ready; } diff --git a/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts b/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts new file mode 100644 index 00000000..fd0e0d09 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundLLMClient.test.ts @@ -0,0 +1,135 @@ +import {beforeEach, describe, expect, it, vi} from "vitest"; + +const mocks = vi.hoisted(() => ({ + generateText: vi.fn(), + createOpenAI: vi.fn(), + createAnthropic: vi.fn(), + createGoogleGenerativeAI: vi.fn(), + openAIResponses: vi.fn(), + anthropicModel: vi.fn(), + googleModel: vi.fn(), +})); + +vi.mock("ai", () => ({ + generateText: mocks.generateText, +})); + +vi.mock("@ai-sdk/openai", () => ({ + createOpenAI: mocks.createOpenAI, +})); + +vi.mock("@ai-sdk/anthropic", () => ({ + createAnthropic: mocks.createAnthropic, +})); + +vi.mock("@ai-sdk/google", () => ({ + createGoogleGenerativeAI: mocks.createGoogleGenerativeAI, +})); + +import {createPlaygroundLLMClient} from "./playgroundLLMClient"; + +describe("createPlaygroundLLMClient", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.generateText.mockResolvedValue({text: "{\"reply\":\"ok\",\"stemscript\":\"\"}"}); + mocks.openAIResponses.mockReturnValue({provider: "openai"}); + mocks.anthropicModel.mockReturnValue({provider: "anthropic"}); + mocks.googleModel.mockReturnValue({provider: "google"}); + mocks.createOpenAI.mockReturnValue({responses: mocks.openAIResponses}); + mocks.createAnthropic.mockReturnValue(mocks.anthropicModel); + mocks.createGoogleGenerativeAI.mockReturnValue(mocks.googleModel); + }); + + it("uses OpenAI Responses with prompt caching provider options", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const client = createPlaygroundLLMClient(fetchImpl); + + const text = await client.generateText({ + key: {provider: "openai", apiKey: "sk-openai", model: "gpt-5.2-codex"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + promptCacheKey: "cache-key", + maxOutputTokens: 1234, + }); + + expect(text).toBe("{\"reply\":\"ok\",\"stemscript\":\"\"}"); + expect(mocks.createOpenAI).toHaveBeenCalledWith({apiKey: "sk-openai", fetch: fetchImpl}); + expect(mocks.openAIResponses).toHaveBeenCalledWith("gpt-5.2-codex"); + expect(mocks.generateText).toHaveBeenCalledWith(expect.objectContaining({ + model: {provider: "openai"}, + system: "System prompt\n\nKnowledge prompt", + prompt: "User request", + maxOutputTokens: 1234, + providerOptions: { + openai: { + promptCacheKey: "cache-key", + promptCacheRetention: "24h", + }, + }, + })); + }); + + it("marks Anthropic knowledge prompt as cacheable and uses browser direct-access headers", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const client = createPlaygroundLLMClient(fetchImpl); + + await client.generateText({ + key: { + provider: "anthropic", + apiKey: "sk-anthropic", + model: "claude-sonnet-4-5-20250929", + }, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + }); + + expect(mocks.createAnthropic).toHaveBeenCalledWith({ + apiKey: "sk-anthropic", + fetch: fetchImpl, + headers: { + "anthropic-dangerous-direct-browser-access": "true", + }, + }); + expect(mocks.anthropicModel).toHaveBeenCalledWith("claude-sonnet-4-5-20250929"); + expect(mocks.generateText).toHaveBeenCalledWith(expect.objectContaining({ + model: {provider: "anthropic"}, + system: [ + {role: "system", content: "System prompt"}, + { + role: "system", + content: "Knowledge prompt", + providerOptions: { + anthropic: { + cacheControl: {type: "ephemeral"}, + }, + }, + }, + ], + })); + }); + + it("uses the Google provider for Gemini and enables structured output support", async () => { + const fetchImpl = vi.fn() as unknown as typeof fetch; + const client = createPlaygroundLLMClient(fetchImpl); + + await client.generateText({ + key: {provider: "gemini", apiKey: "sk-gemini", model: "gemini-2.5-flash"}, + prompt: "User request", + systemPrompt: "System prompt", + knowledgePrompt: "Knowledge prompt", + }); + + expect(mocks.createGoogleGenerativeAI).toHaveBeenCalledWith({apiKey: "sk-gemini", fetch: fetchImpl}); + expect(mocks.googleModel).toHaveBeenCalledWith("gemini-2.5-flash"); + expect(mocks.generateText).toHaveBeenCalledWith(expect.objectContaining({ + model: {provider: "google"}, + providerOptions: { + google: { + structuredOutputs: true, + }, + }, + })); + }); +}); diff --git a/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts b/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts new file mode 100644 index 00000000..84739eef --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundLLMClient.ts @@ -0,0 +1,119 @@ +import type {generateText as generateTextType, LanguageModel, SystemModelMessage} from "ai"; + +import type {CopilotChatKey} from "./playgroundCopilotKeys"; + +export const PLAYGROUND_PROMPT_CACHE_KEY = "stemstudio-playground-copilot-v5"; +export const PLAYGROUND_MAX_OUTPUT_TOKENS = 4096; + +export type PlaygroundLLMGenerateRequest = { + key: CopilotChatKey; + prompt: string; + systemPrompt: string; + knowledgePrompt: string; + promptCacheKey?: string; + maxOutputTokens?: number; + signal?: AbortSignal; +}; + +export type PlaygroundLLMClient = { + generateText(request: PlaygroundLLMGenerateRequest): Promise; +}; + +type ProviderOptions = NonNullable[0]["providerOptions"]>; + +export function createPlaygroundLLMClient(fetchImpl: typeof fetch = fetch.bind(globalThis)): PlaygroundLLMClient { + return { + async generateText(request: PlaygroundLLMGenerateRequest): Promise { + const {generateText} = await import("ai"); + const model = await createLanguageModel(request.key, fetchImpl); + const result = await generateText({ + model, + system: buildSystemPrompt(request), + prompt: request.prompt, + maxOutputTokens: request.maxOutputTokens ?? PLAYGROUND_MAX_OUTPUT_TOKENS, + abortSignal: request.signal, + maxRetries: 1, + providerOptions: buildProviderOptions(request), + }); + + if (result.text.trim()) return result.text; + throw new Error(`${request.key.provider} response did not include text content.`); + }, + }; +} + +async function createLanguageModel(key: CopilotChatKey, fetchImpl: typeof fetch): Promise { + switch (key.provider) { + case "anthropic": { + const {createAnthropic} = await import("@ai-sdk/anthropic"); + const anthropic = createAnthropic({ + apiKey: key.apiKey, + fetch: fetchImpl, + headers: { + "anthropic-dangerous-direct-browser-access": "true", + }, + }); + return anthropic(key.model); + } + case "gemini": { + const {createGoogleGenerativeAI} = await import("@ai-sdk/google"); + const google = createGoogleGenerativeAI({ + apiKey: key.apiKey, + fetch: fetchImpl, + }); + return google(key.model); + } + case "openai": + default: { + const {createOpenAI} = await import("@ai-sdk/openai"); + const openai = createOpenAI({ + apiKey: key.apiKey, + fetch: fetchImpl, + }); + return openai.responses(key.model); + } + } +} + +function buildSystemPrompt(request: PlaygroundLLMGenerateRequest): string | SystemModelMessage[] { + if (request.key.provider !== "anthropic") { + return `${request.systemPrompt}\n\n${request.knowledgePrompt}`; + } + + return [ + { + role: "system", + content: request.systemPrompt, + }, + { + role: "system", + content: request.knowledgePrompt, + providerOptions: { + anthropic: { + cacheControl: {type: "ephemeral"}, + }, + }, + }, + ]; +} + +function buildProviderOptions(request: PlaygroundLLMGenerateRequest): ProviderOptions | undefined { + if (request.key.provider === "openai") { + return { + openai: { + promptCacheKey: request.promptCacheKey ?? PLAYGROUND_PROMPT_CACHE_KEY, + promptCacheRetention: "24h", + }, + }; + } + + if (request.key.provider === "gemini") { + return { + google: { + structuredOutputs: true, + }, + }; + } + + return undefined; +} diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts new file mode 100644 index 00000000..cdc7c6bc --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptKnowledge.ts @@ -0,0 +1,25 @@ +export const PLAYGROUND_STEMSCRIPT_KNOWLEDGE = ` +StemStudio playground knowledge base, distilled from stemstudio-importer docs: +- Work live in the browser editor. Build with primitives, groups, scene settings, materials, textures, physics, cameras, VFX, game settings, navmesh/waypoints, and behavior scripts. Avoid external search/generation for direct playground prompts. +- Scale: 1 unit = 1 meter. A human player is usually a capsule around size=0.5,1.8,0.5 at position=0,0.9,0. Floors can be boxes such as size=50,0.1,50 at position=0,-0.05,0. Platforms are often size=3,0.2,3. Trigger zones should be wider than the player, usually at least 1.5 m. +- Primitive dimensions: prefer size over scale for new geometry. For capsules and cylinders, size.x is diameter and size.y is height or length. For spheres, size.x is diameter. For boxes, size is width,height,depth. +- Object commands: add group name=Arena; add box|sphere|cylinder|cone|plane|torus|torusKnot|triangle|capsule|icosahedron|octahedron|dodecahedron|ring name=Object position=x,y,z size=x,y,z color=#rrggbb parent=Group; clone Target name=Copy position=x,y,z; delete Target; move Target parent=Group keepLocalSpace=true; update Target position=x,y,z rotation=x,y,z scale=x,y,z color=#rrggbb tag=Player. +- Object settings: use objectSettings={isBatchable:true,isStatic:true} for repeated or static level pieces when appropriate. Use update Target tag=Player or tag=Goal after creation; tags are an update operation. Use inspectionStemscript with get/list commands before risky edits: list objects, get Target, get settings Target, get material Target, get physics Target, get behavior Target behaviorId=, get selected, get player, get camera DefaultCamera, get game settings, get scene settings. +- Asset/import inspection: use read-only scene asset commands to see imported resources before referencing them. Commands: list assets type=all|models|imports|files|behaviors|lambdas|packs|media filter= limit=80; get asset assetId=; list models; list imports; list files; list behavior packs; list lambda packs; get import ; get file . Models are 3D model assets, imports are script import assets, files are generic data files, behavior/lambda packs are asset-backed registered packs, media includes image/audio/video, vfx maps to Quarks assets, and prefab/stem maps to Prefab assets. Use asset names, descriptions, tags, formats, and revision IDs from these results to choose reusable existing assets before generating replacement code. +- Physics: enable physics before detailed physics config. Use shape values box, sphere, capsule, convexHull, or concaveHull. Use concaveHull only for static terrain, tracks, or buildings. Use ctype Static, Dynamic, or Kinematic. Static level geometry should usually have mass 0; player-like bodies are usually Dynamic or Kinematic. +- Camera: configure "DefaultCamera". Use cameraType=THIRD_PERSON for 3D character or kart scenes, FIRST_PERSON for first-person scenes, TOP_DOWN for board or arena games, SIDE_SCROLLER for side-view platformers, and NONE when only projection settings are needed. +- Materials and atmosphere: use material Target color=#rrggbb roughness=0.5 metalness=0.1 opacity=1, texture Target textureUrl=|imageAsset= textureType=map, scene background, scene lighting, scene fog, scene tonemapping, scene postprocessing, light, render settings, and game settings to make primitives readable. Keep colors high-contrast enough to understand the play space. +- VFX: vfx add can rely on defaults. Use config.shape typed descriptors such as {type:"PointEmitter"}, {type:"SphereEmitter",radius:1.5}, or {type:"CircleEmitter",radius:0.75}. Generator values use typed objects such as {type:"ConstantValue",value:20} or {type:"IntervalValue",a:1,b:2}. Use worldSpace:true for trails that should detach from moving objects. +- VFX behaviors: use vfx behavior add Effect behaviorType=ColorOverLife config={mode:"gradient"} or vfx behavior remove Effect behaviorIndex=0. Use vfx modify Effect config={...} for particle system changes and vfx get Effect before modifying an existing system. +- Behavior registry: use the Available behavior registry JSON from the user prompt when present; it is the compact list of all currently registered behavior packs/configs. Use behavior list and behavior get behaviorId= for inspection commands when you need full metadata or code, and use list behavior packs/get asset to inspect imported behavior assets. Prefer exact behavior IDs from the registry/catalog; do not invent behaviorId values when an existing pack fits. Use behavior add name="BehaviorName" description="Copilot generated for: ; purpose: ; inspected/reused: " code="..." for new behavior scripts, behavior update behaviorId= description="Copilot revised for: ; changed: " code="..." for revisions, and behavior remove behaviorId= only when the user explicitly asks to remove custom behavior code. +- Behavior attachment: use behavior attach Target behaviorId=BehaviorName config={...}, behavior config Target behaviorId=BehaviorName attributesData={...} enabled=true, and behavior detach Target behaviorId=BehaviorName. When creating a playable experience, attach/configure the needed behavior components in the same script; do not stop after primitives. For trigger-style behavior config, attributesData/config can include if_condition, if_operator, and then_steps. variable_compare conditions read from erth.store keys. +- Built-in behavior catalog: character creates a controllable player (isDefault, walkSpeed, runSpeed, jumpHeight, health, autoForward); consumable creates pickups (pointAmount, healthAmount, speedAmount, jumpAmount, disposable, canReappear); trigger creates interactive/collision/timer logic (if_condition, if_operator, then_steps); tween moves/rotates/scales objects and is good for doors, lifts, animated hazards (startOnTrigger, move, rotate, scale, speed, loopMode); platform creates moving platforms (startOnTrigger, move, speed, loopMode); enemy creates basic hostile AI (enemyType, health, movementSpeed, attackDamage, attackDistance); projectile creates moving damaging objects (speed, gravity, lifetime, damage, radius); npc creates ambient roaming NPCs (movementType, movementSpeed, roamDistance, engageDistance); aiNpc creates conversational NPCs when provider/server support exists (npc_profile, voice_id, range, environment); follow makes an object follow another (followTargetUuid, distance, speed, rotate); jumppad launches characters (strength, enableAngle, angle); teleport moves players to a target (teleportTargetUuid); objectInteractions enables pickup/drop/push/pull (pickUp, drop, push, pull, interactionDistance); enableDisable toggles objects or behaviors (startOnTrigger, action, targetType, targetObject); visualEffect plays particle events (startOnTrigger, triggerEvents, customTriggerEvents); genericSound plays audio (startOnTrigger, audioAsset, positional, looping, autoPlay, volume); cinematicCamera, animation, dayNightCycle, touchControls, spawnpoint, navmesh, and navmesh-connection are also built-in IDs. +- Behavior recipes: playable character scenes usually need add capsule name=Player, update Player tag=Player, behavior attach Player behaviorId=character config={isDefault:true,walkSpeed:3,runSpeed:8,jumpHeight:1.2}, camera "DefaultCamera" cameraType=THIRD_PERSON defaultDistance=6, game settings isGame=true showHUD=true. Collectible goals use behavior attach Coin behaviorId=consumable config={pointAmount:1,disposable:true} plus game settings maxScore=. Triggered doors/lifts use behavior attach Door behaviorId=tween config={startOnTrigger:true,move:{x:0,y:3,z:0},speed:1,loopMode:"Once"} and behavior attach TriggerZone behaviorId=trigger config={if_condition:[{conditionType:"player_touches"}],if_operator:"and",then_steps:[{thenType:"activate",delay:0}]}; if exact trigger target fields are uncertain, create a small custom controller behavior instead of emitting broken target references. +- Behavior code lifecycle: valid behavior methods include this.init = function(game) {}, this.update = function(deltaTime) {}, this.fixedUpdate = function(fixedDeltaTime) {}, this.dispose = function() {}, this.onEvent = function(msg,data) {}, this.onStart = function() {}, and this.onStop = function() {}. Prefer let/const, avoid remote dependencies, keep code short, and avoid hallucinated APIs. Use game/runtime APIs exposed by the editor; inspect existing behavior docs when available. Descriptions are searchable metadata, so generated or revised behavior code should always describe the gameplay purpose and any existing assets/packs it was matched against. +- Lambda API reference: imported lambda YAML/assets can use runtime lambdas APIs such as lambdas.getInstance, lambdas.getInstancesByType, lambdas.registerObject, lambdas.deregisterObject, and lambdas.getObjectLambdas. Lambda code should not use this.erth, this.target, or this.gameObject; lambdas process many objects through this.processObjects(...) and object arguments. The Available lambda registry JSON is the compact list of all currently registered lambda packs/configs. Use lambda list filter=, lambda get lambdaId= includeCode=true, list lambda packs, and get asset assetId= as read-only inspection commands when debugging existing lambda config, component bindings, scene instances, assets, or code. Direct playground chat can explain/import lambda assets but does not auto-open a file picker. +- Navmesh and waypoints: navmesh add target="Default Scene" autoGenerate=true agentHeight=1.8 agentRadius=0.45 debugVisualization=false; navmesh rebuild; navmesh connection add Start target=End bidirectional=false radius=0.75; waypoint path add name=PatrolPath loop=true; waypoint add path=PatrolPath position=x,y,z order=0 waitTime=1 arrivalRadius=0.5. +- Import API reference: terminal StemScript supports import model, behavior, lambda, vfx, image, audio/sound, video, prefab, script/import, and file. Syntax: import [filepath] ["comment"], import name= filepath=, or import name= url=. Examples: import model Kart kart.glb; import behavior NpcController behaviors/npc.yaml; import lambda PatrolBrain lambdas/patrol.yaml; import script name="math-helpers" filepath="imports/math-helpers.js"; import file name="level-data" filepath="data/level.json". Direct playground copilot should not emit import commands unless an import executor is available; instead inspect existing assets with list imports/list files/list models, provide exact import lines in the reply, and use primitives/behaviors for immediate edits. +- Genre recipes: a small platformer needs a Player capsule, Ground, 3 to 5 platforms, a collectible, a goal trigger, physics, and SIDE_SCROLLER or THIRD_PERSON camera. A racing sketch needs a kart body, ground or track blocks, 3 checkpoint triggers, start/finish markers, physics, and THIRD_PERSON camera. A top-down arena needs a player marker, boundaries, obstacles, pickups, and TOP_DOWN camera. +- Gameplay recipes: for an arcade loop, start with project title "" and game settings isGame=true showHUD=true, then create clear spawn, goal, hazards, collectibles, scoring variables, reset rules, camera, lights, and player feedback. Use behavior scripts for motion, scoring, health, timers, AI patrol, pickups, checkpoints, or win/lose logic when primitives alone are not enough. +- Performance: keep live plans compact. Prefer grouped pieces and repeated simple primitives over hundreds of unique meshes. Avoid huge command counts unless the user explicitly asks for a large build. +`.trim(); diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts new file mode 100644 index 00000000..5272b6a5 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.test.ts @@ -0,0 +1,124 @@ +import {describe, expect, it} from "vitest"; + +import { + parseProviderStemscriptPlan, + validateGeneratedStemscript, + validateInspectionStemscript, +} from "./playgroundStemscriptPlan"; + +describe("playgroundStemscriptPlan", () => { + it("parses the JSON contract returned by a provider", () => { + const plan = parseProviderStemscriptPlan(JSON.stringify({ + reply: "Built a small obstacle course.", + inspectionStemscript: "list objects", + stemscript: "add group name=Course\nadd box name=Ground size=12,0.1,12 color=#334455", + notes: ["used primitives only"], + })); + + expect(plan.reply).toBe("Built a small obstacle course."); + expect(plan.inspectionStemscript).toBe("list objects"); + expect(plan.stemscript).toContain("add group name=Course"); + expect(plan.notes).toEqual(["used primitives only"]); + }); + + it("accepts a fenced StemScript fallback", () => { + const plan = parseProviderStemscriptPlan([ + "Here is the plan.", + "```stemscript", + "add sphere name=Pickup position=0,1,0 color=#ffcc00", + "```", + ].join("\n")); + + expect(plan.reply).toBe("Here is the plan."); + expect(plan.stemscript).toBe("add sphere name=Pickup position=0,1,0 color=#ffcc00"); + }); + + it("turns structured command arrays into script lines", () => { + const plan = parseProviderStemscriptPlan(JSON.stringify({ + reply: "Added a platform.", + commands: [ + { + command: "create_primitive", + params: { + type: "box", + name: "Platform A", + position: {x: 0, y: 0, z: 0}, + }, + }, + ], + })); + + expect(plan.stemscript).toContain("create_primitive"); + expect(plan.stemscript).toContain('name="Platform A"'); + expect(plan.stemscript).toContain('position={"x":0,"y":0,"z":0}'); + }); + + it("validates primitive-only live scripts", () => { + const result = validateGeneratedStemscript([ + "# Generated in browser", + "add group name=Arena", + "add box name=Ground size=10,0.1,10 color=#333333 parent=Arena", + ].join("\n")); + + expect(result.executableCommands).toBe(2); + expect(result.script).toContain("add box"); + }); + + it("allows browser-executable behavior authoring commands", () => { + const result = validateGeneratedStemscript([ + 'behavior add name="ScoreController" code="this.update = function(dt) {}"', + 'behavior attach Player behaviorId="ScoreController" config={scorePerSecond:1}', + 'behavior config Player behaviorId="ScoreController" attributesData={scorePerSecond:2}', + "behavior list filter=character", + "behavior get behaviorId=character", + "behavior detach Player behaviorId=character", + "lambda list filter=motion", + "lambda get lambdaId=motionController includeCode=true", + 'behavior update behaviorId="ScoreController" code="this.update = function(dt) {}"', + 'behavior remove behaviorId="ScoreController"', + ].join("\n")); + + expect(result.executableCommands).toBe(10); + expect(result.script).toContain("behavior add"); + expect(result.script).toContain("behavior attach Player"); + expect(result.script).toContain("behavior list"); + expect(result.script).toContain("behavior detach"); + expect(result.script).toContain("behavior update"); + }); + + it("only allows read-only commands in inspection scripts", () => { + const result = validateInspectionStemscript([ + "list objects filter=Player", + "get Player", + "get settings Player", + "get behavior Player behaviorId=character", + "behavior get behaviorId=character", + "lambda list filter=motion", + "lambda get lambdaId=motionController includeCode=true", + "list assets type=models filter=kart", + "list imports filter=helpers", + "list files filter=level", + "get asset assetId=model-1", + "get import math-helpers", + "get file level-data.json", + ].join("\n")); + + expect(result.executableCommands).toBe(13); + expect(result.script).toContain("lambda get"); + expect(result.script).toContain("list assets"); + expect(result.script).toContain("get import"); + expect(() => validateInspectionStemscript("update Player position=1,2,3")).toThrow(/not allowed in inspection/); + expect(() => validateInspectionStemscript("behavior attach Player behaviorId=character")).toThrow(/not allowed in inspection/); + }); + + it("rejects file and external-asset commands in playground mode", () => { + expect(() => validateGeneratedStemscript("import model Tree filepath=models/tree.glb")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("generate model prompt=\"make a spaceship\"")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("search assets phrases=[tree] type=model")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("texture external Ground assetId=brick assetType=textures name=Brick provider=polyhaven")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("prefab add id=coin name=Coin")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("list_project_tasks")).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript('create_project_task title="Add player"')).toThrow(/not allowed/); + expect(() => validateGeneratedStemscript("exec ./game.stemscript")).toThrow(/not allowed/); + }); +}); diff --git a/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts new file mode 100644 index 00000000..d560ffc4 --- /dev/null +++ b/client/packages/editor-oss/src/copilot/playgroundStemscriptPlan.ts @@ -0,0 +1,201 @@ +import {ScriptExecutor} from "../agent/script-tool/ScriptExecutor"; + +export interface PlaygroundStemscriptPlan { + inspectionStemscript: string; + reply: string; + stemscript: string; + notes: string[]; +} + +export interface ValidatedStemscript { + script: string; + executableCommands: number; +} + +const STEMSCRIPT_FENCE_RE = /```(?:stemscript|text|txt)?\s*([\s\S]*?)```/i; +const DISALLOWED_COMMANDS = new Set([ + "add_prefab_to_scene", + "create_prefab", + "exec", + "export", + "generate_3d_model", + "get_library_asset", + "import", + "list_project_tasks", + "create_project_task", + "update_project_task", + "delete_project_task", + "require", + "save", + "search_external_assets", + "search_local_assets", + "add_model_to_scene", + "set_external_texture", +]); +const READ_ONLY_COMMANDS = new Set([ + "get_scene_objects", + "get_object", + "get_object_settings", + "get_material_settings", + "get_behavior_settings", + "get_selected_object", + "get_player", + "list_scene_assets", + "get_scene_asset", + "list_behaviors", + "get_behavior", + "list_lambdas", + "get_lambda", + "get_physics_settings", + "get_light_settings", + "get_vfx", + "list_prefabs", + "get_prefab", + "get_camera_settings", + "get_editor_settings", + "get_scene_setting", +]); + +const stripCodeFence = (value: string): string => { + const trimmed = value.trim(); + const match = trimmed.match(STEMSCRIPT_FENCE_RE); + return (match?.[1] ?? trimmed).trim(); +}; + +const tryParseJsonObject = (value: string): unknown | null => { + const trimmed = stripCodeFence(value); + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start < 0 || end <= start) return null; + + try { + return JSON.parse(trimmed.slice(start, end + 1)); + } catch { + return null; + } +}; + +const stringArray = (value: unknown): string[] => { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); +}; + +const commandArrayToScript = (value: unknown): string => { + if (!Array.isArray(value)) return ""; + return value + .map(item => { + if (typeof item === "string") return item; + if (!item || typeof item !== "object") return ""; + const record = item as Record; + const command = typeof record.command === "string" ? record.command.trim() : ""; + const params = record.params && typeof record.params === "object" + ? Object.entries(record.params as Record) + .map(([key, param]) => `${key}=${formatParamValue(param)}`) + .join(" ") + : ""; + return [command, params].filter(Boolean).join(" "); + }) + .filter(line => line.trim().length > 0) + .join("\n"); +}; + +const formatParamValue = (value: unknown): string => { + if (typeof value === "string") { + if (/^[A-Za-z0-9_.:#/-]+$/.test(value)) return value; + return JSON.stringify(value); + } + if (typeof value === "number" || typeof value === "boolean") return String(value); + return JSON.stringify(value); +}; + +export function parseProviderStemscriptPlan(rawText: string): PlaygroundStemscriptPlan { + const parsed = tryParseJsonObject(rawText); + if (parsed && typeof parsed === "object") { + const record = parsed as Record; + const commands = commandArrayToScript(record.commands); + const stemscript = + typeof record.stemscript === "string" + ? record.stemscript + : typeof record.script === "string" + ? record.script + : commands; + const inspectionStemscript = + typeof record.inspectionStemscript === "string" + ? record.inspectionStemscript + : typeof record.inspectionScript === "string" + ? record.inspectionScript + : typeof record.inspectStemscript === "string" + ? record.inspectStemscript + : commandArrayToScript(record.inspectionCommands ?? record.queries); + + return { + inspectionStemscript: stripCodeFence(inspectionStemscript || ""), + reply: typeof record.reply === "string" ? record.reply.trim() : "", + stemscript: stripCodeFence(stemscript || ""), + notes: stringArray(record.notes), + }; + } + + const fenced = rawText.match(STEMSCRIPT_FENCE_RE); + if (fenced?.[1]) { + return { + inspectionStemscript: "", + reply: rawText.replace(fenced[0], "").trim(), + stemscript: stripCodeFence(fenced[1]), + notes: [], + }; + } + + return { + inspectionStemscript: "", + reply: rawText.trim(), + stemscript: "", + notes: [], + }; +} + +export function validateGeneratedStemscript(script: string): ValidatedStemscript { + return validateStemscript(script, command => DISALLOWED_COMMANDS.has(command)); +} + +export function validateInspectionStemscript(script: string): ValidatedStemscript { + return validateStemscript(script, command => !READ_ONLY_COMMANDS.has(command), "inspection"); +} + +function validateStemscript( + script: string, + isDisallowedCommand: (command: string) => boolean, + label = "playground copilot mode", +): ValidatedStemscript { + const normalized = stripCodeFence(script) + .split("\n") + .map(line => line.trim()) + .filter(line => line.length > 0) + .join("\n"); + + if (!normalized) { + return {script: "", executableCommands: 0}; + } + + const lines = ScriptExecutor.parseScript(normalized); + const disallowed: string[] = []; + let executableCommands = 0; + + for (const line of lines) { + const parsed = line.parsed; + if (!parsed || line.isComment || line.isEmpty) continue; + + executableCommands++; + if (parsed.isBuiltin || isDisallowedCommand(parsed.command)) { + disallowed.push(`line ${line.lineNumber}: ${parsed.raw}`); + } + } + + if (disallowed.length > 0) { + throw new Error( + `Generated StemScript used commands that are not allowed in ${label}: ${disallowed.join("; ")}`, + ); + } + + return {script: normalized, executableCommands}; +} diff --git a/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx new file mode 100644 index 00000000..90c2f15a --- /dev/null +++ b/client/packages/editor-oss/src/editor/assets/v2/AiCopilot/AiCopilot.sessionMode.test.tsx @@ -0,0 +1,346 @@ +import {act, cleanup, fireEvent, render, screen, waitFor} from "@testing-library/react"; +import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; + +vi.mock("./AiCopilot.styles", () => { + const cleanProps = (props: Record) => + Object.fromEntries(Object.entries(props).filter(([key]) => !key.startsWith("$"))); + const div = ({children, ...props}: any) =>
{children}
; + const span = ({children, ...props}: any) => {children}; + const button = ({children, ...props}: any) => ; + const textarea = (props: any) =>