diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 151a0c410..0a307c7ad 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -184,8 +184,18 @@ jobs: compose: needs: plan if: needs.plan.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs || '[]'), 'compose') - runs-on: ${{ vars.OAC_USE_GITHUB_RUNNERS == 'true' && 'ubuntu-22.04' || 'blacksmith-2vcpu-ubuntu-2204' }} - timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + architecture: amd64 + - runner: ubuntu-24.04-arm + architecture: arm64 + runs-on: ${{ matrix.runner }} + env: + GOARCH: ${{ matrix.architecture }} + timeout-minutes: 30 steps: - uses: actions/checkout@v7 with: @@ -193,6 +203,11 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: go.mod + - name: Test the native Core installation lifecycle + run: go test ./services/core/cmd/oac -count=1 -timeout=3m + - uses: ./.github/actions/e2b-provider + - name: Build and verify the architecture-matched E2B helper + run: bash scripts/build-e2b-provider.sh - name: Allocate an isolated Compose project run: python3 -c 'import uuid; print("COMPOSE_SMOKE_PROJECT=oac-smoke-" + uuid.uuid4().hex)' >> "$GITHUB_ENV" - name: Build the images, start the installation and verify it diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 7be750d35..c901adc6f 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -63,6 +63,14 @@ jobs: run: | go build -ldflags "-X github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/cli.Version=$(git rev-parse HEAD)" -o "$RUNNER_TEMP/oac-daemon${{ runner.os == 'Windows' && '.exe' || '' }}" ./apps/daemon/cmd/oac-daemon node --test scripts/build-native-installer.test.mjs + - name: Build and test the shared Core installer + run: | + go build -o "$RUNNER_TEMP/oac${{ runner.os == 'Windows' && '.exe' || '' }}" ./services/core/cmd/oac + go test ./services/core/cmd/oac -count=1 -timeout=3m + - name: Exercise the Windows Core launcher + if: runner.os == 'Windows' + shell: pwsh + run: ./deploy/test_install.ps1 -Binary "$env:RUNNER_TEMP/oac.exe" - name: Verify native download bootstrap and recovery run: go test ./services/core/internal/nativeinstaller -count=1 -timeout=3m - name: Native filesystem, authentication and process lifecycle @@ -163,3 +171,17 @@ jobs: path: ${{ runner.temp }}/native-ci-diagnostics/ retention-days: 7 if-no-files-found: ignore + + core-intel-mac: + name: Core installer (macOS Intel) + runs-on: macos-15-intel + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go test ./services/core/cmd/oac -count=1 -timeout=3m + - run: go build -o "$RUNNER_TEMP/oac" ./services/core/cmd/oac diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d798c1976..c8ff4ab28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,6 +114,9 @@ jobs: path: ${{ runner.temp }}/native-artifacts - name: Assemble the native installation catalog run: node scripts/build-native-catalog.mjs "$RUNNER_TEMP/native-artifacts" "$RUNNER_TEMP/native-installers" + - uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 - uses: ./.github/actions/e2b-provider - name: Build matched artifacts env: @@ -133,8 +136,8 @@ jobs: for asset in "$HOME/.oac/build/core-distribution/"*; do if [[ -f "$asset" ]]; then ln "$asset" "$HOME/.oac/build/release-upload/"; fi done - cp deploy/install.sh "$HOME/.oac/build/release-upload/install.sh" - (cd "$HOME/.oac/build/release-upload" && sha256sum install.sh > install.sh.sha256) + cp deploy/install.sh deploy/install.ps1 "$HOME/.oac/build/release-upload/" + (cd "$HOME/.oac/build/release-upload" && sha256sum install.sh > install.sh.sha256 && sha256sum install.ps1 > install.ps1.sha256) - name: Sign in to GHCR if: github.event_name == 'push' || inputs.draft_release env: diff --git a/Makefile b/Makefile index 24b6a13ab..1958607db 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ check-microsandbox-provider: .PHONY: check-distribution build-core-distribution check-distribution: node --test scripts/build-native-catalog.test.mjs - go test ./services/web -count=1 + go test ./services/web ./services/core/cmd/oac -count=1 PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/node -p 'test_*.py' PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/compose -p 'test_*.py' PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s scripts/acceptance -p 'test_*.py' diff --git a/README.md b/README.md index 5713ac292..d8056f965 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,18 @@ Core keeps durable execution state. The Runtime runs the chosen harness inside t ## Install -On a Linux amd64 host with Docker and Python 3.9+: +On Linux or macOS with [Docker configured](https://openagentcore.dev/docs/getting-started/install#prerequisites): ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash ``` +Windows PowerShell: + +```powershell +irm https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.ps1 | iex +``` + Then: 1. **Sign in to Web**, the admin console, with the Core key the installer created, and **configure the domain and HTTPS**. diff --git a/README.zh-CN.md b/README.zh-CN.md index 5414cea3c..fb55a62a5 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -42,12 +42,18 @@ OpenAgentCore 在你自己的基础设施上运行 AI Agent，对外提供 [Open ## 安装 -在已准备 Docker 和 Python 3.9+ 的 Linux amd64 主机上： +在已按[前置条件](https://openagentcore.dev/zh/docs/getting-started/install#prerequisites)准备 Docker 的 Linux 或 macOS 上： ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash ``` +Windows PowerShell: + +```powershell +irm https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.ps1 | iex +``` + 然后： 1. 用安装器生成的 Core key **登录 Web**（管理控制台），并**配置域名和 HTTPS**。 diff --git a/apps/web/DESIGN.md b/apps/web/DESIGN.md index 64bb19dd3..86c08ae95 100644 --- a/apps/web/DESIGN.md +++ b/apps/web/DESIGN.md @@ -402,7 +402,7 @@ A failed action whose outcome needs a decision (a sandbox change with no answer, A local-only installation has the same amber notice on Overview, Nodes and System: other machines cannot connect, followed by Core's configuration path and apply command as copyable values. If Core has no configuration snapshot, state that those instructions are unavailable; never fill in a path or command. Add node is disabled with its reason beside the action, and Getting started leaves its first step to do with the address fix visible. A pending or failed installation read cannot complete that step; a failed read shows Unknown and Retry. ### Onboarding -Signing in and the console tour share one frame: a dark stage on the left (always dark, whatever the theme) and the task panel on the right, which follows the theme. The stage is the product's one authored moment: a flickering indigo dot grid under slow light rays (Magic UI's flickering grid and light rays), Core as the OpenAgentCore mark on a tile with a travelling border beam, and two orbits of Agents, Sessions, Skills, Vaults, files, templates and machines around it; the OpenAgentCore mark is itself nodes on a ring. Brand copy sits bottom-left in solid ink; it is a paragraph, not a heading, because the panel's title names the task. Signing in asks for one thing, the deployment's Core key, in a single password field; the default key location and a copyable read command stay visible beneath it, with a reminder to substitute a custom installation directory. The key’s authority stays in a help tip. A refused key, too many attempts or an unavailable console is an error beside the field. Signing in opens the console on the Overview. The optional tour has three chapters — Monitor, Resources, Platform — whose stage shows a real dark screenshot of those pages, tilted towards the panel; it takes the place of the console until its last button, Skip or Escape, and then returns the focus to the control that opened it. Entering the console or the tour, and leaving the tour, happen inside a View Transition: the old page dissolves forward and the new one is revealed in a circle growing from the pressed button. With reduced motion the orbits hold their places, the grid is a still frame and no transition runs. +Signing in and the console tour share one frame: a dark stage on the left (always dark, whatever the theme) and the task panel on the right, which follows the theme. The stage is the product's one authored moment: a flickering indigo dot grid under slow light rays (Magic UI's flickering grid and light rays), Core as the OpenAgentCore mark on a tile with a travelling border beam, and two orbits of Agents, Sessions, Skills, Vaults, files, templates and machines around it; the OpenAgentCore mark is itself nodes on a ring. Brand copy sits bottom-left in solid ink; it is a paragraph, not a heading, because the panel's title names the task. Signing in asks for one thing, the deployment's Core key, in a single password field; a copyable Docker Compose command to read the key stays visible beneath it, with a reminder to substitute a custom installation directory. The key’s authority stays in a help tip. A refused key, too many attempts or an unavailable console is an error beside the field. Signing in opens the console on the Overview. The optional tour has three chapters — Monitor, Resources, Platform — whose stage shows a real dark screenshot of those pages, tilted towards the panel; it takes the place of the console until its last button, Skip or Escape, and then returns the focus to the control that opened it. Entering the console or the tour, and leaving the tour, happen inside a View Transition: the old page dissolves forward and the new one is revealed in a circle growing from the pressed button. With reduced motion the orbits hold their places, the grid is a still frame and no transition runs. ### Getting started The first card on the Overview while any step is to do: a card header ("Getting started", "n of 4 done", a help tip, then a ghost Take the tour button and an icon button that hides it) over four rows split by Faint Rules. Each row has a 22px numbered ring (a check on the tile wash when done), a 13px/600 title over one 12.5px Graphite line, a status dot (Done in green, To do in Pencil, Checking pending, Unknown for a failed read) and one outline action while the step is to do: Set up sandboxes, Add node, Open Nodes or Open sandbox backend; Open System; Create project (which continues to the new project's first key) or Issue key; See how to call (the newest active project, preferring one with an active key), or Projects and keys without an active project. Add node, Create project and Issue key open their page with the dialog already open; Open System brings the Default model provider section to the top of the page body and focuses the default harness's Set or Replace; See how to call opens the project and, once its keys, usage and address are read, brings its How to call heading to the top of the page body, focused. Only the page body scrolls; the page header stays. Every step done turns it into one line, "You're set", with Take the tour and Dismiss; it stays, through the tour, until dismissed, and the checklist does not come back on its own. The choice is kept per installation in the browser, also while the deployment cannot be read; Show Getting started, a quiet row above the sidebar's account controls, opens it again at any time. diff --git a/apps/web/PRODUCT.md b/apps/web/PRODUCT.md index 2aa72c28c..090ca06fd 100644 --- a/apps/web/PRODUCT.md +++ b/apps/web/PRODUCT.md @@ -22,7 +22,7 @@ The console runs beside the administrator's own Core, with execution, files and ## Operating Context -- Paired console (`services/web`): the administrator signs in with the deployment's Core key, the administration credential the installer writes to `secrets/core.key` under the installation directory (by default `~/.oac/core/secrets/core.key`; keeping and rotating it is described in [Core key](../../docs/getting-started/operations.md#core-key)). There are no console accounts or usernames. Sign-in shows the default file location and a copyable `cat ~/.oac/core/secrets/core.key` command for the Core host, with a reminder to substitute a custom installation directory. The browser sends the key only to sign in and keeps only the session cookie; the console server holds the Core key and forwards the Web API (`/core/v1/**`, including sandbox administration under `/core/v1/sandbox/**`). The console never calls `/v1`. +- Paired console (`services/web`): the administrator signs in with the deployment's [Core key](../../docs/getting-started/operations.md#core-key). Sign-in shows a copyable Docker Compose command to read the key on the Core host, with a reminder to substitute a custom installation directory. The browser sends the key only to sign in and keeps only the session cookie; the console server holds the Core key and forwards the Web API (`/core/v1/**`, including sandbox administration under `/core/v1/sandbox/**`). The console never calls `/v1`. - The Core key is not an Agents API identity and cannot call `/v1`. An administrator who wants to call the Agents API issues a project API key like any other caller. - `/console/config` reports the node installer (`node_installer`, `node_installer_sha256`), offered only with a 64-hex digest. Native self-hosted installation does not depend on this endpoint. It also lists the providers it has node files for (`node_artifacts`); without the deployment's provider, Add node says so and issues no command. Signing in grants administration, sandbox administration included. - Chinese and English UI; light and dark themes; reduced motion honored. diff --git a/apps/web/e2e/access.spec.ts b/apps/web/e2e/access.spec.ts index 41149ef97..f8d48a28d 100644 --- a/apps/web/e2e/access.spec.ts +++ b/apps/web/e2e/access.spec.ts @@ -18,11 +18,11 @@ test("signs in with the Core key, keeps it out of the browser, and signs out and await page.goto("/"); await expect(page.getByRole("heading", { name: "Sign in to OpenAgentCore" })).toBeVisible(); - await expect(page.getByText("cat ~/.oac/core/secrets/core.key", { exact: true })).toBeVisible(); + await expect(page.getByText('docker compose -f "$HOME/.oac/core/compose.yaml" exec -T web oac-web core-key', { exact: true })).toBeVisible(); await expect(page.getByText("For a custom installation directory, replace the path in this command.")).toBeVisible(); await page.context().grantPermissions(["clipboard-read", "clipboard-write"]); await page.getByRole("button", { name: "Copy key read command" }).click(); - expect(await page.evaluate(() => navigator.clipboard.readText())).toBe("cat ~/.oac/core/secrets/core.key"); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe('docker compose -f "$HOME/.oac/core/compose.yaml" exec -T web oac-web core-key'); await signIn(page, "not-the-core-key"); await expect(page.getByRole("alert")).toHaveText("This Core key is not correct. Check it and try again."); await signIn(page, FIXTURE_CORE_KEY); diff --git a/apps/web/src/features/first-run/ConsoleAccess.tsx b/apps/web/src/features/first-run/ConsoleAccess.tsx index d9ef1f408..5e561f330 100644 --- a/apps/web/src/features/first-run/ConsoleAccess.tsx +++ b/apps/web/src/features/first-run/ConsoleAccess.tsx @@ -12,13 +12,6 @@ import { withTransition } from "../onboarding/view-transition"; import { changeConsoleAuth, ConsoleAuthError, readConsoleAuth, type ConsoleAuth } from "./auth"; import "./console-access.css"; -/** - * Where the installer writes the Core key: its file inside the installation - * directory, and that file under the default installation directory. The - * visible sign-in instructions name both; the actual custom path is not public. - */ -const CORE_KEY_LOCATION = { file: "secrets/core.key", defaultPath: "~/.oac/core/secrets/core.key" } as const; - const ConsoleAccountContext = createContext<{ logout: () => Promise } | null>(null); export const useConsoleAccount = () => useContext(ConsoleAccountContext); @@ -146,8 +139,8 @@ function CoreKeyForm({ onAuthenticated }: { readOnly={busy} aria-invalid={error ? true : undefined} aria-describedby={`${id}-location${error ? ` ${id}-error` : ""}`} />
-

{t("The installer saved the key in {{file}} inside the installation directory. On the Core host, read the default location with:", CORE_KEY_LOCATION)}

- +

{t("On the Core host, run this command to read the Core key:")}

+

{t("For a custom installation directory, replace the path in this command.")}

{error ? : null} diff --git a/apps/web/src/i18n/locales/en/core-errors.ts b/apps/web/src/i18n/locales/en/core-errors.ts index ba11e4086..8b86a85a3 100644 --- a/apps/web/src/i18n/locales/en/core-errors.ts +++ b/apps/web/src/i18n/locales/en/core-errors.ts @@ -1,5 +1,5 @@ export const coreErrors = { - "invalid_admin_key": "The console's Core key was rejected. Update secrets/core.key on the Core host and run oac apply.", + "invalid_admin_key": "The console's Core key was rejected. Rotate it on the Core host, then sign in again.", "console_sign_in_required": "Sign in to the console again.", "console_origin_rejected": "Open the console at its configured address.", "console_request_invalid": "The console request was rejected. Reload the page.", diff --git a/apps/web/src/i18n/locales/zh-CN/core-errors.ts b/apps/web/src/i18n/locales/zh-CN/core-errors.ts index 4334ca3c1..310ade305 100644 --- a/apps/web/src/i18n/locales/zh-CN/core-errors.ts +++ b/apps/web/src/i18n/locales/zh-CN/core-errors.ts @@ -1,5 +1,5 @@ export const coreErrors = { - "invalid_admin_key": "控制台的 Core Key 被拒绝。请更新 Core 主机上的 secrets/core.key，再运行 oac apply。", + "invalid_admin_key": "控制台的 Core Key 被拒绝。请在 Core 主机上轮换密钥，然后重新登录。", "console_sign_in_required": "请重新登录控制台。", "console_origin_rejected": "请通过配置的地址打开控制台。", "console_request_invalid": "控制台请求被拒绝，请重新加载页面。", diff --git a/apps/web/src/lib/console-auth-strings.ts b/apps/web/src/lib/console-auth-strings.ts index c2c8afdf1..c6490daef 100644 --- a/apps/web/src/lib/console-auth-strings.ts +++ b/apps/web/src/lib/console-auth-strings.ts @@ -4,8 +4,8 @@ export const consoleAuthChinese = { "Core key": "Core Key", "The Core key is an administration credential: it cannot call the /v1 Agents API, and the console never keeps it in your browser.": "Core Key 是管理凭据：不能调用 /v1 Agents API，控制台也不会把它保存在你的浏览器里。", - "The installer saved the key in {{file}} inside the installation directory. On the Core host, read the default location with:": - "安装器将 key 保存在安装目录下的 {{file}} 中。在 Core 主机上运行以下命令可读取默认位置：", + "On the Core host, run this command to read the Core key:": + "在 Core 主机上运行以下命令，读取 Core Key：", "Copy key read command": "复制 key 读取命令", "For a custom installation directory, replace the path in this command.": "如果使用了自定义安装目录，请替换命令中的路径。", "Sign in": "登录", diff --git a/deploy/README.md b/deploy/README.md index 716971334..20a61d2e4 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -2,18 +2,18 @@ | Path | Contents | | --- | --- | -| `install.sh` | Host installer published with each release | +| `install.sh`, `install.ps1` | Native command launchers published with each release | | `compose/` | Compose template and its tests | | `distribution/` | Image Dockerfiles | | `node/` | [Node installer](node/README.md), packaged as `node-install.pyz` | ## Installation -`install.sh` downloads its release's `compose.yaml`, checks it against `compose-sha256sums.txt`, writes `.env`, and starts Compose. Core applies database migrations when it starts. The host needs Linux amd64 and Docker Compose 2.26 or newer. [Configuration](../docs/configuration.md) owns the installation layout and settings. +`install.sh` and `install.ps1` select a native `oac` release binary, verify its checksum and invoke `oac install`. That Go command owns the same installation lifecycle on Linux, macOS and Windows: validate Docker, verify release configuration, prepare `.env` and the host command, pull images, publish the installation directory, initialize the data volume and start Compose. [Installation](../docs/getting-started/install.md#prerequisites) owns platform prerequisites; [Configuration](../docs/configuration.md) owns settings and storage. -The Core installer validates downloads and Compose in the private sibling `.staging` directory, and pulls all images before publishing the installation directory. A retry clears leftover staging only when its atomically created ownership link names this installation, including after the previous process was killed. An unrecognized nonempty staging directory is preserved. The staging log moves with the configuration and is removed on normal exit. Once published, failures retain configuration, containers and data. A retry verifies and starts that saved installation without replacing its Compose file, cached images or existing containers. Missing images are downloaded explicitly; Compose never pulls images implicitly. It refuses unrelated nonempty directories. `.install.lock` serializes directory preparation; `.oac.lock` serializes service changes with mutating `oac` commands. Both lock files remain after exit. Download failures have bounded retries and timeouts; service startup failures print Compose status and recent logs. +Preparation happens in `.staging`. On entry, the installer removes an empty stage or one with the empty ownership directory `.oac-installer-`; it preserves unrecognized directories. Ordinary failures remove owned staging immediately. The ownership marker is created atomically before downloading files. A killed process leaves staging for the next attempt. Publication happens only after downloads, checks and pulls succeed. Later failures preserve saved configuration, containers and the Docker data volume. Retries reuse those settings and cached images, pull missing images and start services with `--pull never --no-recreate`. `.lock` is a persistent lock directory shared by installation and mutating operator commands; the existing `runtimefs` platform adapter owns locking. -`oac` is a Go command (`services/core/cmd/oac`) in the Core image and the ingress image. The host copy implements `apply`, `core-key` and `rotate-core-key`; `core-key --show` runs `oac-web core-key` in the Web container. Start, stop, logs and removal are `docker compose`. `apply` runs `oac-core check-config` before recreating services. The ingress image runs data initialization as `oac init`, verifies and copies its bundled node metadata without network access, and contains no Python. No service receives a Docker socket. Initialization writes structured logs to stderr for directory preparation, lock acquisition, existing-installation verification, bundled metadata verification and publication, credential generation or reuse, and receipt persistence. Each step records its start and completion; failures identify the current step, and completion records elapsed milliseconds. Credential values and digests are never logged. View these logs with `docker compose logs --timestamps init`. +The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` checks Core configuration before recreating services. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin. diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 21ecceea9..6fe45a4db 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -1,24 +1,23 @@ # Release template. The publisher pins the initialization image to this release. # Core and Web default to latest; OAC_IMAGE_* selects another reference. -# Data is bind-mounted from ${OAC_DATA_DIR:-./data}. +# Docker owns data permissions on every host platform. x-ingress-image: &ingress-image ${OAC_IMAGE_INGRESS:-__OAC_INIT_IMAGE__} services: init: image: *ingress-image - platform: linux/amd64 restart: "no" security_opt: [no-new-privileges:true] command: [/usr/local/bin/oac, init] environment: OAC_REVISION: __OAC_REVISION__ volumes: - - type: bind - source: ${OAC_DATA_DIR:-./data} + - type: volume + source: data target: /data + volume: {nocopy: true} database: image: postgres:16-alpine - platform: linux/amd64 restart: unless-stopped depends_on: init: {condition: service_completed_successfully} @@ -27,12 +26,14 @@ services: POSTGRES_DB: agents_api POSTGRES_PASSWORD_FILE: /run/database/password volumes: - - type: bind - source: ${OAC_DATA_DIR:-./data}/database + - type: volume + source: data target: /var/lib/postgresql/data - - type: bind - source: ${OAC_DATA_DIR:-./data}/secrets/database + volume: {nocopy: true, subpath: database} + - type: volume + source: data target: /run/database + volume: {nocopy: true, subpath: secrets/database} read_only: true healthcheck: test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U agents_api -d agents_api"] @@ -42,7 +43,6 @@ services: core: image: ${OAC_IMAGE_CORE:-ghcr.io/minimax-ai/openagentcore/core:latest} - platform: linux/amd64 user: "65532:65532" restart: unless-stopped read_only: true @@ -70,23 +70,28 @@ services: OAC_LOG_ADD_SOURCE: ${OAC_LOG_ADD_SOURCE:-} OAC_HISTORY_SETTINGS_FILE: ${OAC_HISTORY_SETTINGS_FILE:-} volumes: - - type: bind - source: ${OAC_DATA_DIR:-./data}/secrets/core + - type: volume + source: data target: /run/oac + volume: {nocopy: true, subpath: secrets/core} read_only: true - - type: bind - source: ${OAC_DATA_DIR:-./data}/secrets/database + - type: volume + source: data target: /run/database + volume: {nocopy: true, subpath: secrets/database} read_only: true - - type: bind - source: ${OAC_DATA_DIR:-./data}/state + - type: volume + source: data target: /state + volume: {nocopy: true, subpath: state} web: ports: - - "${OAC_HOST:-127.0.0.1}:${OAC_WEB_PORT:-8080}:8080" + - target: 8080 + published: "${OAC_WEB_PORT:-8080}" + host_ip: "${OAC_HOST:-127.0.0.1}" + protocol: tcp image: ${OAC_IMAGE_WEB:-ghcr.io/minimax-ai/openagentcore/web:latest} - platform: linux/amd64 user: "65532:65532" restart: unless-stopped read_only: true @@ -102,16 +107,21 @@ services: OAC_LOG_FORMAT: ${OAC_LOG_FORMAT:-} OAC_LOG_ADD_SOURCE: ${OAC_LOG_ADD_SOURCE:-} volumes: - - type: bind - source: ${OAC_DATA_DIR:-./data}/secrets/web + - type: volume + source: data target: /run/oac + volume: {nocopy: true, subpath: secrets/web} read_only: true - - type: bind - source: ${OAC_DATA_DIR:-./data}/node-payload + - type: volume + source: data target: /node-payload + volume: {nocopy: true, subpath: node-payload} read_only: true healthcheck: test: [CMD, /usr/local/bin/oac-web, healthcheck] interval: 5s timeout: 10s retries: 30 + +volumes: + data: diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 11f666e15..ce428f89a 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -61,7 +61,9 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertTrue(service['image'].endswith(':latest') or service['image'] == 'postgres:16-alpine' or service['image'].endswith('@sha256:' + 'e' * 64)) for volume in service.get('volumes', []): self.assertNotIn('docker.sock', json.dumps(volume)) - self.assertEqual(volume['type'], 'bind') + self.assertEqual(volume['type'], 'volume') + self.assertEqual(volume['source'], 'data') + self.assertNotIn('platform', service) self.assertEqual({v['target'] for v in services['web']['volumes']}, {'/run/oac', '/node-payload'}) self.assertIsNone(services['core']['command']) self.assertNotIn('OAC_WEB_INSTALLATION_SOCKET', services['web']['environment']) @@ -95,6 +97,12 @@ def ports(config): ['docker', 'compose', '--env-file', os.devnull, '-f', str(self.compose_file), 'config', '--format', 'json'], env=env)) self.assertEqual(ports(configured), {'web': [('0.0.0.0', '9080')]}) + env['OAC_HOST'] = '::1' + configured = json.loads(subprocess.check_output( + ['docker', 'compose', '--env-file', os.devnull, '-f', str(self.compose_file), + 'config', '--format', 'json'], env=env)) + self.assertEqual(ports(configured), {'web': [('::1', '9080')]}) + def test_platform_network_injection_keeps_the_file_valid(self): # Dokploy isolated deployments attach a project network to every service. diff --git a/deploy/install.dev.sh b/deploy/install.dev.sh index eada59aa3..7a3fb5135 100755 --- a/deploy/install.dev.sh +++ b/deploy/install.dev.sh @@ -47,8 +47,7 @@ if [[ "$install_dir" != /* ]]; then exit 1 fi -mkdir -p "$install_dir/data" -chmod 700 "$install_dir/data" +mkdir -p "$install_dir" build="$install_dir/image-build" rm -rf "$build" mkdir -p "$build" @@ -115,7 +114,6 @@ PY umask 077 cat >"$install_dir/.env" <:; otherwise only from this host. -HTTPS is terminated by your reverse proxy or hosting platform. -EOF -} - +case "$(uname -s)" in Linux) os=linux;; Darwin) os=darwin;; *) echo 'Use install.ps1 on Windows.' >&2; exit 1;; esac +case "$(uname -m)" in x86_64) arch=amd64;; arm64|aarch64) arch=arm64;; *) echo 'Unsupported CPU architecture.' >&2; exit 1;; esac +version=latest +args=("$@") while [[ $# -gt 0 ]]; do - case "$1" in - --version) version="${2:?}"; shift 2 ;; - --install-dir) install_dir="${2:?}"; shift 2 ;; - --public-url) public_url="${2:?}"; shift 2 ;; - --host) host_address="${2:?}"; shift 2 ;; - --web-port) web_port="${2:?}"; shift 2 ;; - -h|--help) usage; exit 0 ;; - *) echo "Unknown argument: $1" >&2; usage >&2; exit 1 ;; - esac + case "$1" in --version) version="${2:?Missing release tag}"; shift 2;; *) shift;; esac done - -if [[ "$(uname -s)" != Linux || "$(uname -m)" != x86_64 ]]; then - echo "Core installs on Linux amd64." >&2 - exit 1 -fi -fail() { printf '%s\n' "$*" >&2; exit 1; } -for tool in docker curl sha256sum flock readlink od sed awk grep; do - command -v "$tool" >/dev/null || fail "Required command missing: $tool. Install it and rerun this command. Docker needs Compose 2.26 or newer." -done -compose_version="$(docker compose version --short 2>/dev/null | sed 's/^v//' || true)" -major="${compose_version%%.*}" -minor="${compose_version#*.}" -minor="${minor%%.*}" -if [[ ! "$major" =~ ^[0-9]+$ || ! "$minor" =~ ^[0-9]+$ ]] || (( major < 2 || (major == 2 && minor < 26) )); then - fail "Docker Compose 2.26 or newer is required (found ${compose_version:-none})." -fi -docker info >/dev/null 2>&1 || fail "Cannot reach Docker. Start Docker and check this account's access, then rerun." -[[ "$install_dir" == /* && "$install_dir" != / ]] || fail "--install-dir must be an absolute directory other than /." -# These values are written as literal dotenv strings, never shell commands. -for value in "$install_dir" "$public_url" "$host_address" "$version"; do - [[ "$value" != *$'\n'* && "$value" != *$'\r'* && "$value" != *"'"* && "$value" != *\\* ]] || fail "Installation options cannot contain newlines, quotes or backslashes." -done -[[ "$web_port" =~ ^[0-9]{1,5}$ ]] && (( 10#$web_port >= 1 && 10#$web_port <= 65535 )) || fail "--web-port must be between 1 and 65535." -[[ ! -L "$install_dir" ]] || fail "Installation directory must not be a symbolic link." -[[ ! -e "$install_dir" || -d "$install_dir" ]] || fail "Installation path must be a directory." -case "$(basename "$install_dir")" in .|..) fail "--install-dir must name the installation directory, not . or ...";; esac +base="https://github.com/${OAC_REPOSITORY:-MiniMax-AI/OpenAgentCore}/releases/latest/download" +if [[ "$version" != latest ]]; then base="https://github.com/${OAC_REPOSITORY:-MiniMax-AI/OpenAgentCore}/releases/download/$version"; fi umask 077 -parent="$(dirname "$install_dir")" -mkdir -p "$parent" -parent="$(cd "$parent" && pwd -P)" -install_dir="$parent/$(basename "$install_dir")" -lock="$install_dir.install.lock" -[[ ! -L "$lock" && ( ! -e "$lock" || ( -f "$lock" && -O "$lock" ) ) ]] || fail "Invalid installation lock: $lock" -exec 9>>"$lock" -flock -n 9 || fail "Another installation is using this directory. Wait for it to finish and rerun." -stage="$install_dir.staging" -[[ ! -L "$stage" && ( ! -e "$stage" || ( -d "$stage" && -O "$stage" ) ) ]] || fail "Invalid installation staging directory: $stage" -if [[ -d "$stage" && -n "$(ls -A "$stage")" ]]; then - [[ -L "$stage/.oac-installer" && "$(readlink "$stage/.oac-installer")" == "$install_dir" ]] || fail "Unrecognized staging directory; preserve it and choose another --install-dir: $stage" -fi -rm -rf "$stage" -mkdir "$stage" -ln -s "$install_dir" "$stage/.oac-installer" -log="$stage/install.log" -: >"$log" -cleanup() { - local status=$? - trap - EXIT - if [[ "$status" != 0 && "$published" == 1 ]]; then - printf '\nInstallation and data retained at %s.\n' "$install_dir" >&2 - (cd "$install_dir" && docker compose ps --all && docker compose logs --no-color --tail 50) >&2 || true - printf 'Fix the reported problem, then rerun install.sh --install-dir %q.\n' "$install_dir" >&2 - fi - [[ -z "$stage" ]] || rm -rf "$stage" - rm -f "$log" - exit "$status" -} -trap cleanup EXIT -trap 'exit 130' INT -trap 'exit 143' TERM HUP - -# step DESCRIPTION COMMAND... prints the command's output only when it fails. -step() { - printf '%s... ' "$1" - shift - if "$@" >"$log" 2>&1; then echo done; else echo failed; cat "$log" >&2; return 1; fi -} - -resume=0 -if [[ -e "$install_dir" && -n "$(ls -A "$install_dir")" ]]; then - for file in compose.yaml compose-sha256sums.txt .env; do - [[ -f "$install_dir/$file" && ! -L "$install_dir/$file" ]] || fail "Directory is not a complete Core installation: $install_dir. Preserve it and choose another directory." - done - resume=1 - published=1 -fi - -port_busy() { - local port="$1" - if command -v ss >/dev/null; then - if ss -ltn | awk '{print $4}' | grep -Eq "(^|:|\\])${port}$"; then - return 0 - fi - return 1 - fi - (echo >/dev/tcp/127.0.0.1/"$port") >/dev/null 2>&1 -} -if [[ "$resume" == 0 ]] && port_busy "$web_port"; then fail "Port $web_port is already in use. Choose another with --web-port."; fi -asset_base="https://github.com/${repository}/releases/latest/download" -if [[ "$version" != latest ]]; then - asset_base="https://github.com/${repository}/releases/download/${version}" -fi - -# The source address of this host's default route, when it is a private one. -private_address() { - command -v ip >/dev/null || return 0 - ip -4 route get 1.1.1.1 2>/dev/null | sed -n 's/.* src \([0-9.]*\).*/\1/p' | - grep -E '^(10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)' || true -} -local_only=0 -if [[ -z "$public_url" && "$host_address" == 0.0.0.0 ]]; then - address="$(private_address)" - if [[ -n "$address" ]]; then public_url="http://$address:$web_port"; fi -fi -if [[ -z "$public_url" ]]; then public_url="http://localhost:$web_port"; local_only=1; fi - -if [[ "$resume" == 0 ]]; then - # Prepare configuration privately until downloads and image pulls succeed. - # Before publication only this invocation's private staging directory is removed. - download() { - curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' \ - --connect-timeout 15 --max-time 120 --retry 2 --retry-connrefused --retry-delay 1 \ - --max-filesize 1048576 "$asset_base/$1" --output "$stage/$1" - } - step "Downloading release checksums" download compose-sha256sums.txt - step "Downloading Compose configuration" download compose.yaml - { - echo "COMPOSE_PROJECT_NAME=oac-$(od -An -N5 -tx1 /dev/urandom | tr -d ' \n')" - printf "OAC_INSTALL_DIR='%s'\nOAC_HOST='%s'\nOAC_WEB_PORT='%s'\nOAC_PUBLIC_URL='%s'\n" "$install_dir" "$host_address" "$web_port" "$public_url" - } >"$stage/.env" -fi - -if [[ "$resume" == 0 ]]; then cd "$stage"; else cd "$install_dir"; fi -[[ ! -L .oac.lock && ( ! -e .oac.lock || ( -f .oac.lock && -O .oac.lock ) ) ]] || fail "Invalid installation lock: $install_dir/.oac.lock" -exec 8>>.oac.lock -flock -n 8 || fail "Another oac command is using this installation. Wait for it to finish and rerun." -[[ ! -L install.log && ( ! -e install.log || ( -f install.log && -O install.log ) ) ]] || fail "Invalid installation log: $install_dir/install.log" -log="$PWD/install.log" -: >"$log" -step "Verifying saved configuration" sha256sum --check --quiet compose-sha256sums.txt -step "Checking Compose configuration" docker compose config --quiet -if [[ "$resume" == 1 ]]; then - printf 'Using saved settings from .env; installation flags only apply to new directories. Existing data is preserved.\n' -fi -public_url="$(docker compose config --environment | sed -n 's/^OAC_PUBLIC_URL=//p')" -local_only=0 -[[ "$public_url" != http://localhost:* && "$public_url" != http://127.0.0.1:* ]] || local_only=1 -pull_images() { - local images image - images="$(docker compose config --images)" || return - while IFS= read -r image; do - [[ -n "$image" ]] || continue - if [[ "$resume" == 0 ]] || ! docker image inspect "$image" >/dev/null 2>&1; then - docker pull --platform linux/amd64 "$image" || return - fi - done <<<"$images" -} -step "Checking and downloading images" pull_images -if [[ "$resume" == 0 ]]; then - # An existing empty directory may be replaced, never a directory with user data. - if [[ -e "$install_dir" ]]; then rmdir "$install_dir"; fi - mv "$stage" "$install_dir" - stage="" - cd "$install_dir" - log="$install_dir/install.log" - published=1 -fi -rm -f .oac-installer -copy_cli() ( - [[ ! -L ./oac.download ]] || fail "Temporary oac command must not be a symbolic link." - trap 'rm -f ./oac.download' EXIT - docker compose create --pull never --no-recreate core && - docker compose cp core:/usr/local/bin/oac ./oac.download && - mv ./oac.download ./oac -) -if [[ ! -x ./oac ]]; then step "Installing the oac command" copy_cli; fi -step "Starting services" docker compose up -d --wait --wait-timeout 180 --pull never --no-recreate -if ! key="$(./oac core-key --show)"; then - fail "Services started, but the Core key could not be read. Inspect Web's logs and retry; data is preserved." -fi - -sudo="" -if [[ "$EUID" == 0 && -n "${SUDO_USER:-}" ]]; then sudo="sudo "; fi -cat </dev/null; then actual="$(sha256sum "$asset")"; else actual="$(shasum -a 256 "$asset")"; fi +[[ "$actual" == "$(cat "$asset.sha256")" ]] || { echo 'Installer checksum mismatch.' >&2; exit 1; } +chmod 700 "$asset" +"./$asset" install "${args[@]}" diff --git a/deploy/test_install.ps1 b/deploy/test_install.ps1 new file mode 100644 index 000000000..bed472823 --- /dev/null +++ b/deploy/test_install.ps1 @@ -0,0 +1,34 @@ +# Exercise the actual launcher with a native oac binary and local release assets. +param([Parameter(Mandatory=$true)][string]$Binary) +$ErrorActionPreference = 'Stop' +$testAssets = @{ Binary = $Binary; Corrupt = $false } +function Invoke-WebRequest { + param([switch]$UseBasicParsing, [string]$Uri, [string]$OutFile) + if ($Uri.EndsWith('.sha256')) { + $digest = (Get-FileHash -Algorithm SHA256 $testAssets.Binary).Hash.ToLowerInvariant() + if ($testAssets.Corrupt) { $digest = '0' * 64 } + [IO.File]::WriteAllText($OutFile, "$digest oac-windows-amd64.exe`n") + } else { + Copy-Item $testAssets.Binary $OutFile + } +} +& "$PSScriptRoot/install.ps1" --help + +$testAssets.Corrupt = $true +$caught = $false +try { & "$PSScriptRoot/install.ps1" --help } +catch { + if ($_.Exception.Message -notlike '*checksum mismatch*') { throw } + $caught = $true +} +if (-not $caught) { throw 'A corrupt binary was executed.' } + +$testAssets.Corrupt = $false +$caught = $false +try { & "$PSScriptRoot/install.ps1" --unknown-option } +catch { + if ($_.Exception.Message -notlike '*Installation failed*') { throw } + $caught = $true +} +if (-not $caught) { throw 'A failed native command was reported as successful.' } +exit 0 diff --git a/deploy/test_install.py b/deploy/test_install.py index f4bb80cdd..5b3a3ec05 100644 --- a/deploy/test_install.py +++ b/deploy/test_install.py @@ -1,294 +1,55 @@ -"""install.sh writes .env and starts Compose without host Python.""" +"""The POSIX launcher selects and verifies a binary; lifecycle tests live in oac.""" +import hashlib import os -import fcntl -import sys from pathlib import Path -import stat import subprocess import tempfile -import textwrap import unittest - -ROOT = Path(__file__).resolve().parents[1] -INSTALL = ROOT / "deploy/install.sh" - - -class InstallScriptTests(unittest.TestCase): - def install(self, root, *args, compose_up=0, docker_info=0, key_status=0, download_status=0, kill_download=False, kill_start=False, pull_status=0, kill_pull=False, file_limit=False, missing_image="", route="1.1.1.1 via 10.0.0.1 dev eth0 src 10.0.0.5 uid 0"): - bin_dir = root / "bin" - bin_dir.mkdir(exist_ok=True) - log = root / "docker.log" - self.write_executable(bin_dir / "docker", textwrap.dedent(f"""\ - #!/bin/sh - [ {1 if file_limit else 0} -eq 1 ] || printf '%s\\n' "$*" >> {log} - if [ "$1" = info ]; then exit {docker_info}; fi - if [ "$1" = pull ]; then {'kill -KILL "$PPID"' if kill_pull else ':'}; exit {pull_status}; fi - if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = "{missing_image}" ]; then exit 1; fi - if [ "$1" = compose ] && [ "$2" = config ] && [ "$3" = --images ]; then printf 'fixture-core:latest\\nfixture-web:latest\\n'; fi - if [ "$1" = compose ] && [ "$2" = logs ]; then echo service-diagnostic >&2; fi - if [ "$1" = compose ] && [ "$2" = config ] && [ "$3" = --environment ]; then sed "s/'//g" .env; fi - if [ "$1" = compose ] && [ "$2" = version ]; then printf 'v2.29.1\\n'; exit 0; fi - if [ "$1" = compose ] && [ "$2" = cp ]; then printf '#!/bin/sh\\necho oac_core_fixture\\nexit {key_status}\\n' > ./oac.download; chmod +x ./oac.download; exit 0; fi - if [ "$1" = compose ] && [ "$2" = up ]; then {'kill -KILL "$PPID"' if kill_start else ':'}; exit {compose_up}; fi - exit 0 - """)) - self.write_executable(bin_dir / "curl", textwrap.dedent(f"""\ - #!/bin/sh - exit_status={download_status} - [ "$exit_status" = 0 ] || exit "$exit_status" - output="" - while [ $# -gt 0 ]; do - if [ "$1" = --output ]; then output="$2"; shift 2; continue; fi - shift - done - printf 'fixture\\n' > "$output" - {'kill -KILL "$PPID"' if kill_download else ':'} - """)) - self.write_executable(bin_dir / "sha256sum", "#!/bin/sh\nexit 0\n") - self.write_executable(bin_dir / "uname", '#!/bin/sh\ncase "$1" in -s) echo Linux;; -m) echo x86_64;; esac\n') - self.write_executable(bin_dir / "flock", f'#!{sys.executable}\nimport fcntl, sys\ntry: fcntl.flock(int(sys.argv[-1]), fcntl.LOCK_EX | fcntl.LOCK_NB)\nexcept BlockingIOError: sys.exit(1)\n') - self.write_executable(bin_dir / "ss", "#!/bin/sh\nexit 0\n") - self.write_executable(bin_dir / "ip", f"#!/bin/sh\nprintf '%s\\n' '{route}'\n") - env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], HOME=str(root)) - command = ["bash", str(INSTALL), "--install-dir", str(root / "oac"), *args] - if file_limit: - command = ["bash", "-c", 'ulimit -f 0; exec "$@"', "--", *command] - completed = subprocess.run(command, - env=env, capture_output=True, text=True) - return completed, log.read_text() if log.exists() else "" - - def test_a_failed_start_preserves_configuration_and_reports_service_logs(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, recorded = self.install(root, "--host", "127.0.0.1", "--web-port", "59991", - "--public-url", "https://core.example", compose_up=1) - self.assertNotEqual(completed.returncode, 0, completed.stderr) - self.assertTrue((root / "oac/.env").exists()) - self.assertNotIn("compose down", recorded) - self.assertIn("service-diagnostic", completed.stderr) - self.assertIn("data retained", completed.stderr) - self.assertIn("pull --platform linux/amd64 fixture-core:latest", recorded) - self.assertIn("compose up -d --wait", recorded) - self.assertIn("compose logs", recorded, "a failed start must show the services' logs") - - def test_the_private_address_is_the_default_public_url(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertIn("OAC_PUBLIC_URL='http://10.0.0.5:8080'\n", (root / "oac/.env").read_text()) - self.assertIn("Console http://10.0.0.5:8080", completed.stdout) - self.assertIn("Core key oac_core_fixture", completed.stdout) - self.assertNotIn("Only this host", completed.stdout) - - def test_without_a_private_address_only_this_host_reaches_web(self): - for args, route in ((["--web-port", "59992"], "1.1.1.1 dev eth0 src 203.0.113.5 uid 0"), - (["--host", "127.0.0.1", "--web-port", "59992"], "1.1.1.1 dev eth0 src 10.0.0.5 uid 0")): - with self.subTest(args=args), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, _ = self.install(root, *args, route=route) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertIn("OAC_PUBLIC_URL='http://localhost:59992'\n", (root / "oac/.env").read_text()) - self.assertIn("Only this host can open the console", completed.stdout) - - def test_env_holds_only_the_installation_choices(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, _ = self.install(root, "--public-url", "https://core.example") - self.assertEqual(completed.returncode, 0, completed.stderr) - env = dict((lambda pair: (pair[0], pair[1].strip("'")))(line.split("=", 1)) for line in (root / "oac/.env").read_text().splitlines()) - self.assertEqual(env["OAC_PUBLIC_URL"], "https://core.example") - self.assertEqual(sorted(env), ["COMPOSE_PROJECT_NAME", "OAC_HOST", "OAC_INSTALL_DIR", "OAC_PUBLIC_URL", "OAC_WEB_PORT"]) - - def test_key_failure_never_removes_a_started_installation(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, recorded = self.install(root, key_status=1) - self.assertNotEqual(completed.returncode, 0) - self.assertTrue((root / "oac/.env").exists()) - self.assertNotIn("compose down", recorded) - self.assertIn("Core key could not be read", completed.stderr) - - def test_retry_reuses_saved_settings_and_keeps_data(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - failed, _ = self.install(root, "--public-url", "https://core.example", compose_up=1) - self.assertNotEqual(failed.returncode, 0) - configuration = (root / "oac/.env").read_bytes() - data = root / "oac/data" - data.mkdir() - (data / "keep").write_text("user data") - (root / "docker.log").unlink() - completed, recorded = self.install(root, download_status=22) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual((root / "oac/.env").read_bytes(), configuration) - self.assertEqual((data / "keep").read_text(), "user data") - self.assertIn("Console https://core.example", completed.stdout) - self.assertNotIn("pull --platform", recorded) - self.assertNotIn("Only this host", completed.stdout) - - def test_retry_downloads_only_missing_images_and_prevents_implicit_updates(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - failed, _ = self.install(root, compose_up=1) - self.assertNotEqual(failed.returncode, 0) - (root / "docker.log").unlink() - (root / "oac/oac").unlink() - completed, recorded = self.install(root, missing_image="fixture-web:latest") - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertNotIn("pull --platform linux/amd64 fixture-core:latest", recorded) - self.assertIn("pull --platform linux/amd64 fixture-web:latest", recorded) - for line in recorded.splitlines(): - if line.startswith(("compose create", "compose up")): - self.assertIn("--pull never", line) - self.assertIn("--no-recreate", line) - - def test_retry_after_file_size_limit_recovers_staging(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - failed, _ = self.install(root, file_limit=True) - self.assertNotEqual(failed.returncode, 0) - self.assertIn("Downloading release checksums", failed.stdout) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertFalse((root / "oac.staging").exists()) - - def test_failed_initial_pull_never_adopts_old_cached_images_on_retry(self): - for kill in (False, True): - with self.subTest(kill=kill), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - failed, _ = self.install(root, pull_status=1, kill_pull=kill) - self.assertNotEqual(failed.returncode, 0) - self.assertFalse((root / "oac").exists()) - (root / "docker.log").unlink() - # Every image inspect succeeds: another installation cached old tags. - completed, recorded = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - for image in ("fixture-core:latest", "fixture-web:latest"): - self.assertIn("pull --platform linux/amd64 " + image, recorded) - self.assertFalse((root / "oac.staging").exists()) - - def test_invalid_port_and_unavailable_docker_fail_before_installation(self): - for args, status in [(["--web-port", "0"], 0), (["--web-port", "70000"], 0), ([], 1)]: - with self.subTest(args=args, status=status), tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, recorded = self.install(root, *args, docker_info=status) - self.assertNotEqual(completed.returncode, 0) - self.assertFalse((root / "oac").exists()) - self.assertNotIn("pull --platform", recorded) - - def test_failed_download_leaves_no_installation_or_staging(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, recorded = self.install(root, download_status=22) - self.assertNotEqual(completed.returncode, 0) - self.assertFalse((root / "oac").exists()) - self.assertFalse((root / "oac.staging").exists()) - self.assertNotIn("compose down", recorded) - - def test_retry_after_sigkill_cleans_unfinished_download(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - killed, _ = self.install(root, kill_download=True) - self.assertEqual(killed.returncode, -9, killed.stderr) - self.assertTrue((root / "oac.staging").exists()) - self.assertFalse((root / "oac").exists()) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertFalse((root / "oac.staging").exists()) - self.assertTrue((root / "oac/oac").exists()) - - def test_retry_after_start_sigkill_cleans_published_log(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - killed, _ = self.install(root, kill_start=True) - self.assertEqual(killed.returncode, -9, killed.stderr) - self.assertTrue((root / "oac/install.log").exists()) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertFalse((root / "oac/install.log").exists()) - self.assertFalse((root / "oac.staging").exists()) - - def test_retry_clears_staging_from_an_interrupted_process(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - staging = root / "oac.staging" - staging.mkdir() - (staging / ".oac-installer").symlink_to((root / "oac").resolve()) - (staging / "partial").write_text("unfinished download") - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertFalse(staging.exists()) - self.assertFalse((root / "oac/partial").exists()) - - def test_unrecognized_staging_directory_is_never_deleted(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - staging = root / "oac.staging" - staging.mkdir() - (staging / "keep").write_text("user data") - completed, recorded = self.install(root) - self.assertNotEqual(completed.returncode, 0) - self.assertIn("Unrecognized staging", completed.stderr) - self.assertEqual((staging / "keep").read_text(), "user data") - self.assertNotIn("compose up", recorded) - - def test_existing_unrelated_directory_is_never_deleted(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - (root / "oac").mkdir() - (root / "oac/keep").write_text("user data") - completed, recorded = self.install(root) - self.assertNotEqual(completed.returncode, 0) - self.assertEqual((root / "oac/keep").read_text(), "user data") - self.assertNotIn("compose down", recorded) - - def test_another_installation_holds_the_lock(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - with (root / "oac.install.lock").open("w") as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - completed, recorded = self.install(root) - self.assertNotEqual(completed.returncode, 0) - self.assertIn("Another installation", completed.stderr) - self.assertFalse((root / "oac").exists()) - self.assertNotIn("pull --platform", recorded) - - def test_retry_does_not_overlap_an_oac_mutation(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - (root / "docker.log").unlink() - with (root / "oac/.oac.lock").open("r+") as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - completed, recorded = self.install(root) - self.assertNotEqual(completed.returncode, 0) - self.assertIn("Another oac command", completed.stderr) - self.assertNotIn("compose up", recorded) - - def test_rerun_does_not_silently_replace_settings(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - completed, _ = self.install(root) - self.assertEqual(completed.returncode, 0, completed.stderr) - before = (root / "oac/.env").read_bytes() - completed, _ = self.install(root, "--web-port", "9000") - self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertIn("Using saved settings", completed.stdout) - self.assertIn("only apply to new directories", completed.stdout) - self.assertEqual((root / "oac/.env").read_bytes(), before) - - def test_help_does_not_need_docker(self): - help_text = subprocess.run(["bash", str(INSTALL), "--help"], capture_output=True, text=True, check=True) - self.assertIn("--web-port", help_text.stdout) - self.assertNotIn("--external-proxy", help_text.stdout) - - def write_executable(self, path, text): - path.write_text(text) - path.chmod(path.stat().st_mode | stat.S_IEXEC) - - -if __name__ == "__main__": +SCRIPT = Path(__file__).with_name('install.sh').resolve() + + +class LauncherTests(unittest.TestCase): + def run_launcher(self, system='Darwin', machine='arm64', corrupt=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + binary = b'#!/bin/sh\nprintf "%s\\n" "$@"\n' + digest = hashlib.sha256(binary).hexdigest() + scripts = { + 'uname': '#!/bin/sh\nif [ "$1" = -s ]; then echo ' + system + '; else echo ' + machine + '; fi\n', + 'curl': '''#!/usr/bin/env python3 +import pathlib,sys +args=sys.argv +url=args[-3]; target=pathlib.Path(args[-1]) +asset=url.rsplit('/',1)[1] +if asset.endswith('.sha256'): + target.write_text(DIGEST + ' ' + asset.removesuffix('.sha256') + '\\n') +else: + target.write_bytes(BINARY) +'''.replace('DIGEST', repr('0' * 64 if corrupt else digest)).replace('BINARY', repr(binary)), + } + for name, contents in scripts.items(): + (root / name).write_text(contents); (root / name).chmod(0o700) + return subprocess.run(['bash', str(SCRIPT), '--install-dir', '/path with spaces/core', '--version', 'v1.2.3'], + env={**os.environ, 'PATH': str(root) + os.pathsep + os.environ['PATH']}, capture_output=True, text=True) + + def test_linux_and_mac_share_argument_forwarding(self): + for system, machine in [('Linux', 'x86_64'), ('Linux', 'aarch64'), ('Darwin', 'arm64'), ('Darwin', 'x86_64')]: + with self.subTest(system=system, machine=machine): + result = self.run_launcher(system, machine) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), ['install', '--install-dir', '/path with spaces/core', '--version', 'v1.2.3']) + + def test_corrupt_binary_is_never_executed(self): + result = self.run_launcher(corrupt=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('checksum mismatch', result.stderr) + self.assertEqual(result.stdout, '') + + def test_unsupported_host_fails_before_downloading(self): + result = self.run_launcher(system='FreeBSD') + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': unittest.main() diff --git a/docs/configuration.md b/docs/configuration.md index f91a35562..fcdc5b4a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,7 +9,7 @@ Every setting of a Core installation has exactly one home. There are two kinds: | [Process settings](#process-settings-configjson) | Public URL, ports, logging, harnesses, execution concurrency, audit retention, OAuth origins, Runtime history export | `.env` in the installation directory (default `~/.oac/core`) | Edit `.env`, then run `oac apply` | `oac apply` recreates the services that read the changed settings | | [Runtime settings](#runtime-settings-web) | Sandbox backend and size, nodes, Projects and keys, default models, executor credentials | Core's PostgreSQL database | Web, or the Core API (`/core/v1`) with the Core key | Saved without a Core restart; nodes prepare Runtime changes asynchronously | -Web's **System** page shows the installation's addresses, the default models, the sandbox configuration and, under **Startup settings**, the process settings Core loaded. Secrets live in [`data/secrets/`](#installation-directory), one copy each. No configuration file defines Projects or API keys. +Web's **System** page shows the installation's addresses, the default models, the sandbox configuration and, under **Startup settings**, the process settings Core loaded. Secrets live in [`secrets/`](#compose-installations), one copy each. No configuration file defines Projects or API keys. ## Process settings {#process-settings-configjson} @@ -23,9 +23,8 @@ Installer flags in [installation options](./getting-started/install-options.md) 1. It runs `oac-core check-config` with the `.env` you edited and changes nothing if a value is invalid. 2. It runs `docker compose up -d --wait`. Compose recreates only the services whose configuration changed. -3. If the check fails, no container is recreated. See [stop and restart](./getting-started/operations.md#stop-and-restart) for what a restart interrupts. -`docker compose ps` shows the services. Domain state is `data/domain/status.json`. +Use `docker compose ps` to check the services. See [stop and restart](./getting-started/operations.md#stop-and-restart) for what a restart interrupts. ### Changing the public URL {#changing-the-public-url} @@ -44,7 +43,7 @@ To change it, point the reverse proxy at the new address first, then edit `OAC_P | Variable | Default | Meaning | | --- | --- | --- | | `OAC_PUBLIC_URL` | `http://localhost:8080` | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) | -| `OAC_HOST` | `127.0.0.1` | Web bind address published by `compose.yaml`. `install.sh` sets `0.0.0.0` | +| `OAC_HOST` | `127.0.0.1` | Web bind address published by `compose.yaml`. The installer sets `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | @@ -81,7 +80,7 @@ Core approves a node's capacity when you generate its Add node command: **Sandbo ### Default models -Set a default in **System** → **Default model configuration**, or use `PUT /core/v1/harnesses/{harness}/model-configuration`. Core encrypts provider keys with `secrets/credential.key` and never returns them. [Model execution](../contracts/agents-api/model-execution.md#deployment-defaults) owns the request fields and replacement rules, and [precedence](../contracts/agents-api/model-execution.md#saved-defaults-and-precedence) says which Sessions use a default. +Set a default in **System** → **Default model configuration**, or use `PUT /core/v1/harnesses/{harness}/model-configuration`. Core encrypts provider keys with `secrets/core/credential.key` and never returns them. [Model execution](../contracts/agents-api/model-execution.md#deployment-defaults) owns the request fields and replacement rules, and [precedence](../contracts/agents-api/model-execution.md#saved-defaults-and-precedence) says which Sessions use a default. ## Compose installations @@ -100,7 +99,7 @@ The initialization service generates secrets and the installation ID once, then Initialization prepares this directory; application services receive their secret directories read-only. `docker compose exec web oac-web core-key` prints the Core key to the operator terminal without writing it to container logs. Database passwords and credential encryption keys are never printed. -`OAC_DATA_DIR` selects the directory and defaults to `./data` beside the Compose file. Preserve it together with that project's definition and public URL. Removing only the secret directories does not reset an installation; initialization refuses to start over an existing database. Core also binds the installation ID to its database. Runtime settings continue to live in [Core's database](#runtime-settings-web). +The named Docker volume `_data` contains these paths. Docker manages Linux ownership on every host; each service mounts only its required subdirectories. Preserve this volume together with the project definition and public URL. Removing only the secret directories does not reset an installation; initialization refuses to start over an existing database. Core also binds the installation ID to its database. Runtime settings continue to live in [Core's database](#runtime-settings-web). ## Docker node configuration @@ -118,37 +117,31 @@ The [Docker adapter](./sandbox-provider.md#docker-adapter) owns container isolat ## Installation directory -The installer creates the installation directory, `~/.oac/core` by default, with mode `0700`. Secret files are `0600`. +The installer creates `~/.oac/core` by default (`$HOME/.oac/core` on Windows). Its files contain process settings and the native operator command; persistent service data lives in the [Compose data volume](#compose-installations). | Path | Content | Changed by | | --- | --- | --- | -| `.env` | [Process settings](#process-settings-configjson). The file you edit | You, then `oac apply` | -| `compose.yaml` | The release's service definition. Do not edit them | The release | -| `oac` | The [management command](./getting-started/operations.md#the-oac-command), copied from the Core image | The installer | -| `data/secrets/web/core.key` | The [Core key](./getting-started/operations.md#core-key) | `oac rotate-core-key` | -| `data/secrets/core/credential.key` | Encryption key for what Core stores sealed in the database | Nothing. Keep it with the database | -| `data/secrets/core/core-key-digests.json` | SHA-256 of the Core key | `oac rotate-core-key` | -| `data/secrets/database/password` | PostgreSQL password | Nothing. PostgreSQL reads it only when the database is created | -| `data/database/` | PostgreSQL data | PostgreSQL | -| `data/node-payload/` | Node files Web serves at `/node-install/` | Initialization | -| `data/state/` | Private Provider state, including E2B receipts | Core | -| `.oac.lock` | The installation lock | Mutating `oac` commands | - -The Compose project is named `oac-<10 hex digits>`. Its services are `init`, `database`, `core` and `web`. Core applies database migrations when it starts. `web` serves the console and forwards `/v1` and `/api/v1` to Core, and it is the only service with a published port, `OAC_WEB_PORT`. No service receives a Docker socket. Apart from Docker's storage, nothing is written outside the installation directory. +| `.env` | Process settings and the stable Compose project name | You, then `oac apply` | +| `compose.yaml`, `compose-sha256sums.txt` | Verified release service definition | The release | +| `oac` (`oac.exe` on Windows) | Native management command | The installer | + +The sibling `.lock` directory remains for synchronization; `.staging` holds unpublished installation files. Neither contains service data. On Unix the installer creates private directories with mode `0700` and configuration files with mode `0600`. + +The Compose project is named `oac-<10 hex digits>`. Its services are `init`, `database`, `core` and `web`. Core applies database migrations when it starts. Web serves the console and forwards `/v1` and `/api/v1` to Core; it is the only service with a published port, `OAC_WEB_PORT`. No service receives a Docker socket. ## Appendix: Core environment without the installer -Core reads only its environment. Compose interpolates `.env` into the service environment. Compose must be 2.26.0 or newer. If you run Core yourself, set these variables; see the [service guide](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md). +Core reads its process environment. Compose interpolates `.env` into it and mounts secrets at the container paths below. When running Core directly, set the file variables to absolute paths readable by the Core process; see the [service guide](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md). | Variable | Set from | | --- | --- | | `OAC_PUBLIC_URL` | The public origin. Core derives the daemon WebSocket URL, the self-hosted `remote_url`, the hosted sandbox address and the deployment's read-only `core_url` from it, never from request headers. Without it, Core runs no Runtime gateway and executes no Sessions | | `OAC_ADDR` | The image sets `:8091`. Independently started Core defaults to `127.0.0.1:8091` when unset or empty | | `OAC_DATABASE_URL` | PostgreSQL without a password | -| `OAC_DATABASE_PASSWORD_FILE` | `data/secrets/database/password`. The URL must then carry no password | -| `OAC_CREDENTIAL_KEY_FILE` | `data/secrets/core/credential.key` | -| `OAC_CORE_KEY_DIGESTS_FILE` | `data/secrets/core/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | -| `OAC_INSTALLATION_ID_FILE` | `data/secrets/core/installation.id`: the installation ID, a canonical UUID. It enables the sandbox deployment and node routes and requires `OAC_PUBLIC_URL` and `OAC_CORE_KEY_DIGESTS_FILE`. Core refuses an ID other than the one its database recorded | +| `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`. The URL must then carry no password | +| `OAC_CREDENTIAL_KEY_FILE` | `/run/oac/credential.key` | +| `OAC_CORE_KEY_DIGESTS_FILE` | `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | +| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`: the installation ID, a canonical UUID. It enables the sandbox deployment and node routes and requires `OAC_PUBLIC_URL` and `OAC_CORE_KEY_DIGESTS_FILE`. Core refuses an ID other than the one its database recorded | | `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS` | The matching [process settings](#settings). `oac-core check-config` validates them without starting Core | | `OAC_HISTORY_SETTINGS_FILE` | Optional Runtime history file. Sensitive; the installation report says only whether it is set | | `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | Logging; Web reads the same three | @@ -162,7 +155,7 @@ Invalid explicit OAuth trusted origins stop Core at startup. Entries must be HTT ## Appendix: Web environment without the installer -Compose sets these for Web. Set them yourself only when you run the console without Compose. Of the installation's secrets, Web receives only `data/secrets/web/core.key`. +Compose sets these for Web. Set them yourself only when you run the console without Compose. Compose mounts the data volume's `secrets/web/` at `/run/oac` and sets `OAC_WEB_CORE_KEY_FILE=/run/oac/core.key`. | Variable | Default | Meaning | | --- | --- | --- | diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 41c4e50e7..08cdd34e5 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -2,7 +2,7 @@ title: "Installation options and advanced deployments" --- -The [default installation](./install.md) needs no options. Use this page to run behind an existing reverse proxy or install without internet access. +The [default installation](./install.md) needs no options. Use this page to set installation options, deploy with Compose or configure a reverse proxy. Pass options to the downloaded script: @@ -10,11 +10,11 @@ Pass options to the downloaded script: ./install.sh --public-url https://core.example ``` -With the one-line command, append them after `bash -s --`. `--version TAG` selects a published release; otherwise the script selects the latest stable release and verifies each Compose file's SHA-256. A failed step stops installation without a success message. +On Windows, download `install.ps1` and pass the same flags with `& ./install.ps1 --public-url https://core.example`. With the Unix one-line command, append them after `bash -s --`. `--version TAG` selects a published release; otherwise the script selects the latest stable release and verifies the native command and Compose files against their SHA-256 checksums. ## Docker Compose and hosting platforms -Use the `compose.yaml` from a release with Docker Compose 2.26 or newer on Linux amd64. The release pins its initialization image and source revision in the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). Core and Web use the `latest` images, and PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data is bind-mounted from a directory. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. +Use the `compose.yaml` from a release on any [supported Core host](./install.md#prerequisites). The release pins its initialization image and source revision in the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). Core and Web use the `latest` images, and PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data lives in a named Docker volume. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. For a local trial, download `compose.yaml` from a release into an empty directory, then run: @@ -23,7 +23,7 @@ docker compose up -d --wait --wait-timeout 900 docker compose exec web oac-web core-key ``` -`oac-web core-key` prints the generated Core key to your terminal without writing it to container logs. Open `http://localhost:8080` and use that key to sign in. All installation secrets are generated automatically; keep the same Compose project and its data directory when restarting. +`oac-web core-key` prints the generated Core key to your terminal without writing it to container logs. Open `http://localhost:8080` and use that key to sign in. All installation secrets are generated automatically; keep the same Compose project and its data volume when restarting. The initialization image contains only the small node installation metadata alongside the initialization command. First startup verifies and copies that metadata without downloading the control archive or requiring access to GitHub Releases. Later starts verify the saved files. Container images still need to be pulled. An interrupted first initialization can be rerun; an existing database with missing installation secrets is refused. @@ -41,7 +41,7 @@ On either platform, open its server terminal and run `docker compose ls` to find After signing in, choose the sandbox backend and add nodes using [Nodes](./nodes.md). The Compose stack deploys the control plane; execution machines remain separate. -Stop with `docker compose stop` using the same files and environment. Back up the data directory together while the services are stopped. Follow the [installation version policy](./operations.md#installation-version-policy): a different release needs a new Compose project and a fresh data directory. +Stop with `docker compose stop` using the same files and environment. Back up the data volume and configuration together while the services are stopped. Follow the [installation version policy](./operations.md#installation-version-policy): a different release needs a new Compose project and a fresh data volume. ## Process settings @@ -60,7 +60,7 @@ These flags are written to `.env` once. After installation, edit that file and r | Option | Purpose | | --- | --- | -| `--install-dir DIR` | Absolute installation directory; defaults to `~/.oac/core`. A new installation requires an empty or missing directory, or one holding an [installation that never started](./install.md#install) | +| `--install-dir DIR` | Absolute installation directory; defaults to `~/.oac/core`. Use an empty or missing directory for a new installation, or an existing installation directory to [retry](./install.md#install) | Several installations can share a machine when they use distinct installation directories and ports. Use distinct IP addresses or a shared reverse proxy for more. Each installation has its own database, Core key and nodes. @@ -131,4 +131,4 @@ The address changes whenever `cloudflared` restarts; nodes bound to the old addr ## Offline hosts -This installer does not install from an offline bundle. It downloads Compose files and container images from the release. +Core installation requires access to GitHub Releases and the container registries to download release files and images. diff --git a/docs/getting-started/install.md b/docs/getting-started/install.md index acf59bd48..9e1b17e73 100644 --- a/docs/getting-started/install.md +++ b/docs/getting-started/install.md @@ -2,7 +2,7 @@ title: "Install Core and Web" --- -One command installs Core, the Web console and PostgreSQL on a Linux host. Sign in to Web with the Core key, set a default model and issue Project API keys. Applications call Core with those keys. Sessions run in sandboxes on nodes you add, or on E2B. +One command installs Core, the Web console and PostgreSQL on Linux, macOS or Windows. Sign in to Web with the Core key, set a default model and issue Project API keys. Applications call Core with those keys. Sessions run in sandboxes on nodes you add, or on E2B. 1. [Check the prerequisites](#prerequisites). 2. [Run the installer](#install). @@ -16,21 +16,29 @@ This page follows the default path. Every flag, existing reverse proxies and off ## Prerequisites -- Linux amd64 and curl. -- Docker Engine with Docker Compose 2.26.0 or newer (`docker compose version`). +- Linux amd64/arm64 or macOS Intel/Apple Silicon with curl; Windows x64 with PowerShell. +- Docker Engine 26 or newer and Docker Compose 2.26.0 or newer. On macOS and Windows, install and start Docker Desktop using Linux containers. - An account that can run `docker` and write to its home directory. Ordinary users and root both work; the installer never calls sudo. - Free port 8080 for Web. See [ports](./install-options.md#ports). Docker must be able to publish it; the installer does not change host policy. - For anything off this machine, the origin in `OAC_PUBLIC_URL` must be the address browsers, nodes and executors use. You can sign in on this machine first. -The Core host needs no KVM; nodes that run microsandbox do. +Sandbox nodes run on Linux amd64. When Core runs on macOS or Windows, connect a Linux node or use E2B. ## Install +Linux and macOS: + ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash ``` -On a host whose default route has a private-network address, the installer sets the public URL to `http://:8080`, so machines on the same network can open Web and add nodes; otherwise only this machine can. If a reverse proxy already serves this host, pass its HTTPS address: +Windows PowerShell: + +```powershell +irm https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.ps1 | iex +``` + +When binding to all IPv4 addresses, the installer uses the private address of the default route if available; otherwise the console address is `http://localhost:8080`. To choose another reachable origin, pass `--public-url`; if a reverse proxy already serves this host, use its HTTPS address: ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash -s -- --public-url https://core.example @@ -38,13 +46,13 @@ curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/ The script downloads that release's Compose files, checks their SHA-256, and: -1. checks Linux amd64, Docker Compose 2.26 or newer, and that the ports it will publish are free; -2. creates the [installation directory](../configuration.md#installation-directory), `~/.oac/core`, writes `.env`, and copies the `oac` command out of the Core image; +1. checks Docker is running Linux containers, meets the required versions, and can publish the chosen port; +2. prepares the [installation directory](../configuration.md#installation-directory), `~/.oac/core`, with `.env` and the native `oac` command (`oac.exe` on Windows); 3. starts the services with Docker Compose. Web serves the console on port 8080 and forwards `/v1`, `/api/v1` and `/docs` to Core. Core and PostgreSQL are not published. -It saves no sandbox backend, adds no node, creates no Project or key and makes no model request. It ends by printing the console address and the Core key. +The installer prints the console address and Core key. After signing in, configure execution resources and create Projects in Web. -Downloads and configuration checks happen before the installation directory is published. After that, failures preserve the configuration and data and show the service logs. Fix the reported cause and rerun the same command, or specify the installation directory: +Downloads and configuration checks happen before the installation directory is published. After that, failures preserve the configuration and data and report the failed step; inspect it with `docker compose logs --tail 100` in the installation directory. Fix the reported cause and rerun the same command, or specify the installation directory: ```bash curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash -s -- --install-dir "$HOME/.oac/core" @@ -63,6 +71,8 @@ For insufficient space or quota, free space on the filesystem named by the error ~/.oac/core/oac core-key --show ``` + On Windows, use `& "$HOME/.oac/core/oac.exe" core-key --show`. The same command arguments work on every platform. + ## Configure the public address {#configure-the-domain-and-https} Applications, nodes and sandboxes reach Core at one address, the public URL. HTTP is enough on the local network. When you expose Core beyond it, put a reverse proxy in front and set the public URL to the HTTPS origin it serves. E2B guests reach Core from the internet, so they need a public URL that is not loopback. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 83e63c196..a15d9174d 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -6,7 +6,7 @@ The installation operator owns the Core host, its storage and its availability. ## The oac command -Each installation has its own management command in its directory. It needs neither the bundle nor root: +Each installation has its own native management command in its directory: `oac` on Unix, `oac.exe` on Windows. It needs Docker access and no root privileges: ```sh docker compose -f ~/.oac/core/compose.yaml ps @@ -18,11 +18,11 @@ docker compose -f ~/.oac/core/compose.yaml ps | `docker compose start` | Starts the services | | `docker compose stop` | Stops the services. Data, nodes and sandboxes are kept | | `oac apply` | Runs `oac-core check-config`, then `docker compose up -d --wait`. A failed check changes no service | -| `oac core-key [--show]` | Prints the Core key path, or the key itself with `--show` | +| `oac core-key [--show]` | Identifies the key location in the data volume, or prints the key with `--show` | | `oac rotate-core-key` | Replaces the Core key and restarts Core and Web | | `docker compose down` | Removes the containers. Data is kept; to delete it, [uninstall](#uninstall) | -For a second installation, use its directory, such as `~/.oac/second`. +The examples use the default installation directory. On Windows, invoke the management command with `& "$HOME/.oac/core/oac.exe"` followed by the same arguments. For a custom installation directory, replace the path in each command. ## Service health @@ -63,13 +63,13 @@ A Web restart, including one caused by `oac apply`, signs everyone out of the co ## Core key -Each installation has one administrator credential, the Core key. The installer generates a key with the `oac_admin_` prefix followed by 64 random lowercase hexadecimal characters in `data/secrets/web/core.key`. Read it with `oac core-key --show`; the file is owned by the container user. The Core key: +Each installation has one administrator credential, the Core key. The installer generates a key with the `oac_admin_` prefix followed by 64 random lowercase hexadecimal characters in `secrets/web/core.key`. Read it with `oac core-key --show`; the file is owned by the container user. The Core key: - signs in to Web. The browser gets an HttpOnly session cookie, never the key; - authorizes Core API (`/core/v1`) requests sent as `Authorization: Bearer `; - never authorizes the Agents API (`/v1`). Applications use Project API keys, which in turn can't call `/core/v1`. -Keep it private. Web reads `data/secrets/web/core.key`. Core reads only its SHA-256 from `data/secrets/core/core-key-digests.json`. A Core key has at least 32 characters and no whitespace. Web limits failed sign-ins. +Keep it private. Web reads `secrets/web/core.key`. Core reads only its SHA-256 from `secrets/core/core-key-digests.json`. A Core key has at least 32 characters and no whitespace. Web limits failed sign-ins. ### Script the Core API @@ -103,7 +103,7 @@ The [Core administration API](../../contracts/agents-api/admin-api.md) lists eve ~/.oac/core/oac rotate-core-key ``` -It writes a new key to `data/secrets/web/core.key`, regenerates `data/secrets/core/core-key-digests.json`, and restarts Core and Web. The old key stops working as soon as Core restarts, and every console session ends: sign in again and update your scripts. +It runs in the initialization container, updates `secrets/web/core.key` and `secrets/core/core-key-digests.json` in the data volume, and restarts Core and Web. The old key stops working as soon as Core restarts, and every console session ends: sign in again and update your scripts. ## Projects and API keys @@ -123,37 +123,33 @@ Core records which key made each public resource write; the retention of that hi Back up these together; a restore needs all of them: -- the PostgreSQL volume `_database`. It holds Projects, key digests, nodes, default models, encrypted credentials and all execution history, including large objects. A logical dump: +- the Docker volume `_data`, including its `database/`, `secrets/` and `state/` directories. It holds Projects, key digests, nodes, default models, encrypted credentials and all execution history, including large objects. A logical dump: ```sh docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ pg_dump -U agents_api agents_api > oac-backup.sql ``` -- the installation directory, especially `data/`. `data/secrets/core/credential.key` must stay with the database, or stored credentials can't be decrypted. +- the installation directory containing `.env`, `compose.yaml` and the command. The data volume's `secrets/core/credential.key` must stay with the database, or stored credentials cannot be decrypted. - each node's state directory on its host, `/var/lib/oac-node/.oac/nodes//`, with its provider storage: Docker volumes or microsandbox's store. See [when a node host fails](./nodes.md#when-a-node-host-fails) for restoring them. -Stop with `docker compose stop`, archive the installation directory, then `docker compose start`. - -Never prune Docker volumes or delete native harness history to make a retry pass. A deleted Session does not prove that all provider resources were reclaimed. +Stop with `docker compose stop`, export the complete data volume and archive the installation directory, then `docker compose start`. Docker Desktop supports volume export from its **Volumes** view. A SQL dump alone does not include the encryption key or Provider state. ## Uninstall ```sh cd ~/.oac/core -docker compose down --remove-orphans -docker compose run --rm --no-deps --entrypoint find init /data -mindepth 1 -delete -docker compose down --rmi all +docker compose down --volumes --remove-orphans --rmi all cd && rm -rf ~/.oac/core ``` -The containers own `data/`, so the `init` image deletes its contents; then `down --rmi all` removes the images and `rm` removes the installation directory. Run these only when you mean to delete the data. +`down --volumes` deletes the installation data volume. Remove the installation directory afterward; on Windows use `Remove-Item -Recurse "$HOME/.oac/core"`. All data goes with it: Projects and API keys, Session history, stored credentials and the Core key. To keep the data, stop the installation with `docker compose stop` instead, or [back it up](#back-up) first. Uninstall stops no sandbox: node sandboxes keep running on their nodes, and E2B sandboxes keep running, and billing, at E2B. While Core is still up, archive their Sessions or [reset the deployment](./nodes.md#change-the-sandbox-configuration) and let it complete; the command shows how many sandboxes Core has in use. -Nodes on other hosts keep running. To uninstall them the usual way, remove them in Web first, as in [Remove a node](./nodes.md#remove-a-node). After the installation directory is gone, their Core is gone: on each node host, run the node uninstall command with `--force`, using `node-install.pyz` from the release that installed them. The installation ID is `data/secrets/core/installation.id`. +Nodes on other hosts keep running. To uninstall them the usual way, remove them in Web first, as in [Remove a node](./nodes.md#remove-a-node). After the installation directory is gone, their Core is gone: on each node host, run the node uninstall command with `--force`, using `node-install.pyz` from the release that installed them. The installation ID is `secrets/core/installation.id` in the data volume. ## Installation version policy @@ -163,7 +159,7 @@ To move to a new release, install it into a new, empty directory, with its own d An interrupted installation can [resume with its saved configuration](./install.md#install). An unrelated nonempty directory is refused. -The installer and mutating `oac` commands hold `.oac.lock`. The installer also holds a sibling `.install.lock` while preparing the directory. If another command holds either lock, retry after it finishes. Never delete a lock file to get past a busy installation. +Installation and mutating `oac` commands share the [installation lock](../configuration.md#installation-directory). If another command is running, wait for it to finish before retrying. ## Troubleshooting diff --git a/docs/maintainers.md b/docs/maintainers.md index 47d162b70..166c39feb 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -6,7 +6,10 @@ This guide is for maintainers who build and publish OpenAgentCore. To install Co ## Build a distribution -A distribution is the matched set of Linux amd64 release assets built from one commit: the control archive (the installer, the `oac` command, and the Core, Web, ingress and PostgreSQL images), the Runtime image and node artifacts as separate files, and the native installers. +A distribution is a matched set of release assets built from one commit: the control archive (the installer, the `oac` command, and the Core, Web, ingress and PostgreSQL images), the Runtime image and node artifacts as separate files, and the native installers. + +Core, Web and ingress images are published as verified Linux amd64/arm64 indexes. The arm64 control archive contains those three images; Node, hosted Runtime and offline payloads use Linux amd64. Release builders use QEMU for ARM image steps, including the E2B helper. Host `oac` binaries are built from the same command for Linux amd64/arm64, macOS amd64/arm64 and Windows amd64; launchers only select, verify and invoke them. Each version index is checked against its platform archives before floating tags move. + Build on Linux x86_64 with a glibc compatible with Debian 12, Docker, the Go version in `go.mod`, a C compiler (the microsandbox helper is a CGO build), Node, pnpm, Python 3.9 or newer, curl, tar, pigz and sha256sum. The source must be clean and committed. First prepare the pinned Codex package and MiniMax Code companion, then build: @@ -96,7 +99,7 @@ The distribution combines the three Harness images into one Runtime image (`depl make build-e2b-provider ``` -Docker builds the Linux amd64 helper with the pinned CPython and Debian 12 image. The Python dependency closure, including PyInstaller, is hash-locked in `services/core/tools/e2b-provider/requirements.lock`; no E2B account key is needed. Set `E2B_PROVIDER_BUILD_DIR` for another output directory. The build is a pure function of the helper sources, `LICENSE` and the build script, so it is cached under `~/.oac/cache/e2b-provider/` by their hash and rebuilt only when they change. The output is `oac-e2b-provider-linux-amd64.tar.gz` with its `.sha256`; it extracts to `oac-e2b-provider/` with the executable, `_internal/`, `licenses/`, `requirements.lock` and `manifest.json`. The Core image uses that tree; the host needs a compatible glibc and CA certificates, not Python. +Docker builds the Linux helper for `GOARCH=amd64` (default) or `GOARCH=arm64` with the pinned CPython and Debian 12 image. The Python dependency closure, including PyInstaller, is hash-locked in `services/core/tools/e2b-provider/requirements.lock`; no E2B account key is needed. Set `E2B_PROVIDER_BUILD_DIR` for another output directory. The build is a pure function of the helper sources, `LICENSE` and the build script, so it is cached under `~/.oac/cache/e2b-provider/` by their hash and rebuilt only when they change. The output is `oac-e2b-provider-linux-.tar.gz` with its `.sha256`; it extracts to `oac-e2b-provider/` with the executable, `_internal/`, `licenses/`, `requirements.lock` and `manifest.json`. The Core image uses that tree; the host needs a compatible glibc and CA certificates, not Python. **microsandbox helper.** Linux only, with a C compiler: @@ -111,9 +114,9 @@ The helper is written to `~/.oac/build/microsandbox-provider/oac-microsandbox-pr ### Standalone Core builds -`make build-core` builds `oac-core`, `oac-core-device`, `oac-core-environment-key` and `oac-node` into `${OAC_DEV_HOME:-$HOME/.oac}/build/oac-core` (`OAC_DEV_CORE_BUILD_DIR` selects another absolute directory). The build copies only the source set listed in `scripts/build-core.sh` (the Core service, its contracts, the shared packages it needs and the root Go module files) into a temporary context and builds with CGO disabled, read-only modules and trimmed paths. It needs no Node, Docker or other application. When Core gains a shared dependency, add that package to the list; never copy the whole repository to make it compile. +`make build-core` builds `oac-core`, `oac-core-device`, `oac-core-environment-key` `oac-node` and `oac` into `${OAC_DEV_HOME:-$HOME/.oac}/build/oac-core` (`OAC_DEV_CORE_BUILD_DIR` selects another absolute directory). The build copies only the source set listed in `scripts/build-core.sh` (the Core service, its contracts, the shared packages it needs and the root Go module files) into a temporary context and builds with CGO disabled, read-only modules and trimmed paths. It needs no Node, Docker or other application. When Core gains a shared dependency, add that package to the list; never copy the whole repository to make it compile. -`make docker-build-core` builds the image `oac-core:dev` (`OAC_DEV_CORE_IMAGE` selects another name) from those five commands and the E2B helper. The base is the digest-pinned `debian:bookworm-slim` with CA certificates and the glibc runtime the helper needs; the default user is UID/GID 65532 and Core listens on `:8091`. The image is Linux amd64 only and is not pushed to a registry. Changes to the image or its build need `make check-core-container` in addition to the relevant source checks: it runs the official-client suite against the image with a read-only root filesystem and needs Linux Docker, a non-root user, and the [test database and pinned SDK](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md#official-client-verification) of the service checks (`OAC_TEST_DATABASE_URL` naming an `oac_*_tests` database with the migrations applied, and `OAC_TEST_OFFICIAL_SDK_PYTHON`). +`make docker-build-core` builds the image `oac-core:dev` (`OAC_DEV_CORE_IMAGE` selects another name) from those five commands and the E2B helper. The base is the digest-pinned `debian:bookworm-slim` with CA certificates and the glibc runtime the helper needs; the default user is UID/GID 65532 and Core listens on `:8091`. This local target builds Linux amd64; the [distribution build](#build-a-distribution) builds both architectures. Changes to the image or its build need `make check-core-container` in addition to the relevant source checks: it runs the official-client suite against the image with a read-only root filesystem and needs Linux Docker, a non-root user, and the [test database and pinned SDK](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md#official-client-verification) of the service checks (`OAC_TEST_DATABASE_URL` naming an `oac_*_tests` database with the migrations applied, and `OAC_TEST_OFFICIAL_SDK_PYTHON`). ## Publish a version @@ -132,13 +135,13 @@ Distribution and Runtime archives use `pigz` level 6 with at most four compressi ### Container registry -Version releases and manual `build-` drafts publish Linux amd64 images as `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. For example, `ghcr.io/minimax-ai/openagentcore/core:v1.2.3`. A draft uses the tag `build-`. PostgreSQL uses its upstream image and is not republished. The registry images are loaded from the release archives without rebuilding. Existing version tags are reused only when their image config digest matches the release; a different image stops publication. A stable release also moves each component's `latest` tag to that image. Prereleases and drafts leave `latest` unchanged. SemVer build metadata uses `_` in place of `+` in container tags; version strings longer than 128 characters cannot be published to GHCR. After the images are verified, the publisher uploads the single `compose.yaml` and its checksum list, rendered for that release. Compose pins the ingress image by its registry digest; initialization rejects an image whose build revision differs from the Compose revision. A draft Release stays unpublished. +Version releases and manual `build-` drafts publish `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. Core, Web and ingress indexes contain Linux amd64 and arm64 images; Runtime contains Linux amd64. Platform images use `-` tags and are loaded from the release archives. Existing version tags must match the release images and platform set. The publisher verifies every version index before updating `latest` for a stable release; prereleases and drafts leave `latest` unchanged. PostgreSQL uses its upstream image. SemVer build metadata uses `_` in place of `+` in container tags; version strings are limited to 128 characters. After verifying the images, the publisher uploads the release's `compose.yaml` and checksum list. Compose pins ingress by its index digest, and initialization checks its build revision against the Compose revision. The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all four packages to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. GHCR and GitHub Releases do not share a transaction. A failed release may leave some matching version tags in GHCR; preserve those images and follow the draft recovery procedure below using the original artifacts. Registry failures other than a missing manifest stop publication. The job summary records digest-pinned references. These images and the rendered Compose files still require the configuration, secrets and routing described in [Configuration](./configuration.md). -`install.sh` downloads the Compose files for the latest stable release, or the release named by `--version`, verifies their SHA-256 and starts that release. The [installation guide](./getting-started/install.md#install) covers its use. +The [installation guide](./getting-started/install.md#install) covers release selection and the platform launchers. Go check and build jobs share Go module and compiler-cache directories under `~/.oac/cache/`, keyed by runner OS and architecture, all Go module files, the check/build partition and the commit. Partitioned keys prevent concurrent jobs from saving different compiler subsets under one key. Release builds can seed their cache from backend checks as well as earlier release builds. An older cache only seeds downloads and compilation; every check still runs. Release jobs also cache npm package downloads and the pinned microsandbox archive, whose checksum is verified on every build. Actions cache visibility follows GitHub ref scoping; a tag-specific cache is not shared with other release tags. New keys are saved only after a successful job. @@ -179,7 +182,7 @@ The planner compares the PR event's tested merge commit with its verified first `.github/actionlint.yaml` selects hygiene and lint. Known workflow changes select their consumers: the CI review and actionlint workflows run hygiene and lint; native workflow changes add native checks; API acceptance workflow changes add API checks with container acceptance enabled; website workflow changes add website checks. The shared Node action selects every job that uses it plus lint. A new or unclassified workflow/action selects the full gate until its consumers are declared in the planner. Planner tests and CI measurement scripts run hygiene; changing the planner itself runs the full gate. -Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout; Web serves a placeholder page instead of the console build. Build-time node metadata comes from the release pinned in `deploy/compose/smoke-pins.json`; the initialization container runs with networking disabled. This checks generic Compose behavior; it does not run a Dokploy/Coolify instance or execute a model. +Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout; Web serves a placeholder page instead of the console build. Build-time node metadata comes from the release pinned in `deploy/compose/smoke-pins.json`; the initialization container runs with networking disabled. The smoke matrix runs on native Linux amd64 and arm64 runners; the native matrix builds and tests the shared Core installer on Linux, macOS and Windows. Go module and workspace inputs select backend, API (including the container), native and distribution checks. Each Node module owns its manifest and lockfile. Website dependencies select website checks; Web dependencies select Web and browser checks; example dependencies select example checks; shared TypeScript client dependencies select Web, browser and example checks; Claude adapter dependencies select Harness, native and distribution checks. Shared package-manager configuration selects all Node consumers. The root TypeScript configuration selects Web and example checks; the adapter TypeScript configuration selects Harness and native checks. Each selected set includes hygiene. Mixed changes accumulate their consumers, and every job reads the same plan instead of maintaining its own path list. For example, a notification-only PR skips database, browser and native jobs, while a notification plus Core change adds backend and API checks. diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 2ef6abe1b..79465aa1f 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: e697e320a2df0c130cb004f5e8bbda50deff6515ef7727bea066fc0d4f688b7d +source_hash: 8eeef9a9742c5fd8a0bf27a5a31870a77dec59a436518dbe9d34770527c021a2 --- Core 安装的每项设置都恰好只有一个归属位置。共有两类： @@ -11,7 +11,7 @@ Core 安装的每项设置都恰好只有一个归属位置。共有两类： | [进程设置](#process-settings-configjson) | 公共 URL、端口、日志、Harness、执行并发度、审计保留期、OAuth 来源、Runtime 历史记录导出 | 安装目录中的 `.env`（默认 `~/.oac/core`） | 编辑 `.env`，然后运行 `oac apply` | `oac apply` 会重新创建读取了这些已更改设置的服务 | | [运行时设置](#runtime-settings-web) | 沙箱后端和大小、节点、项目和密钥、默认模型、执行器凭据 | Core 的 PostgreSQL 数据库 | 在 Web 中修改，或使用 Core 密钥调用 Core API（`/core/v1`） | 保存时无需重启 Core；节点会异步准备 Runtime 变更 | -Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置，并在 **Startup settings** 下以只读方式显示 Core 加载的进程设置。机密信息存放在 [`data/secrets/`](#installation-directory) 中，每项仅保存一份。没有任何配置文件定义项目或 API 密钥。 +Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置，并在 **Startup settings** 下以只读方式显示 Core 加载的进程设置。机密信息存放在 [`secrets/`](#compose-installations) 中，每项仅保存一份。没有任何配置文件定义项目或 API 密钥。 ## 进程设置 {#process-settings-configjson} @@ -25,9 +25,8 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 1. 它用你改过的 `.env` 运行 `oac-core check-config`。值无效时什么都不改。 2. 它运行 `docker compose up -d --wait`。Compose 只重新创建配置有变化的服务。 -3. 校验失败时，不会重新创建任何容器。重启会中断哪些操作，见[停止和重启](getting-started/operations.md#stop-and-restart)。 -`docker compose ps` 展示服务。域名状态在 `data/domain/status.json`。 +用 `docker compose ps` 检查服务。重启会中断哪些操作，见[停止和重启](getting-started/operations.md#stop-and-restart)。 ### 更改公共 URL {#changing-the-public-url} @@ -48,7 +47,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | Variable | Default | Meaning | | --- | --- | --- | | `OAC_PUBLIC_URL` | `http://localhost:8080` | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[修改公开 URL](#changing-the-public-url) | -| `OAC_HOST` | `127.0.0.1` | `compose.yaml` 发布的 Web 绑定地址。`install.sh` 设置为 `0.0.0.0` | +| `OAC_HOST` | `127.0.0.1` | `compose.yaml` 发布的 Web 绑定地址。安装器设置为 `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | @@ -85,7 +84,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 ### 默认模型 {#default-models} -在 **System** → **Default model configuration** 中设置默认值，或使用 `PUT /core/v1/harnesses/{harness}/model-configuration`。Core 使用 `secrets/credential.key` 加密提供商密钥，并且绝不返回这些密钥。[模型执行](../../contracts/agents-api/zh/model-execution.md#deployment-defaults) 定义了请求字段和替换规则，[优先级](../../contracts/agents-api/zh/model-execution.md#saved-defaults-and-precedence)说明了哪些 Session 使用默认值。 +在 **System** → **Default model configuration** 中设置默认值，或使用 `PUT /core/v1/harnesses/{harness}/model-configuration`。Core 使用 `secrets/core/credential.key` 加密提供商密钥，并且绝不返回这些密钥。[模型执行](../../contracts/agents-api/zh/model-execution.md#deployment-defaults) 定义了请求字段和替换规则，[优先级](../../contracts/agents-api/zh/model-execution.md#saved-defaults-and-precedence)说明了哪些 Session 使用默认值。 ## Compose 安装 {#compose-installations} @@ -104,7 +103,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 初始化会准备该目录；应用服务以只读方式接收各自的机密目录。`docker compose exec web oac-web core-key` 把 Core 密钥打印到运维人员终端，不写入容器日志。数据库密码和凭据加密密钥绝不打印。 -`OAC_DATA_DIR` 选择该目录，默认是 Compose 文件旁的 `./data`。必须将该项目的定义和公共 URL 与该目录一同保留。仅删除机密目录不会重置安装；如果数据库已经存在，初始化会拒绝重新开始。Core 还会将安装 ID 与其数据库绑定。运行时设置仍存储在 [Core 的数据库](#runtime-settings-web)中。 +上述路径位于 Docker 命名卷 `_data` 中。所有宿主机平台都由 Docker 管理 Linux 文件权限；每个服务只挂载需要的子目录。数据卷必须和项目定义、公共 URL 一同保留。仅删除机密目录不会重置安装；数据库已存在时初始化会拒绝重建。Core 也会校验安装 ID 与数据库的绑定。运行时设置仍保存在 [Core 数据库](#runtime-settings-web)中。 ## Docker 节点配置 {#docker-node-configuration} @@ -122,37 +121,31 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 ## 安装目录 {#installation-directory} -安装程序会创建安装目录，默认路径为 `~/.oac/core`，权限模式为 `0700`。机密文件为 `0600`。 +安装目录默认为 `~/.oac/core`（Windows 为 `$HOME/.oac/core`）。其中保存进程设置和原生管理命令；服务持久数据位于 [Compose 数据卷](#compose-installations)。 | 路径 | 内容 | 修改者 | | --- | --- | --- | -| `.env` | [进程设置](#process-settings-configjson)。由你编辑的文件 | 你，然后运行 `oac apply`；托管域名设置写入 `OAC_PUBLIC_URL` | -| `compose.yaml` | 发行版的服务定义。不要编辑 | 发行版 | -| `oac` | [管理命令](getting-started/operations.md#the-oac-command)，从 Core 镜像复制 | 安装程序 | -| `data/secrets/web/core.key` | [Core 密钥](getting-started/operations.md#core-key) | `oac rotate-core-key` | -| `data/secrets/core/credential.key` | 加密 Core 在数据库中封存内容的密钥 | 无。必须与数据库一同保留 | -| `data/secrets/core/core-key-digests.json` | Core 密钥的 SHA-256 | `oac rotate-core-key` | -| `data/secrets/database/password` | PostgreSQL 密码 | 无。PostgreSQL 仅在创建数据库时读取 | -| `data/database/` | PostgreSQL 数据 | PostgreSQL | -| `data/node-payload/` | Web 在 `/node-install/` 提供的节点文件 | 初始化 | -| `data/state/` | 私有 Provider 状态，包括 E2B 回执 | Core | -| `.oac.lock` | 安装锁 | 会修改安装状态的 `oac` 命令 | - -Compose 项目名为 `oac-<10 hex digits>`。服务包括 `init`、`database`、`core` 和 `web`。Core 启动时执行数据库迁移。`web` 提供控制台并把 `/v1` 和 `/api/v1` 转发到 Core，是唯一发布端口（`OAC_WEB_PORT`）的服务。没有服务持有 Docker 套接字。除 Docker 存储外，不会向安装目录之外写入任何内容。 +| `.env` | 进程设置和固定的 Compose 项目名 | 用户修改后运行 `oac apply` | +| `compose.yaml`、`compose-sha256sums.txt` | 已校验的发行版服务定义 | 发行流程 | +| `oac`（Windows 为 `oac.exe`） | 原生管理命令 | 安装程序 | + +同级 `.lock` 目录用于同步操作并一直保留；`.staging` 保存尚未就位的安装文件。两者都不保存服务数据。Unix 上安装程序以 `0700` 创建私有目录，以 `0600` 创建配置文件。 + +Compose 项目名为 `oac-<10 hex digits>`，服务包括 `init`、`database`、`core` 和 `web`。Core 启动时执行数据库迁移。Web 提供控制台并把 `/v1`、`/api/v1` 转发到 Core，是唯一发布端口（`OAC_WEB_PORT`）的服务。没有服务持有 Docker 套接字。 ## 附录：没有安装程序时的 Core 环境 {#appendix-core-environment-without-the-installer} -Core 只读取其环境。Compose 把 `.env` 插值进服务环境。Compose 必须为 2.26.0 或更高版本。如果你自行运行 Core，请设置这些变量；见[服务指南](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md)。 +Core 读取进程环境。Compose 将 `.env` 插值到环境中，并把机密文件挂载到下表中的容器路径。直接运行 Core 时，将文件变量设为 Core 进程可读取的绝对路径；见[服务指南](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md)。 | 变量 | 设置来源 | | --- | --- | | `OAC_PUBLIC_URL` | `public_url`，或 Core 的回环源地址。Core 从中派生守护进程 WebSocket URL、自托管 `remote_url`、托管沙箱地址和部署的只读 `core_url`，绝不从请求标头派生。未设置时，Core 不运行 Runtime 网关，也不执行任何 Session | | `OAC_ADDR` | 安装程序在容器中设置为 `:8091`。独立启动的 Core 在未设置或为空时，默认使用 `127.0.0.1:8091` | | `OAC_DATABASE_URL` | 该安装不含密码的 PostgreSQL URL，并将 `core.database_pool` 作为 `pool_*` 查询参数附加到其中 | -| `OAC_DATABASE_PASSWORD_FILE` | `data/secrets/database/password`。此时 URL 不得包含密码 | -| `OAC_CREDENTIAL_KEY_FILE` | `data/secrets/core/credential.key` | -| `OAC_CORE_KEY_DIGESTS_FILE` | `data/secrets/core/core-key-digests.json`：一个包含 Core 密钥 SHA-256 的 JSON 数组 | -| `OAC_INSTALLATION_ID_FILE` | `data/secrets/core/installation.id`：安装 ID，采用规范 UUID 格式。它会启用沙箱部署和节点路由，并要求设置 `OAC_PUBLIC_URL` 和 `OAC_CORE_KEY_DIGESTS_FILE`。如果 ID 与数据库记录的 ID 不一致，Core 会拒绝它，因此必须将两者一同保留 | +| `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`。此时 URL 不得包含密码 | +| `OAC_CREDENTIAL_KEY_FILE` | `/run/oac/credential.key` | +| `OAC_CORE_KEY_DIGESTS_FILE` | `/run/oac/core-key-digests.json`：一个包含 Core 密钥 SHA-256 的 JSON 数组 | +| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`：安装 ID，采用规范 UUID 格式。它会启用沙箱部署和节点路由，并要求设置 `OAC_PUBLIC_URL` 和 `OAC_CORE_KEY_DIGESTS_FILE`。如果 ID 与数据库记录的 ID 不一致，Core 会拒绝它，因此必须将两者一同保留 | | `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS` | 对应的[进程设置](#settings)。`oac-core check-config` 会在不启动 Core 的情况下校验它们 | | `OAC_HISTORY_SETTINGS_FILE` | 可选的 Runtime 历史文件。敏感；安装报告只说明它是否已设置 | | `OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | `log.*`；Web 也读取这三个设置 | @@ -166,7 +159,7 @@ Core 会记录所加载文件的路径，但绝不记录环境变量的值或文 ## 附录：没有安装程序时的 Web 环境 {#appendix-web-environment-without-the-installer} -Compose 为 Web 设置这些变量。仅在不使用 Compose 运行控制台时才自行设置。该安装的机密信息中，Web 只收到 `data/secrets/web/core.key`。 +Compose 为 Web 设置这些变量。仅在不使用 Compose 运行控制台时才自行设置。Compose 把数据卷的 `secrets/web/` 挂载到 `/run/oac`，并设置 `OAC_WEB_CORE_KEY_FILE=/run/oac/core.key`。 | 变量 | 默认值 | 含义 | | --- | --- | --- | diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index b0faa317d..fe55a0610 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,10 +1,10 @@ --- title: "安装选项与高级部署" source: docs/getting-started/install-options.md -source_hash: e063d7dcd615ef5d6ba8b43f1cbe5a11c75325925605f376161f50b9383297c0 +source_hash: 2e7717f76a46694af3c1ef33a56b195c0a321f54fd7318b488179b4b4c61f336 --- -[默认安装](install.md)无需任何选项。使用本页可以在现有反向代理后运行，或者在无法访问互联网时进行安装。 +[默认安装](install.md)无需任何选项。本页介绍安装选项、Compose 部署和反向代理配置。 向下载的脚本传递选项： @@ -12,13 +12,11 @@ source_hash: e063d7dcd615ef5d6ba8b43f1cbe5a11c75325925605f376161f50b9383297c0 ./install.sh --public-url https://core.example ``` -使用单行命令时，请将选项追加在 `bash -s --` 之后。发布包下载器还接受 `--version TAG` 来选择已发布的版本；否则会选择最新的稳定版本。它会在解压前验证捆绑包的 SHA-256，并保留已验证的捆绑包以供[修复](operations.md#installation-version-policy)。 - -安装程序会打印每个阶段，然后打印地址、登录信息和后续步骤的摘要。设置 `NO_COLOR=1` 可禁用彩色输出。任一步骤失败都会停止安装，并且不会显示成功消息。 +Windows 可下载 `install.ps1`，然后使用相同参数，例如 `& ./install.ps1 --public-url https://core.example`。Unix 单行命令的参数追加在 `bash -s --` 后。`--version TAG` 选择已发布的版本；默认选择最新稳定版，并校验原生命令和 Compose 文件的 SHA-256。 ## Docker Compose 与托管平台 {#docker-compose-and-hosting-platforms} -在 Linux amd64 上使用发行版中的 `compose.yaml` 和 Docker Compose 2.26 或更高版本。发行流程会在 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)中固定初始化镜像及源码版本。Core 和 Web 使用 `latest` 镜像，PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据通过目录 bind mount 挂载。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序；Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 +在任一[支持的 Core 主机](install.md#prerequisites)上使用发行版中的 `compose.yaml`。发行流程会在 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)中固定初始化镜像及源码版本。Core 和 Web 使用 `latest` 镜像，PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据保存在 Docker 命名卷中。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序；Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 进行本地试用时，请将发行版的 `compose.yaml` 下载到一个空目录，然后运行： @@ -27,7 +25,7 @@ docker compose up -d --wait --wait-timeout 900 docker compose exec web oac-web core-key ``` -`oac-web core-key` 会将生成的 Core 密钥打印到终端，而不会将其写入容器日志。打开 `http://localhost:8080` 并使用该密钥登录。所有安装机密信息都会自动生成；重启时请保留同一个 Compose 项目及其数据目录。 +`oac-web core-key` 会将生成的 Core 密钥打印到终端，而不会将其写入容器日志。打开 `http://localhost:8080` 并使用该密钥登录。所有安装机密信息都会自动生成；重启时请保留同一个 Compose 项目及其数据卷。 初始化镜像除初始化命令外，仅包含较小的节点安装元数据。首次启动会验证并复制这些元数据，无需下载控制归档或访问 GitHub Releases。后续启动会验证已保存的文件。容器镜像仍需拉取。首次初始化中断后可以重新运行；如果现有数据库缺少安装机密信息，初始化会被拒绝。 @@ -45,7 +43,7 @@ docker compose exec web oac-web core-key 登录后，使用 [Nodes](nodes.md)选择沙箱后端并添加节点。Compose 堆栈部署控制平面；执行机器仍需单独部署。 -使用相同的文件和环境运行 `docker compose stop` 以停止服务。停止服务后，备份数据目录。请遵循[安装版本策略](operations.md#installation-version-policy)：使用不同发布版本时，需要创建新的 Compose 项目并使用全新的数据目录。 +使用相同的文件和环境运行 `docker compose stop` 以停止服务。停止服务后，备份数据卷。请遵循[安装版本策略](operations.md#installation-version-policy)：使用不同发布版本时，需要创建新的 Compose 项目并使用全新的数据卷。 ## 进程设置 {#process-settings} @@ -64,7 +62,7 @@ docker compose exec web oac-web core-key | 选项 | 用途 | | --- | --- | -| `--install-dir DIR` | 绝对安装目录；默认为 `~/.oac/core`。新安装要求目录为空或不存在，或者包含一个[从未启动过的安装](install.md#install) | +| `--install-dir DIR` | 绝对安装目录；默认为 `~/.oac/core`。新安装使用空目录或不存在的目录；[重试](install.md#install)使用已有安装目录 | 只要使用不同的安装目录和端口，多个安装就可以共用一台机器。需要更多安装时，请使用不同的 IP 地址或共享反向代理。每个安装都有自己的数据库、Core 密钥和节点。 @@ -137,4 +135,4 @@ http://:8443 { ## 离线主机 {#offline-hosts} -本安装程序不支持从离线捆绑包安装。它从发布版本下载 Compose 文件和容器镜像。 +Core 安装需要访问 GitHub Releases 和容器注册表，以下载发行文件和镜像。 diff --git a/docs/zh/getting-started/install.md b/docs/zh/getting-started/install.md index da6dbe184..656ada863 100644 --- a/docs/zh/getting-started/install.md +++ b/docs/zh/getting-started/install.md @@ -1,10 +1,10 @@ --- title: "安装 Core 和 Web" source: docs/getting-started/install.md -source_hash: d034ab565564ae9447c6e8b31b3d13d8868cae738f623a7471ba56ba517d67c8 +source_hash: 1366a76858085e015480c48c18891987397cdc1da0d859182da3685ed77355be --- -一条命令即可在 Linux 主机上安装 Core、Web 控制台和 PostgreSQL。用 Core 密钥登录 Web，设置默认模型并签发 Project API 密钥。应用使用这些密钥调用 Core。Session 在你添加的节点上的沙箱中运行，也可以在 E2B 上运行。 +一条命令即可在 Linux、macOS 或 Windows 上安装 Core、Web 控制台和 PostgreSQL。用 Core 密钥登录 Web，设置默认模型并签发 Project API 密钥。应用使用这些密钥调用 Core。Session 在你添加的节点上的沙箱中运行，也可以在 E2B 上运行。 1. [检查前置条件](#prerequisites)。 2. [运行安装程序](#install)。 @@ -18,21 +18,29 @@ source_hash: d034ab565564ae9447c6e8b31b3d13d8868cae738f623a7471ba56ba517d67c8 ## 前置条件 {#prerequisites} -- Linux amd64 和 curl。 -- Docker Engine 和 Docker Compose 2.26.0 或更高版本（`docker compose version`）。 +- Linux amd64/arm64 或 macOS Intel/Apple Silicon，需安装 curl；Windows x64 需 PowerShell。 +- Docker Engine 26 或更高版本，以及 Docker Compose 2.26.0 或更高版本。macOS 和 Windows 使用已启动的 Docker Desktop，并选择 Linux 容器。 - 能运行 `docker` 并向自己的主目录写入文件的账号。普通用户和 root 均可；安装程序不会调用 sudo。 - Web 的 8080 端口空闲。参阅[端口](install-options.md#ports)。Docker 必须能发布该端口；安装程序不会修改主机策略。 - 本机以外的访问要求 `OAC_PUBLIC_URL` 就是浏览器、节点和执行器使用的地址。可以先在本机登录。 -Core 主机不需要 KVM；运行 microsandbox 的节点需要。 +沙箱节点运行在 Linux amd64 上。在 macOS 或 Windows 上部署 Core 时，可连接 Linux 节点，或使用 E2B。 ## 安装 {#install} +Linux 和 macOS： + ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash ``` -如果主机默认路由的源地址是私有网络地址，安装程序把公开 URL 设为 `http://:8080`，同一网络的机器即可打开 Web 并添加节点；否则只有本机可以访问。反向代理已经提供这台主机时，传入它的 HTTPS 地址： +Windows PowerShell： + +```powershell +irm https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.ps1 | iex +``` + +绑定所有 IPv4 地址时，安装器会使用默认路由的私网地址；如果没有可用私网地址，则使用 `http://localhost:8080`。也可以通过 `--public-url` 指定可访问的源地址；如果已有反向代理，就使用它的 HTTPS 地址： ```sh curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash -s -- --public-url https://core.example @@ -40,13 +48,13 @@ curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/ 脚本下载该发布版的 Compose 文件，校验 SHA-256，然后： -1. 检查 Linux amd64、Docker Compose 2.26 或更高版本，以及将要发布的端口是否空闲； -2. 创建[安装目录](../configuration.md#installation-directory) `~/.oac/core`，写入 `.env`，并从 Core 镜像复制 `oac` 命令； +1. 检查 Docker 使用 Linux 容器、符合版本要求，并确认所选端口空闲； +2. 准备[安装目录](../configuration.md#installation-directory) `~/.oac/core`，写入 `.env` 和原生 `oac` 命令（Windows 为 `oac.exe`）； 3. 用 Docker Compose 启动服务。Web 在 8080 端口提供控制台，并把 `/v1`、`/api/v1` 和 `/docs` 转发到 Core。Core 和 PostgreSQL 不发布端口。 -安装程序不保存沙箱后端，不添加节点，不创建 Project 或密钥，也不发起模型请求。完成后输出控制台地址和 Core 密钥。 +安装器会打印控制台地址和 Core 密钥。登录后，在 Web 中配置执行资源并创建 Project。 -安装目录就位前，先完成下载和配置检查。之后的失败会保留配置与数据，并显示服务日志。修复报错后，重新执行同一命令，或指定安装目录即可继续： +安装目录就位前，先完成下载和配置检查。之后的失败会保留配置与数据，并报告失败步骤；在安装目录运行 `docker compose logs --tail 100` 查看日志。修复报错后，重新执行同一命令，或指定安装目录即可继续： ```bash curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash -s -- --install-dir "$HOME/.oac/core" @@ -65,6 +73,8 @@ curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/ ~/.oac/core/oac core-key --show ``` + Windows 使用 `& "$HOME/.oac/core/oac.exe" core-key --show`。各平台的命令参数相同。 + ## 配置公开地址 {#configure-the-domain-and-https} 应用、节点和沙箱通过同一个地址访问 Core，即公开 URL。局域网上用 HTTP 即可。对外暴露时，在前面放反向代理，并把公开 URL 设为它提供的 HTTPS 源地址。E2B 客户机从互联网访问 Core，因此需要非回环的公开 URL。 diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index 9a2a87cc0..545f99a7b 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,14 +1,14 @@ --- title: "管理你的安装" source: docs/getting-started/operations.md -source_hash: 8b5ec8893b3e844a2c4173a4122cee17bf08350cd1ea2d5128e9a6c72f590907 +source_hash: f63789e0f3982b9f6633381d3c93441e5185b04398541b95c3e1d0505de588eb --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 ## oac 命令 {#the-oac-command} -每个安装目录中都有自己的管理命令，无需发行包或 root： +每个安装目录都有自己的原生管理命令：Unix 使用 `oac`，Windows 使用 `oac.exe`。命令需要 Docker 访问权限，不需要 root： ```sh docker compose -f ~/.oac/core/compose.yaml ps @@ -20,11 +20,11 @@ docker compose -f ~/.oac/core/compose.yaml ps | `docker compose start` | 启动服务 | | `docker compose stop` | 停止服务。保留数据、节点和沙箱 | | `oac apply` | 先运行 `oac-core check-config`，再执行 `docker compose up -d --wait`。校验失败时不改动任何服务 | -| `oac core-key [--show]` | 打印 Core 密钥路径；加上 `--show` 时打印密钥本身 | +| `oac core-key [--show]` | 指出 Core 密钥在数据卷中的位置；加上 `--show` 时打印密钥本身 | | `oac rotate-core-key` | 替换 Core 密钥并重启 Core 和 Web | | `docker compose down` | 移除容器。数据保留；要删除数据，请[卸载](#uninstall) | -第二个安装使用自己的目录，例如 `~/.oac/second`。 +示例使用默认安装目录。Windows 上使用 `& "$HOME/.oac/core/oac.exe"` 调用管理命令，后接相同参数。使用自定义安装目录时，替换各命令中的路径。 ## 服务健康状态 {#service-health} @@ -65,13 +65,13 @@ Web 重启（包括 `oac apply` 引起的重启）会让所有控制台用户退 ## Core 密钥 {#core-key} -每个安装有一个管理员凭据，即 Core 密钥。安装程序在 `data/secrets/web/core.key` 生成以 `oac_admin_` 为前缀、后接 64 个随机小写十六进制字符的密钥。用 `oac core-key --show` 读取；该文件属于容器用户。Core 密钥： +每个安装有一个管理员凭据，即 Core 密钥。安装程序在 `secrets/web/core.key` 生成以 `oac_admin_` 为前缀、后接 64 个随机小写十六进制字符的密钥。用 `oac core-key --show` 读取；该文件属于容器用户。Core 密钥： - 用于登录 Web。浏览器获得 HttpOnly 会话 cookie，不持有密钥； - 通过 `Authorization: Bearer ` 授权 Core API（`/core/v1`）请求； - 不授权 Agents API（`/v1`）。应用使用 Project API 密钥，后者也不能调用 `/core/v1`。 -请保密。Web 读取 `data/secrets/web/core.key`。Core 只读取 `data/secrets/core/core-key-digests.json` 中的 SHA-256。Core 密钥至少 32 字符且不含空白。Web 限制失败登录。 +请保密。Web 读取 `secrets/web/core.key`。Core 只读取 `secrets/core/core-key-digests.json` 中的 SHA-256。Core 密钥至少 32 字符且不含空白。Web 限制失败登录。 ### 用脚本调用 Core API {#script-the-core-api} @@ -105,7 +105,7 @@ core() ( # core METHOD PATH [JSON body] ~/.oac/core/oac rotate-core-key ``` -它把新密钥写入 `data/secrets/web/core.key`，重新生成 `data/secrets/core/core-key-digests.json`，并重启 Core 和 Web。Core 重启后旧密钥立即失效，所有控制台会话结束：重新登录并更新脚本。 +它在初始化容器中更新数据卷内的 `secrets/web/core.key` 和 `secrets/core/core-key-digests.json`，然后重启 Core 与 Web。Core 重启后旧密钥立即失效，控制台会话也会结束；请重新登录并更新脚本。 ## Project 和 API 密钥 {#projects-and-api-keys} @@ -125,38 +125,34 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` 一起备份这些内容；恢复时全部需要： -- PostgreSQL 卷 `_database`。其中包含 Project、密钥摘要、节点、默认模型、加密凭据和全部执行历史（含大对象）。逻辑备份： +- Docker 卷 `_data`，包括其中的 `database/`、`secrets/` 和 `state/` 目录。其中包含 Project、密钥摘要、节点、默认模型、加密凭据和全部执行历史（含大对象）。逻辑备份： ```sh docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ pg_dump -U agents_api agents_api > oac-backup.sql ``` -- 安装目录，尤其是 `data/`。`data/secrets/core/credential.key` 必须与数据库一起保留，否则无法解密存储的凭据。 +- 安装目录中的 `.env`、`compose.yaml` 和管理命令。数据卷内的 `secrets/core/credential.key` 必须与数据库一起保留，否则存储的凭据无法解密。 -先 `docker compose stop`，打包安装目录，再 `docker compose start`。 - 各节点主机上的状态目录 `/var/lib/oac-node/.oac/nodes//` 及提供商存储：Docker 卷或 microsandbox 存储。恢复方法见[节点主机故障时](nodes.md#when-a-node-host-fails)。 -- 安装所使用的发行包，用于修复同一版本。 -不要通过清理 Docker 卷或删除原生 Harness 历史来让重试成功。Session 已删除不证明所有提供商资源已回收。 +运行 `docker compose stop`，导出完整数据卷并归档安装目录，再运行 `docker compose start`。Docker Desktop 的 **Volumes** 页面支持导出数据卷。SQL 转储不包含加密密钥和 Provider 状态。 ## 卸载 {#uninstall} ```sh cd ~/.oac/core -docker compose down --remove-orphans -docker compose run --rm --no-deps --entrypoint find init /data -mindepth 1 -delete -docker compose down --rmi all +docker compose down --volumes --remove-orphans --rmi all cd && rm -rf ~/.oac/core ``` -`data/` 归容器所有，因此由 `init` 镜像删除其内容；随后 `down --rmi all` 移除镜像，`rm` 删除安装目录。只有确定要删数据时才执行这些命令。 +`down --volumes` 会删除安装数据卷。之后删除安装目录；Windows 使用 `Remove-Item -Recurse "$HOME/.oac/core"`。 全部数据随之删除：Project 和 API 密钥、Session 历史、存储的凭据和 Core 密钥。要保留数据，请用 `docker compose stop` 停止安装，或先[备份](#back-up)。 卸载不停止沙箱：节点沙箱在节点继续运行，E2B 沙箱在 E2B 继续运行并计费。Core 仍运行时，归档它们的 Session，或[重置部署](nodes.md#change-the-sandbox-configuration)并等待完成；命令展示 Core 正在使用的沙箱数量。 -其他主机上的节点继续运行。按常规方式卸载时，先在 Web 移除，见[移除节点](nodes.md#remove-a-node)。安装目录删除后，它们的 Core 已不存在：在各节点主机使用当时发布版的 `node-install.pyz`，执行带 `--force` 的节点卸载命令。安装 ID 在 `data/secrets/core/installation.id`。 +其他主机上的节点继续运行。按常规方式卸载时，先在 Web 移除，见[移除节点](nodes.md#remove-a-node)。安装目录删除后，它们的 Core 已不存在：在各节点主机使用当时发布版的 `node-install.pyz`，执行带 `--force` 的节点卸载命令。安装 ID 在数据卷的 `secrets/core/installation.id` 中。 ## 安装版本策略 {#installation-version-policy} @@ -166,7 +162,7 @@ cd && rm -rf ~/.oac/core 中断的安装可以[沿用已保存配置继续](install.md#install)。与本安装无关的非空目录会被拒绝。 -安装程序和修改状态的 `oac` 命令持有 `.oac.lock`。安装程序准备目录时还持有同级的 `.install.lock`。其他命令持有锁时，等待其结束后重试。不要删除锁文件来绕过忙碌安装。 +安装程序和修改状态的 `oac` 命令共用[安装锁](../configuration.md#installation-directory)。其他命令正在运行时，等待其结束后重试。 ## 问题排查 {#troubleshooting} diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index 3b553a3da..aebb1a172 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,14 +1,17 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: 679fb7cf8af9fc6aa2adfb9b04e1766497a32937fd184162614ae813707cb90b +source_hash: 4f9fd38af5afd47e21589160930347f6693dd4786f56fb2dff2abd4853b3f72f --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 ## 构建分发包 {#build-a-distribution} -分发包是从同一个提交构建的一组相互匹配的 Linux amd64 发布资源：控制归档（安装器、`oac` 命令，以及 Core、Web、ingress 和 PostgreSQL 镜像）、作为独立文件的 Runtime 镜像和节点构件，以及原生安装器。 +分发包是从同一个提交构建的一组相互匹配的发布资源：控制归档（安装器、`oac` 命令，以及 Core、Web、ingress 和 PostgreSQL 镜像）、作为独立文件的 Runtime 镜像和节点构件，以及原生安装器。 + +Core、Web 和 ingress 镜像发布为经过校验的 Linux amd64/arm64 多架构索引。arm64 控制归档包含这三个镜像；Node、托管 Runtime 和离线包使用 Linux amd64。发行构建使用 QEMU 执行 ARM 镜像步骤，包括 E2B helper。宿主机 `oac` 从同一份实现构建为 Linux amd64/arm64、macOS amd64/arm64 和 Windows amd64 二进制；启动脚本只选择、校验并运行它们。所有版本索引校验通过后才更新浮动标签。 + 请在 Linux x86_64 上构建，所需环境包括与 Debian 12 兼容的 glibc、Docker、`go.mod` 中指定的 Go 版本、C 编译器（microsandbox 辅助程序使用 CGO 构建）、Node、pnpm、Python 3.9 或更高版本、curl、tar、pigz 和 sha256sum。源代码必须保持干净并已提交。请先准备固定版本的 Codex 包和 MiniMax Code 配套程序，然后执行构建： @@ -98,7 +101,7 @@ docker build --platform linux/amd64 -t oac-runtime:mcode "${OAC_DEV_HOME:-$HOME/ make build-e2b-provider ``` -Docker 使用固定版本的 CPython 和 Debian 12 镜像构建 Linux amd64 辅助程序。Python 依赖闭包（including PyInstaller）在 `services/core/tools/e2b-provider/requirements.lock` 中按哈希锁定；不需要 E2B 账户密钥。要使用其他输出目录，请设置 `E2B_PROVIDER_BUILD_DIR`。构建结果完全由辅助程序源代码、`LICENSE` 和构建脚本决定，因此会按它们的哈希缓存在 `~/.oac/cache/e2b-provider/` 下，仅在它们变化时重新构建。输出为 `oac-e2b-provider-linux-amd64.tar.gz` 及其 `.sha256`；解压后会得到 `oac-e2b-provider/`，其中包含可执行文件、`_internal/`、`licenses/`、`requirements.lock` 和 `manifest.json`。Core 镜像使用该目录树；主机需要兼容的 glibc 和 CA 证书，而不需要 Python。 +Docker 使用固定版本的 CPython 和 Debian 12 镜像按 `GOARCH=amd64`（默认）或 `GOARCH=arm64` 构建 Linux 辅助程序。Python 依赖闭包（including PyInstaller）在 `services/core/tools/e2b-provider/requirements.lock` 中按哈希锁定；不需要 E2B 账户密钥。要使用其他输出目录，请设置 `E2B_PROVIDER_BUILD_DIR`。构建结果完全由辅助程序源代码、`LICENSE` 和构建脚本决定，因此会按它们的哈希缓存在 `~/.oac/cache/e2b-provider/` 下，仅在它们变化时重新构建。输出为 `oac-e2b-provider-linux-.tar.gz` 及其 `.sha256`；解压后会得到 `oac-e2b-provider/`，其中包含可执行文件、`_internal/`、`licenses/`、`requirements.lock` 和 `manifest.json`。Core 镜像使用该目录树；主机需要兼容的 glibc 和 CA 证书，而不需要 Python。 **microsandbox 辅助程序。** 仅支持 Linux，并且需要 C 编译器： @@ -113,9 +116,9 @@ make check-microsandbox-provider ### 独立 Core 构建 {#standalone-core-builds} -`make build-core` 会将 `oac-core`、`oac-core-device`、`oac-core-environment-key` 和 `oac-node` 构建到 `${OAC_DEV_HOME:-$HOME/.oac}/build/oac-core`（`OAC_DEV_CORE_BUILD_DIR` 可选择其他绝对目录）。构建过程仅将 `scripts/build-core.sh` 中列出的源文件集（Core 服务、其契约、所需的共享软件包以及根 Go 模块文件）复制到临时上下文，并使用禁用 CGO、只读模块和裁剪路径的方式构建。它不需要 Node、Docker 或其他应用程序。Core 新增共享依赖时，请将该软件包加入列表；绝不能复制整个仓库来使其完成编译。 +`make build-core` 会将 `oac-core`、`oac-core-device`、`oac-core-environment-key`、`oac-node` 和 `oac` 构建到 `${OAC_DEV_HOME:-$HOME/.oac}/build/oac-core`（`OAC_DEV_CORE_BUILD_DIR` 可选择其他绝对目录）。构建过程仅将 `scripts/build-core.sh` 中列出的源文件集（Core 服务、其契约、所需的共享软件包以及根 Go 模块文件）复制到临时上下文，并使用禁用 CGO、只读模块和裁剪路径的方式构建。它不需要 Node、Docker 或其他应用程序。Core 新增共享依赖时，请将该软件包加入列表；绝不能复制整个仓库来使其完成编译。 -`make docker-build-core` 会根据这五个命令和 E2B 辅助程序构建 `oac-core:dev` 镜像（`OAC_DEV_CORE_IMAGE` 可选择其他名称）。基础镜像是通过摘要固定的 `debian:bookworm-slim`，包含 CA 证书以及辅助程序所需的 glibc 运行时；默认用户的 UID/GID 为 65532，Core 监听 `:8091`。该镜像仅支持 Linux amd64，并且不会推送到注册表。对镜像或其构建进行更改时，除了相关的源代码检查外，还必须运行 `make check-core-container`：它会在只读根文件系统上针对该镜像运行官方客户端测试套件，并且需要 Linux Docker、非 root 用户，以及服务检查中的[测试数据库和固定版本 SDK](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md#official-client-verification)（`OAC_TEST_DATABASE_URL` 指向一个已应用迁移的 `oac_*_tests` 数据库，并设置 `OAC_TEST_OFFICIAL_SDK_PYTHON`）。 +`make docker-build-core` 会根据这五个命令和 E2B 辅助程序构建 `oac-core:dev` 镜像（`OAC_DEV_CORE_IMAGE` 可选择其他名称）。基础镜像是通过摘要固定的 `debian:bookworm-slim`，包含 CA 证书以及辅助程序所需的 glibc 运行时；默认用户的 UID/GID 为 65532，Core 监听 `:8091`。此本地构建目标生成 Linux amd64 镜像；[分发构建](#build-a-distribution)生成两种架构的镜像。对镜像或其构建进行更改时，除了相关的源代码检查外，还必须运行 `make check-core-container`：它会在只读根文件系统上针对该镜像运行官方客户端测试套件，并且需要 Linux Docker、非 root 用户，以及服务检查中的[测试数据库和固定版本 SDK](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/README.md#official-client-verification)（`OAC_TEST_DATABASE_URL` 指向一个已应用迁移的 `oac_*_tests` 数据库，并设置 `OAC_TEST_OFFICIAL_SDK_PYTHON`）。 ## 发布版本 {#publish-a-version} @@ -134,13 +137,13 @@ git push origin v1.2.3 ### 容器注册表 {#container-registry} -版本发布和手动的 `build-` 草稿都会将 Linux amd64 镜像发布为 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。例如，`ghcr.io/minimax-ai/openagentcore/core:v1.2.3`。草稿使用标签 `build-`。PostgreSQL 使用其上游镜像，不会重新发布。注册表镜像从发布归档中加载，不会重新构建。仅当现有版本标签的镜像配置摘要与本次发布相同时才复用该标签；如果镜像不同，则停止发布。稳定版还会把每个组件的 `latest` 标签移到该镜像。预发布和草稿不会改动 `latest`。SemVer 构建元数据在容器标签中使用 `_` 代替 `+`；长度超过 128 个字符的版本字符串无法发布到 GHCR。镜像验证之后，发布器会上传为该发行版渲染的单个 `compose.yaml` 及其校验和清单。Compose 使用注册表摘要固定 ingress 镜像；如果镜像构建版本与 Compose 版本不同，初始化会拒绝运行。草稿 Release 保持未发布。 +版本发布和手动 `build-` 草稿使用 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。Core、Web 和 ingress 索引包含 Linux amd64 和 arm64 镜像，Runtime 包含 Linux amd64。各平台镜像使用 `-` 标签，从发行归档加载。已有版本标签必须与发行镜像及平台集合一致。发布器校验全部版本索引后，才为稳定版更新 `latest`；预发布版和草稿保持 `latest` 不变。PostgreSQL 使用上游镜像。容器标签中的 SemVer 构建元数据用 `_` 替换 `+`，版本字符串上限为 128 个字符。镜像校验后，发布器上传该版本的 `compose.yaml` 和校验和清单。Compose 用索引摘要固定 ingress，初始化时检查其构建版本与 Compose 版本一致。 合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时，GitHub 会将每个容器软件包创建为私有：软件包管理员必须先在各自的软件包设置中将全部四个软件包改为 **Public**，用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后，请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 GHCR 和 GitHub Releases 不共享事务。发布失败后，GHCR 中可能仍会保留一些匹配的版本标签；请保留这些镜像，并使用原始构件按照下文的草稿恢复流程操作。除清单缺失以外，注册表故障都会停止发布。作业摘要会记录按摘要固定的引用。这些镜像和渲染后的 Compose 文件仍需要[配置](configuration.md)中描述的配置、机密和路由。 -`install.sh` 会下载最新稳定版的 Compose 文件，或 `--version` 指定的发布版，校验 SHA-256 后启动该发布版。用法见[安装指南](getting-started/install.md#install)。 +[安装指南](getting-started/install.md#install)介绍版本选择和各平台的启动命令。 Go 检查和构建作业共享 `~/.oac/cache/` 下的 Go 模块和编译器缓存目录，缓存键由运行器 OS 和架构、全部 Go 模块文件、检查/构建分区以及提交确定。分区键可防止并发作业在同一个键下保存不同的编译器子集。发布构建既可以使用后端检查的缓存，也可以使用更早发布构建的缓存。较旧的缓存只会为下载和编译提供初始内容；每项检查仍会运行。发布作业还会缓存 npm 软件包下载内容和固定版本的 microsandbox 归档，并在每次构建时验证后者的校验和。Actions 缓存可见性遵循 GitHub ref 的作用域；特定标签的缓存不会与其他发布标签共享。只有作业成功后才会保存新键。 @@ -181,7 +184,9 @@ gh workflow run core-release --repo MiniMax-AI/OpenAgentCore --ref main \ `.github/actionlint.yaml` 会选择 hygiene 和 lint。已知工作流变更会选择其使用方：CI review 和 actionlint 工作流运行 hygiene 和 lint；原生工作流变更会添加原生检查；API 验收工作流变更会添加启用容器验收的 API 检查；网站工作流变更会添加网站检查。共享 Node 操作会选择使用它的每个作业以及 lint。新工作流或未分类的工作流/操作会选择完整门禁，直至在计划器中声明其使用方。计划器测试和 CI 测量脚本运行 hygiene；更改计划器本身会运行完整门禁。 -Compose 模板和 Compose 测试发生变更时，会同时选择 `distribution` 固定数据和 `compose` 冒烟作业；Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后，可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口，并将在 `~/.oac/tests/` 下生成构件；退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态，但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建；Web 提供占位页面而不是控制台构建。构建时的节点元数据来自 `deploy/compose/smoke-pins.json` 固定的发布版本；初始化容器禁用网络运行。该测试检查通用 Compose 行为；它不会运行 Dokploy/Coolify 实例，也不会执行模型。 +Core 安装器在 Linux、macOS 和 Windows 原生 CI 中构建并测试。Compose 冒烟测试分别使用 Linux amd64 和 arm64 原生 runner。 + +Compose 模板和 Compose 测试发生变更时，会同时选择 `distribution` 固定数据和 `compose` 冒烟作业；Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后，可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口，并将在 `~/.oac/tests/` 下生成构件；退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态，但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建；Web 提供占位页面而不是控制台构建。构建时的节点元数据来自 `deploy/compose/smoke-pins.json` 固定的发布版本；初始化容器禁用网络运行。 Go 模块和工作区输入会选择后端、API（包括容器）、原生和分发检查。每个 Node 模块都拥有自己的清单和锁文件。网站依赖项会选择网站检查；Web 依赖项会选择 Web 和浏览器检查；示例依赖项会选择示例检查；共享 TypeScript 客户端依赖项会选择 Web、浏览器和示例检查；Claude 适配器依赖项会选择 Harness、原生和分发检查。共享包管理器配置会选择所有 Node 使用方。根 TypeScript 配置会选择 Web 和示例检查；适配器 TypeScript 配置会选择 Harness 和原生检查。每个所选集合都包含 hygiene。混合变更会累加其使用方，并且每个作业都读取同一计划，而不是维护各自的路径列表。例如，仅修改通知的 PR 会跳过数据库、浏览器和原生作业，而同时修改通知和 Core 的 PR 会添加后端和 API 检查。 diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index e6635f418..e2db4d6dd 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -44,7 +44,7 @@ case "$build_network" in default|host|none) ;; *) printf 'CORE_DISTRIBUTION_BUILD_NETWORK must be default, host, or none\n' >&2; exit 1 ;; esac -# Build one Linux amd64 image and write the ID the local image store gives it to +# Build one selected Linux architecture image and write the ID the local image store gives it to # $stage/NAME.id. BuildKit's --iidfile reports the config digest, which is the # image ID only in Docker's classic store; the containerd store (Docker 29's # default) uses the manifest digest and cannot resolve the config digest. The @@ -58,14 +58,14 @@ build_image() { # declaration persists the operator's network configuration in the images. # The metadata file omits build provenance, so it does not record them either. BUILDX_METADATA_PROVENANCE=disabled docker build --network "$build_network" \ - --platform linux/amd64 --provenance=false --metadata-file "$stage/$name.build.json" \ + --platform "linux/$GOARCH" --provenance=false --metadata-file "$stage/$name.build.json" \ --label "org.opencontainers.image.revision=$revision" \ --build-arg HTTP_PROXY --build-arg HTTPS_PROXY --build-arg ALL_PROXY --build-arg NO_PROXY \ --build-arg "http_proxy=${http_proxy:-${HTTP_PROXY:-}}" \ --build-arg "https_proxy=${https_proxy:-${HTTPS_PROXY:-}}" \ --build-arg "all_proxy=${all_proxy:-${ALL_PROXY:-}}" \ --build-arg "no_proxy=${no_proxy:-${NO_PROXY:-}}" "$@" - python3 scripts/core-distribution-manifest.py built-image "$stage/$name.build.json" > "$stage/$name.id" + python3 scripts/core-distribution-manifest.py built-image "$stage/$name.build.json" "$GOARCH" > "$stage/$name.id" } require_clean_source() { @@ -255,3 +255,28 @@ fi mv "$stage/artifacts/"* "$output_dir/" if [[ -d "$stage/native-artifacts" ]]; then mv "$stage/native-artifacts/"* "$output_dir/"; fi mv "$bundle" "$output_dir/" + +# Core, Web and initialization also run natively in ARM64 Linux containers. +# Node and hosted Runtime payloads above remain linux/amd64. +export GOARCH=arm64 +arm_bundle="$stage/oac-$revision-linux-arm64" +mkdir -p "$arm_bundle/images" +OAC_DEV_BUILD_REVISION="$revision" scripts/build-core-image-context.sh "$stage/core" +OAC_DEV_WEB_BUILD_DIR="$stage/web" scripts/build-web.sh +cp "$stage/core/bin/oac" "$stage/ingress/oac" +for name in core web ingress; do + build_image "$name" "$stage/$name" + docker image save --output "$arm_bundle/images/$name.tar" "$(cat "$stage/$name.id")" +done +python3 scripts/core-distribution-manifest.py control-archive "$arm_bundle" "$stage" "$revision" arm64 +mv "$arm_bundle.tar.gz" "$arm_bundle.tar.gz.sha256" "$output_dir/" + +# The launchers and operator commands use this same portable implementation. +for platform in linux-amd64 linux-arm64 darwin-amd64 darwin-arm64 windows-amd64; do + extension="" + if [[ "$platform" == windows-* ]]; then extension=.exe; fi + asset="oac-$platform$extension" + GOOS="${platform%-*}" GOARCH="${platform#*-}" CGO_ENABLED=0 go build -mod=readonly -trimpath \ + -ldflags "-X main.buildRevision=$revision" -o "$output_dir/$asset" ./services/core/cmd/oac + (cd "$output_dir" && sha256sum "$asset" > "$asset.sha256") +done diff --git a/scripts/build-core-image-context.sh b/scripts/build-core-image-context.sh index 76d73eb32..fa8d3f0ee 100755 --- a/scripts/build-core-image-context.sh +++ b/scripts/build-core-image-context.sh @@ -13,8 +13,8 @@ if [[ "$context" != /* ]]; then fi mkdir -p "$context/bin" "$context/e2b" "$context/native-installers" -GOOS=linux GOARCH=amd64 OAC_DEV_CORE_BUILD_DIR="$context/bin" "$repo_root/scripts/build-core.sh" +GOOS=linux GOARCH="${GOARCH:-amd64}" OAC_DEV_CORE_BUILD_DIR="$context/bin" "$repo_root/scripts/build-core.sh" E2B_PROVIDER_BUILD_DIR="$context/e2b-build" "$repo_root/scripts/build-e2b-provider.sh" -tar -xzf "$context/e2b-build/oac-e2b-provider-linux-amd64.tar.gz" --strip-components=1 -C "$context/e2b" +tar -xzf "$context/e2b-build/oac-e2b-provider-linux-${GOARCH:-amd64}.tar.gz" --strip-components=1 -C "$context/e2b" rm -rf "$context/e2b-build" cp "$repo_root/deploy/distribution/Dockerfile" "$context/Dockerfile" diff --git a/scripts/build-e2b-provider.sh b/scripts/build-e2b-provider.sh index cd91fb5ec..b17f5e071 100755 --- a/scripts/build-e2b-provider.sh +++ b/scripts/build-e2b-provider.sh @@ -8,23 +8,25 @@ case "$output_dir" in *) printf 'E2B_PROVIDER_BUILD_DIR must be absolute\n' >&2; exit 1 ;; esac mkdir -p "$output_dir" -archive=oac-e2b-provider-linux-amd64.tar.gz +architecture="${GOARCH:-amd64}" +case "$architecture" in amd64|arm64) ;; *) echo "Unsupported E2B architecture" >&2; exit 1;; esac +archive="oac-e2b-provider-linux-$architecture.tar.gz" # The helper is a pure function of these files, so a build is reused by their hash. inputs="$(cd "$repo_root" && { find services/core/tools/e2b-provider -type f ! -path '*/__pycache__/*' -print0 | sort -z | xargs -0 sha256sum sha256sum LICENSE scripts/build-e2b-provider.sh } | sha256sum | cut -c1-64)" -cache="${OAC_DEV_HOME:-$HOME/.oac}/cache/e2b-provider/$inputs" +cache="${OAC_DEV_HOME:-$HOME/.oac}/cache/e2b-provider/$architecture-$inputs" if [[ ! -f "$cache/$archive.sha256" ]]; then mkdir -p "${cache%/*}" build="$(mktemp -d "${cache%/*}/.build.XXXXXX")" - image="oac-e2b-provider-build:${inputs:0:12}" + image="oac-e2b-provider-build:$architecture-${inputs:0:12}" # Proxy values are build-only operator settings; no account key is needed. - docker build --platform linux/amd64 --build-arg HTTP_PROXY --build-arg HTTPS_PROXY \ + docker build --platform "linux/$architecture" --build-arg HTTP_PROXY --build-arg HTTPS_PROXY \ --build-arg ALL_PROXY --build-arg NO_PROXY \ --file "$repo_root/services/core/tools/e2b-provider/Build.Dockerfile" \ --tag "$image" "$repo_root/services/core/tools/e2b-provider" - docker run --rm --platform linux/amd64 \ + docker run --rm --platform "linux/$architecture" \ --env HTTP_PROXY --env HTTPS_PROXY --env ALL_PROXY --env NO_PROXY \ --mount "type=bind,src=$repo_root,dst=/source,readonly" \ --mount "type=bind,src=$build,dst=/output" "$image" diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 7b45e8bbf..b8356b4fc 100644 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -58,6 +58,8 @@ (("docs/", "contracts/"), ("",), ("website",)), (("website/",), (*WEB, ".vue", ".md"), ("website",)), (("services/core/",), CORE, ("backend", "api", "compose")), + (("services/core/cmd/oac/",), GO, ("native", "distribution")), + (("deploy/install.sh", "deploy/install.ps1", "deploy/test_install.ps1"), (".sh", ".ps1"), ("native", "distribution")), (("services/core/internal/nativeinstaller/",), GO, ("native", "distribution")), (("services/core/deploy/", "services/core/tools/"), CORE, ("distribution",)), (("apps/daemon/",), GO, ("backend", "native")), diff --git a/scripts/ci_plan_test.py b/scripts/ci_plan_test.py index 23def3412..d5ee84204 100644 --- a/scripts/ci_plan_test.py +++ b/scripts/ci_plan_test.py @@ -20,7 +20,7 @@ def test_published_documents_also_build_the_website(self): self.assertEqual(self.jobs("README.md"), {"hygiene"}) def test_installer_does_not_download_a_browser_or_run_database_tests(self): - self.assertEqual(self.jobs("deploy/install.sh", "deploy/node/node_payload.py"), {"hygiene", "distribution"}) + self.assertEqual(self.jobs("deploy/install.sh", "deploy/node/node_payload.py"), {"hygiene", "distribution", "native"}) def test_compose_inputs_select_live_and_fixture_checks_without_image_builds(self): for path in ("deploy/compose/compose.yaml", "deploy/compose/https.yaml", "deploy/compose/dokploy.toml", @@ -165,13 +165,13 @@ def test_every_job_has_a_plan_condition(self): def test_mixed_changes_accumulate(self): self.assertEqual(self.jobs("docs/maintainers.md", "deploy/install.sh", "apps/web/src/app.tsx"), - {"hygiene", "distribution", "web", "web-acceptance", "website"}) + {"hygiene", "distribution", "native", "web", "web-acceptance", "website"}) def test_installer_pr_300_replay(self): self.assertEqual(self.jobs( "deploy/install.sh", "deploy/README.md", "deploy/node/node_install.py", "deploy/node/install_display.py", "deploy/node/node_payload.py", - "docs/getting-started/install.md"), {"hygiene", "distribution", "website"}) + "docs/getting-started/install.md"), {"hygiene", "distribution", "native", "website"}) def test_workflow_graph_cannot_silently_omit_or_add_a_gate_dependency(self): workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/check.yml").read_text().split("jobs:\n", 1)[1] diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py index 2a6b27fa8..f2c4c9946 100644 --- a/scripts/compose-smoke.py +++ b/scripts/compose-smoke.py @@ -55,7 +55,7 @@ def prepare_pinned_payload(destination): def build_images(directory, tag): revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() protocol = re.search(r'const Version = "([^"]+)"', (ROOT / 'internal/agentdaemon/proto/version.go').read_text()).group(1) - go_env = {**os.environ, 'CGO_ENABLED': '0', 'GOOS': 'linux', 'GOARCH': 'amd64'} + go_env = {**os.environ, 'CGO_ENABLED': '0', 'GOOS': 'linux', 'GOARCH': os.environ.get('GOARCH', 'amd64')} def go_build(package, output, build_revision=revision): output.parent.mkdir(parents=True, exist_ok=True) @@ -88,7 +88,7 @@ def go_build(package, output, build_revision=revision): images = {} for name, context in contexts.items(): images[name] = f'oac-smoke/{name}:{tag}' - subprocess.run(['docker', 'build', '-q', '--platform', 'linux/amd64', '-t', images[name], str(context)], + subprocess.run(['docker', 'build', '-q', '--platform', 'linux/' + go_env['GOARCH'], '-t', images[name], str(context)], check=True, stdout=subprocess.DEVNULL) return images @@ -113,7 +113,7 @@ def main(): images = build_images(directory, project.removeprefix('oac-smoke-')) override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}}})) - env = {**os.environ, 'COMPOSE_PROGRESS': 'plain', 'OAC_DATA_DIR': str(data), + env = {**os.environ, 'COMPOSE_PROGRESS': 'plain', 'OAC_HOST': '127.0.0.1', 'OAC_WEB_PORT': '0', **{'OAC_IMAGE_' + name.upper(): image for name, image in images.items()}} env.pop('OAC_PUBLIC_URL', None) @@ -212,7 +212,18 @@ def terminate(_signum, _frame): assert any(p['id'] == project_data['id'] for p in get('/core/v1/projects')['data']), 'Project was lost' assert get('/v1/files/' + uploaded['id'], headers=api)['bytes'] == len(content), 'Uploaded file metadata was lost' assert 'Bundled node installation metadata verified' not in private_logs(key, project_key), 'Completed initialization recopied metadata' - print('PASS: startup, origin validation, sign-in, API, upload, node installer and persistent installation', flush=True) + compose('run', '--rm', '--no-deps', 'init', '/usr/local/bin/oac', 'rotate-volume-key') + compose('restart', 'core', 'web') + compose('up', '-d', '--wait', '--wait-timeout', '120', timeout=180) + rotated = compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() + assert rotated != key, 'Core key was not rotated' + browser = client() + request('/console/auth/login', {'core_key': rotated}) + compose('down') + compose('up', '-d', '--wait', '--wait-timeout', '120', timeout=180) + assert compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() == rotated, 'Rotated key was not retained' + private_logs(key, rotated, project_key) + print('PASS: startup, origin validation, sign-in, API, upload, node installer, key rotation and persistent installation', flush=True) except BaseException: # Service status identifies failed containers without dumping secret-bearing logs. status = subprocess.run(command + ['ps', '--all'], env=env, capture_output=True, timeout=30) @@ -220,9 +231,6 @@ def terminate(_signum, _frame): raise finally: compose('down', '--volumes', '--remove-orphans', timeout=60) - # Match the host installer's cleanup without requiring tools in scratch init. - compose('run', '--rm', '--no-deps', '--volume', str(data) + ':/data', - '--entrypoint', 'find', 'database', '/data', '-mindepth', '1', '-delete') subprocess.run(['docker', 'image', 'rm', '-f', *images.values()], capture_output=True, timeout=60) diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index c00f363b1..3529fb8fe 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -70,26 +70,26 @@ def sha256(path): return digest.hexdigest() -def verify_image(image): +def verify_image(image, architecture="amd64"): if not DIGEST.fullmatch(image): raise ValueError("Distribution image inputs must be immutable sha256 image IDs") details = json.loads(subprocess.check_output(["docker", "image", "inspect", image], text=True))[0] - if details["Id"] != image or details["Os"] != "linux" or details["Architecture"] != "amd64": - raise ValueError("Distribution images must be the selected Linux amd64 image") + if details["Id"] != image or details["Os"] != "linux" or details["Architecture"] != architecture: + raise ValueError("Distribution images must match the selected Linux architecture") return details -def built_image(metadata_file): +def built_image(metadata_file, architecture="amd64"): """Print the local store ID of the image one BuildKit build just produced.""" metadata = json.loads(pathlib.Path(metadata_file).read_text()) # The classic store names an image by its config digest, the containerd store # by its manifest digest; the other value never resolves to itself there. config = metadata.get("containerimage.config.digest") manifest = metadata.get("containerimage.digest", config) - print(resolve_image(config, manifest)) + print(resolve_image(config, manifest, architecture)) -def resolve_image(config, manifest): +def resolve_image(config, manifest, architecture="amd64"): """Resolve the archive identities in either supported Docker image store.""" if not all(isinstance(value, str) and DIGEST.fullmatch(value) for value in (config, manifest)): raise ValueError("Build metadata lacks valid image digests") @@ -101,11 +101,11 @@ def resolve_image(config, manifest): resolved.append(candidate) if len(resolved) != 1: raise ValueError("The local image store does not identify the built image by exactly one of its digests") - verify_image(resolved[0]) + verify_image(resolved[0], architecture) return resolved[0] -def image_identities(archive, build_id): +def image_identities(archive, build_id, architecture="amd64"): """Bind both Docker store identities to one exported Linux amd64 image.""" if not DIGEST.fullmatch(build_id): raise ValueError("Missing immutable distribution image identity") @@ -161,7 +161,7 @@ def blob(descriptor, parse=False): config_descriptor = image.get("config", {}) config = blob(config_descriptor, parse=True) config_digest = config_descriptor["digest"] - if config.get("os") != "linux" or config.get("architecture") != "amd64": + if config.get("os") != "linux" or config.get("architecture") != architecture: raise ValueError("Image archive contains an unexpected platform") for layer in image.get("layers", []): blob(layer) @@ -170,6 +170,21 @@ def blob(descriptor, parse=False): return config_digest, manifest_digest +def control_archive(bundle, stage, revision, architecture): + bundle, stage = pathlib.Path(bundle), pathlib.Path(stage) + manifest = {"source_commit": revision, "platform": "linux/" + architecture, + "images": {}, "image_manifest_digests": {}} + for name in ("core", "web", "ingress"): + config, digest = image_identities(bundle / "images" / (name + ".tar"), + (stage / (name + ".id")).read_text().strip(), architecture) + manifest["images"][name], manifest["image_manifest_digests"][name] = config, digest + (bundle / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + target = bundle.with_name(bundle.name + ".tar.gz") + with tarfile.open(target, "w:gz") as output: + output.add(bundle, arcname=bundle.name) + target.with_name(target.name + ".sha256").write_text(sha256(target) + " " + target.name + "\n") + + def verify_runtime(image, daemon, source): details = verify_image(image) source = pathlib.Path(source) @@ -583,7 +598,7 @@ def check(image, link): if __name__ == "__main__": - commands = {"extract-runtime": extract_runtime, "verify-runtime": verify_runtime, "verify-image": verify_image, + commands = {"control-archive": control_archive, "extract-runtime": extract_runtime, "verify-runtime": verify_runtime, "verify-image": verify_image, "built-image": built_image, "node-payload": node_payload, "manifest": manifest, "archive": archive, "bootstraps": bootstraps, "release-base": release_base, "docs": docs, "native-catalog": native_catalog, "native-offline": native_offline} try: diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index 4719f05a1..cb4b78ce8 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -143,6 +143,13 @@ def test_oci_manifest_identity_is_distinct_from_docker_config_identity(self): digest, name = line.split(" ", 1) self.assertEqual(digest, distribution.sha256(self.bundle / name)) + def test_arm_archive_uses_explicit_platform_validation(self): + path = self.stage / "arm.tar" + config, digest = image_archive(path, "core", architecture="arm64") + self.assertEqual(distribution.image_identities(path, config, "arm64"), (config, digest)) + with self.assertRaisesRegex(ValueError, "unexpected platform"): + distribution.image_identities(path, config, "amd64") + def test_built_image_records_the_id_each_docker_store_resolves(self): config, manifest, other = ("sha256:" + digit * 64 for digit in "123") metadata = self.stage / "build.json" @@ -163,7 +170,7 @@ def test_built_image_records_the_id_each_docker_store_resolves(self): mock.patch("builtins.print") as output: if expected.startswith("sha256:"): distribution.built_image(metadata) - verify.assert_called_once_with(expected) + verify.assert_called_once_with(expected, "amd64") output.assert_called_once_with(expected) else: with self.assertRaisesRegex(ValueError, expected): diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index 7f27e0306..8ae61caa6 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -97,7 +97,7 @@ def registry_image(reference): if manifest is not None and "manifests" in manifest: descriptors = manifest["manifests"] if len(descriptors) != 1: - raise ValueError("Expected one Linux amd64 registry image: " + reference) + raise ValueError("Expected one platform registry image: " + reference) digest = descriptors[0]["digest"] if not distribution.DIGEST.fullmatch(digest): raise ValueError("Invalid registry image descriptor") @@ -109,87 +109,112 @@ def registry_image(reference): def publish_images(assets, repository, revision, tag, floating_latest=False): - """Load the checked release archives; never rebuild or replace another image.""" + """Verify both architectures before publishing immutable platform tags and indexes.""" image_tag = tag.replace("+", "_") - if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", image_tag): + if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,120}", image_tag): raise ValueError("Release version exceeds the container tag format") - stem = "oac-" + revision + "-linux-amd64" - # Extract named regular members only, never archive-controlled paths. with tempfile.TemporaryDirectory(prefix="oac-ghcr-") as directory: directory = pathlib.Path(directory) - with tarfile.open(assets / (stem + ".tar.gz"), "r:gz") as archive: - manifest = json.load(archive.extractfile(stem + "/manifest.json")) - if manifest["source_commit"] != revision or manifest["platform"] != "linux/amd64": - raise ValueError("Registry images do not match the release") - for name in IMAGE_NAMES: - if name == "runtime": - continue - member = archive.getmember(stem + "/images/" + name + ".tar") - if not member.isfile(): - raise ValueError("Expected a regular image archive") - with archive.extractfile(member) as source, (directory / (name + ".tar")).open("wb") as target: - shutil.copyfileobj(source, target) - runtime = manifest["artifacts"]["images/runtime.tar.gz"] - filename = runtime["filename"] - if pathlib.Path(filename).name != filename: - raise ValueError("Invalid Runtime asset filename") - runtime_path = assets / filename - if runtime_path.is_symlink() or distribution.sha256(runtime_path) != runtime["sha256"]: - raise ValueError("Runtime image checksum mismatch") - with gzip.open(runtime_path, "rb") as source, (directory / "runtime.tar").open("wb") as target: - shutil.copyfileobj(source, target) - for name in IMAGE_NAMES: - expected = (manifest["images"][name], manifest["image_manifest_digests"][name]) - if distribution.image_identities(directory / (name + ".tar"), expected[0]) != expected: - raise ValueError("Release image identity mismatch: " + name) + entries = {} + for architecture in ("amd64", "arm64"): + stem = "oac-" + revision + "-linux-" + architecture + with tarfile.open(assets / (stem + ".tar.gz"), "r:gz") as archive: + manifest = json.load(archive.extractfile(stem + "/manifest.json")) + if manifest["source_commit"] != revision or manifest["platform"] != "linux/" + architecture: + raise ValueError("Registry images do not match the release") + names = IMAGE_NAMES if architecture == "amd64" else ("core", "web", "ingress") + for name in names: + path = directory / (name + "-" + architecture + ".tar") + if name == "runtime": + artifact = manifest["artifacts"]["images/runtime.tar.gz"] + filename = artifact["filename"] + if pathlib.Path(filename).name != filename: + raise ValueError("Invalid Runtime asset filename") + compressed = assets / filename + if compressed.is_symlink() or distribution.sha256(compressed) != artifact["sha256"]: + raise ValueError("Runtime image checksum mismatch") + with gzip.open(compressed, "rb") as source, path.open("wb") as target: + shutil.copyfileobj(source, target) + else: + member = archive.getmember(stem + "/images/" + name + ".tar") + if not member.isfile(): + raise ValueError("Expected a regular image archive") + with archive.extractfile(member) as source, path.open("wb") as target: + shutil.copyfileobj(source, target) + expected = (manifest["images"][name], manifest["image_manifest_digests"][name]) + if distribution.image_identities(path, expected[0], architecture) != expected: + raise ValueError("Release image identity mismatch: " + name) + entries[name, architecture] = (path, *expected) references = {} - # Validate every local image and every existing tag before the first push. - for name in IMAGE_NAMES: - path = directory / (name + ".tar") + for (name, architecture), (path, config, digest) in entries.items(): subprocess.run(["docker", "load", "--input", str(path)], check=True) - config = manifest["images"][name] - local = distribution.resolve_image(config, manifest["image_manifest_digests"][name]) - reference = "ghcr.io/" + repository.lower() + "/" + name + ":" + image_tag - remote, selected = registry_image(reference) + local = distribution.resolve_image(config, digest, architecture) + reference = "ghcr.io/" + repository.lower() + "/" + name + ":" + image_tag + "-" + architecture + remote, _ = registry_image(reference) if remote is not None and remote.get("config", {}).get("digest") != config: raise ValueError("Registry tag already names a different image: " + reference) - references[name] = (reference, config, local, remote) + references[name, architecture] = (reference, config, local, remote) + # Version indexes are immutable too. Check every existing one before pushing. + bases = {name: "ghcr.io/" + repository.lower() + "/" + name + ":" + image_tag for name in IMAGE_NAMES} + for name, reference in bases.items(): + current = registry_manifest(reference) + if current is not None: + verify_index(reference, current, {arch: entry[1] for (n, arch), entry in entries.items() if n == name}) + def push_image(item): - name, (reference, config, local, remote) = item - print("Publishing registry image " + name, flush=True) + key, (reference, config, local, remote) = item if remote is None: subprocess.run(["docker", "tag", local, reference], check=True) subprocess.run(["docker", "push", reference], check=True) remote, selected = registry_image(reference) if remote is None or remote.get("config", {}).get("digest") != config: raise ValueError("Registry image verification failed: " + reference) - # Inspect the registry's descriptor, not the local Docker image ID. details = json.loads(subprocess.check_output( ["docker", "manifest", "inspect", "--verbose", selected], text=True)) digest = details["Descriptor"]["digest"] if not distribution.DIGEST.fullmatch(digest): raise ValueError("Invalid registry manifest digest") - print("Verified registry image " + name, flush=True) - return name, {"tag": reference, "digest": reference.rsplit(":", 1)[0] + "@" + digest} - result = dict(sorted(parallel_each(push_image, references.items()))) + return key, reference.rsplit(":", 1)[0] + "@" + digest + platforms = dict(parallel_each(push_image, references.items())) + result = {} + for name, reference in bases.items(): + sources = [value for (n, _), value in platforms.items() if n == name] + expected = {arch: entry[1] for (n, arch), entry in entries.items() if n == name} + if registry_manifest(reference) is None: + subprocess.run(["docker", "buildx", "imagetools", "create", "--tag", reference, *sources], check=True) + verify_index(reference, registry_manifest(reference), expected) + digest = subprocess.check_output(["docker", "buildx", "imagetools", "inspect", reference, + "--format", "{{.Manifest.Digest}}"], text=True).strip() + if not distribution.DIGEST.fullmatch(digest): + raise ValueError("Invalid registry index digest") + pinned = reference.rsplit(":", 1)[0] + "@" + digest + result[name] = {"tag": reference, "digest": pinned} + # Advance floating tags only after all version indexes are verified. if floating_latest: - def push_latest(item): - name, (reference, config, local, remote) = item - latest = reference.rsplit(":", 1)[0] + ":latest" - current, _selected = registry_image(latest) - if current is not None and current.get("config", {}).get("digest") == config: - return name, latest - print("Publishing registry image " + name + ":latest", flush=True) - subprocess.run(["docker", "tag", local, latest], check=True) - subprocess.run(["docker", "push", latest], check=True) - current, selected = registry_image(latest) - if current is None or current.get("config", {}).get("digest") != config: - raise ValueError("Registry image verification failed: " + latest) - return name, latest - parallel_each(push_latest, references.items()) + for name, entry in result.items(): + latest = bases[name].rsplit(":", 1)[0] + ":latest" + expected = {arch: value[1] for (n, arch), value in entries.items() if n == name} + subprocess.run(["docker", "buildx", "imagetools", "create", "--tag", latest, entry["digest"]], check=True) + verify_index(latest, registry_manifest(latest), expected) return result +def verify_index(reference, index, expected): + if not index or len(index.get("manifests", [])) != len(expected): + raise ValueError("Registry index has unexpected platforms: " + reference) + actual = {} + for descriptor in index["manifests"]: + platform = descriptor.get("platform", {}) + architecture = platform.get("architecture") + digest = descriptor.get("digest", "") + if platform.get("os") != "linux" or architecture in actual or not distribution.DIGEST.fullmatch(digest): + raise ValueError("Invalid registry platform descriptor") + child = registry_manifest(reference.rsplit(":", 1)[0] + "@" + digest) + actual[architecture] = (child or {}).get("config", {}).get("digest") + if actual != expected: + raise ValueError("Registry index names a different image: " + reference) + + def publish(assets, repository, revision, tag, mode): if not REPOSITORY.fullmatch(repository): raise ValueError("Expected an owner/repository") @@ -209,7 +234,8 @@ def publish(assets, repository, revision, tag, mode): # The builder validates the manifest and Runtime assets. Verify archives again # after the Actions artifact transfer between jobs. stem = "oac-" + revision + "-linux-amd64" - archives = [assets / (stem + ".tar.gz"), assets / "install.sh"] + archives = [assets / (stem + ".tar.gz"), assets / ("oac-" + revision + "-linux-arm64.tar.gz"), assets / "install.sh", assets / "install.ps1"] + archives.extend(assets / name for name in ("oac-linux-amd64", "oac-linux-arm64", "oac-darwin-amd64", "oac-darwin-arm64", "oac-windows-amd64.exe")) if mode == "publish" or (assets / (stem + "-offline.tar.gz")).exists(): archives.append(assets / (stem + "-offline.tar.gz")) for archive in archives: @@ -244,7 +270,7 @@ def publish(assets, repository, revision, tag, mode): release = api(repository, "releases", "--method", "POST", "-f", "tag_name=" + tag, "-f", "target_commitish=" + revision, "-f", "name=OpenAgentCore " + tag, - "-f", "body=Linux amd64 distribution from commit " + revision + ".", + "-f", "body=Core for Linux, macOS and Windows; Linux amd64 Node distribution from commit " + revision + ".", "-F", "draft=true", "-F", "prerelease=" + str(prerelease).lower()) verify_draft(release, tag, revision) release_id = release["id"] diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index 29ff5258b..175bc9cd6 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -29,7 +29,7 @@ def setUp(self): self.assets = pathlib.Path(self.temp.name) self.revision = "a" * 40 self.stem = "oac-" + self.revision + "-linux-amd64" - for name in (self.stem + ".tar.gz", self.stem + "-offline.tar.gz", "install.sh"): + for name in (self.stem + ".tar.gz", self.stem + "-offline.tar.gz", "install.sh", "install.ps1", "oac-" + self.revision + "-linux-arm64.tar.gz", "oac-linux-amd64", "oac-linux-arm64", "oac-darwin-amd64", "oac-darwin-arm64", "oac-windows-amd64.exe"): (self.assets / name).write_bytes(b"archive fixture") (self.assets / (name + ".sha256")).write_text( hashlib.sha256(b"archive fixture").hexdigest() + " " + name + "\n") @@ -68,7 +68,7 @@ def test_draft_publishes_images_and_stays_unpublished(self): self.publish(tag="build-" + self.revision, mode="draft") self.images.assert_called_once() self.assertTrue(self.release["draft"]) - self.assertEqual(len(self.release["assets"]), 14) + self.assertEqual(len(self.release["assets"]), 28) def test_missing_native_asset_refuses_release_creation(self): (self.assets / f"oac-native-{self.revision}-windows-amd64.tar.gz").unlink() @@ -142,7 +142,7 @@ def response(repo, endpoint, *args): return result if endpoint == "releases/7": self.assertEqual(active, 0) - self.assertIn(len(self.release["assets"]), (12, 14)) + self.assertIn(len(self.release["assets"]), (26, 28)) return self.response(repo, endpoint, *args) self.api.side_effect = response self.publish() @@ -153,7 +153,7 @@ def test_version_tag_publishes_complete_fixed_id(self): self.publish() self.assertFalse(self.release["draft"]) self.assertFalse(self.release["prerelease"]) - self.assertEqual(len(self.release["assets"]), 14) + self.assertEqual(len(self.release["assets"]), 28) self.assertEqual({a["name"] for a in self.release["assets"] if a["name"].endswith(".yaml")}, {"compose.yaml"}) self.assertEqual(self.api.call_args.args[1:], ("releases/7", "--method", "PATCH", "-F", "draft=false")) @@ -343,142 +343,112 @@ def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.assets = pathlib.Path(self.temp.name) - self.revision = "a" * 40 - self.config = "sha256:" + "b" * 64 - self.digest = "sha256:" + "c" * 64 - runtime = self.assets / "runtime.tar.gz" - runtime.write_bytes(gzip.compress(b"runtime")) - self.manifest = { - "source_commit": self.revision, "platform": "linux/amd64", - "images": dict.fromkeys(publisher.IMAGE_NAMES, self.config), - "image_manifest_digests": dict.fromkeys(publisher.IMAGE_NAMES, self.digest), - "artifacts": {"images/runtime.tar.gz": { - "filename": runtime.name, "sha256": publisher.distribution.sha256(runtime)}}} - with tarfile.open(self.assets / ("oac-" + self.revision + "-linux-amd64.tar.gz"), "w:gz") as archive: - for name, data in [("manifest.json", json.dumps(self.manifest).encode())] + [ - ("images/" + name + ".tar", b"image") for name in ("core", "web", "ingress")]: - member = tarfile.TarInfo("oac-" + self.revision + "-linux-amd64/" + name) - member.size = len(data) - archive.addfile(member, io.BytesIO(data)) - stack = contextlib.ExitStack() - self.addCleanup(stack.close) - self.identities = stack.enter_context(mock.patch.object(publisher.distribution, "image_identities", return_value=(self.config, self.digest))) - stack.enter_context(mock.patch.object(publisher.distribution, "resolve_image", return_value=self.digest)) - self.run = stack.enter_context(mock.patch.object(publisher.subprocess, "run")) - stack.enter_context(mock.patch.object(publisher.subprocess, "check_output", return_value=json.dumps({"Descriptor": {"digest": self.digest}}))) - self.remote = stack.enter_context(mock.patch.object(publisher, "registry_manifest", return_value={"config": {"digest": self.config}})) - - def publish(self, tag="v1.2.3"): - return publisher.publish_images(self.assets, "MiniMax-AI/OpenAgentCore", self.revision, tag) + self.revision = 'a' * 40 + self.configs = {'amd64': 'sha256:' + '1' * 64, 'arm64': 'sha256:' + '2' * 64} + self.digests = {'amd64': 'sha256:' + '3' * 64, 'arm64': 'sha256:' + '4' * 64} + self.index_digest = 'sha256:' + '5' * 64 + self.remote_images = {} + for arch in ('amd64', 'arm64'): + names = publisher.IMAGE_NAMES if arch == 'amd64' else ('core', 'web', 'ingress') + manifest = {'source_commit': self.revision, 'platform': 'linux/' + arch, + 'images': dict.fromkeys(names, self.configs[arch]), + 'image_manifest_digests': dict.fromkeys(names, self.digests[arch])} + if arch == 'amd64': + runtime = self.assets / 'runtime.tar.gz' + runtime.write_bytes(gzip.compress(b'runtime')) + manifest['artifacts'] = {'images/runtime.tar.gz': {'filename': runtime.name, 'sha256': publisher.distribution.sha256(runtime)}} + stem = 'oac-' + self.revision + '-linux-' + arch + with tarfile.open(self.assets / (stem + '.tar.gz'), 'w:gz') as archive: + for name, raw in [('manifest.json', json.dumps(manifest).encode())] + [('images/' + n + '.tar', b'image') for n in names if n != 'runtime']: + member = tarfile.TarInfo(stem + '/' + name); member.size = len(raw) + archive.addfile(member, io.BytesIO(raw)) + stack = contextlib.ExitStack(); self.addCleanup(stack.close) + self.identities = stack.enter_context(mock.patch.object(publisher.distribution, 'image_identities', side_effect=lambda path, config, arch: (self.configs[arch], self.digests[arch]))) + stack.enter_context(mock.patch.object(publisher.distribution, 'resolve_image', side_effect=lambda config, digest, arch: digest)) + self.run = stack.enter_context(mock.patch.object(publisher.subprocess, 'run', side_effect=self.execute)) + stack.enter_context(mock.patch.object(publisher.subprocess, 'check_output', side_effect=self.output)) + self.remote = stack.enter_context(mock.patch.object(publisher, 'registry_manifest', side_effect=self.lookup)) + + def lookup(self, reference): + if '@' in reference: + for arch, digest in self.digests.items(): + if reference.endswith(digest): return {'config': {'digest': self.configs[arch]}} + return self.remote_images.get(reference) + + def execute(self, command, **kwargs): + if command[1] == 'push': + ref = command[-1]; arch = ref.rsplit('-', 1)[1] + self.remote_images[ref] = {'config': {'digest': self.configs[arch]}} + if command[1:4] == ['buildx', 'imagetools', 'create']: + ref = command[5]; name = ref.rsplit('/', 1)[1].split(':')[0] + arches = ('amd64',) if name == 'runtime' else ('amd64', 'arm64') + self.remote_images[ref] = {'manifests': [{'platform': {'os': 'linux', 'architecture': arch}, 'digest': self.digests[arch]} for arch in arches]} + + def output(self, command, **kwargs): + if command[1] == 'buildx': return self.index_digest + ref = command[-1] + arch = 'arm64' if ref.endswith('arm64') else 'amd64' + return json.dumps({'Descriptor': {'digest': self.digests[arch]}}) + + def publish(self, **kwargs): + return publisher.publish_images(self.assets, 'MiniMax-AI/OpenAgentCore', self.revision, 'v1.2.3', **kwargs) def pushes(self): - return [c.args[0] for c in self.run.call_args_list if c.args[0][1] == "push"] - - def test_matching_tags_are_reused_and_receipts_use_registry_digest(self): - result = self.publish() - self.assertEqual(self.pushes(), []) - self.assertEqual(result["core"]["digest"], "ghcr.io/minimax-ai/openagentcore/core@" + self.digest) - - def test_stable_release_moves_latest_after_the_version_tags(self): - seen = {} - def remote(reference): - seen[reference] = seen.get(reference, 0) + 1 - if seen[reference] == 1: - return None - return {"config": {"digest": self.config}} - self.remote.side_effect = remote - publisher.publish_images(self.assets, "MiniMax-AI/OpenAgentCore", self.revision, "v1.2.3", floating_latest=True) - pushed = [command[-1].rsplit("/", 1)[-1] for command in self.pushes()] - self.assertEqual(sorted(name for name in pushed if name.endswith(":v1.2.3")), - sorted(name + ":v1.2.3" for name in publisher.IMAGE_NAMES)) - self.assertEqual(sorted(name for name in pushed if name.endswith(":latest")), - sorted(name + ":latest" for name in publisher.IMAGE_NAMES)) - - def test_matching_latest_tag_is_reused(self): - publisher.publish_images(self.assets, "MiniMax-AI/OpenAgentCore", self.revision, "v1.2.3", floating_latest=True) - self.assertEqual(self.pushes(), []) + return [call.args[0] for call in self.run.call_args_list if call.args[0][1] == 'push'] - def test_new_images_use_resolved_store_identity_and_version_only(self): - self.remote.side_effect = [None] * 4 + [{"config": {"digest": self.config}}] * 4 - result = self.publish("v1.2.3-rc.1+build.2") - self.assertEqual(len(self.pushes()), 4) - self.assertTrue(all(c[-1].endswith(":v1.2.3-rc.1_build.2") for c in self.pushes())) - tags = [c.args[0] for c in self.run.call_args_list if c.args[0][1] == "tag"] - self.assertTrue(all(c[2] == self.digest for c in tags)) - self.assertEqual(len(result), 4) - - def test_single_platform_indexes_are_verified_by_child_config(self): - index = {"manifests": [{"digest": self.digest}]} - image = {"config": {"digest": self.config}} - self.remote.side_effect = lambda reference: image if "@" in reference else index + def test_publishes_and_reuses_verified_multiarch_indexes(self): result = self.publish() + self.assertEqual(len(self.pushes()), 7) + self.assertEqual(result['ingress']['digest'], 'ghcr.io/minimax-ai/openagentcore/ingress@' + self.index_digest) + self.assertEqual(len(self.remote_images['ghcr.io/minimax-ai/openagentcore/core:v1.2.3']['manifests']), 2) + self.run.reset_mock(); self.publish(); self.assertEqual(self.pushes(), []) + + def test_latest_updates_after_all_version_indexes(self): + self.publish(floating_latest=True) + creates = [c.args[0][5] for c in self.run.call_args_list if c.args[0][1:4] == ['buildx', 'imagetools', 'create']] + self.assertTrue(all(ref.endswith(':v1.2.3') for ref in creates[:4])) + self.assertTrue(all(ref.endswith(':latest') for ref in creates[4:])) + + def test_conflicting_platform_prevents_every_push(self): + self.remote_images['ghcr.io/minimax-ai/openagentcore/web:v1.2.3-arm64'] = {'config': {'digest': 'different'}} + with self.assertRaisesRegex(ValueError, 'different image'): self.publish() self.assertEqual(self.pushes(), []) - self.assertEqual(result["core"]["digest"], "ghcr.io/minimax-ai/openagentcore/core@" + self.digest) - self.assertTrue(any("@" in call.args[0] for call in self.remote.call_args_list)) - def test_multi_image_index_is_rejected(self): - self.remote.return_value = {"manifests": [{"digest": self.digest}] * 2} - with self.assertRaisesRegex(ValueError, "Expected one"): - self.publish() + def test_conflicting_index_prevents_every_push(self): + self.remote_images['ghcr.io/minimax-ai/openagentcore/core:v1.2.3'] = {'manifests': []} + with self.assertRaisesRegex(ValueError, 'unexpected platforms'): self.publish() self.assertEqual(self.pushes(), []) - def test_conflicting_tag_prevents_all_pushes(self): - self.remote.side_effect = [None, {"config": {"digest": "different"}}] - with self.assertRaisesRegex(ValueError, "different image"): - self.publish() - self.assertEqual(self.pushes(), []) + def test_missing_arm_archive_prevents_loading(self): + (self.assets / ('oac-' + self.revision + '-linux-arm64.tar.gz')).unlink() + with self.assertRaises(FileNotFoundError): self.publish() + self.run.assert_not_called() def test_corrupt_runtime_prevents_loading(self): - (self.assets / "runtime.tar.gz").write_bytes(b"corrupt") - with self.assertRaisesRegex(ValueError, "checksum"): - self.publish() + (self.assets / 'runtime.tar.gz').write_bytes(b'corrupt') + with self.assertRaisesRegex(ValueError, 'checksum'): self.publish() self.run.assert_not_called() - def test_archive_identity_mismatch_prevents_loading(self): - self.identities.return_value = (self.config, "different") - with self.assertRaisesRegex(ValueError, "identity mismatch"): - self.publish() + def test_wrong_archive_identity_prevents_loading(self): + self.identities.side_effect = None; self.identities.return_value = ('wrong', 'wrong') + with self.assertRaisesRegex(ValueError, 'identity mismatch'): self.publish() self.run.assert_not_called() - def test_failed_push_stops_publication(self): - self.remote.return_value = None - def run(command, **kwargs): - if command[1] == "push": - raise subprocess.CalledProcessError(1, command) - self.run.side_effect = run - with self.assertRaises(subprocess.CalledProcessError): - self.publish() - self.assertGreaterEqual(len(self.pushes()), 1) - self.assertEqual(len({command[-1] for command in self.pushes()}), len(self.pushes())) - - def test_registry_pushes_overlap_after_all_preflight_checks(self): - barrier = threading.Barrier(4, timeout=5) - pushed = set() - lock = threading.Lock() - def remote(reference): - with lock: - return {"config": {"digest": self.config}} if reference in pushed else None - def run(command, **kwargs): - if command[1] == "push": - self.assertEqual(sum(c.args[0][1] == "load" for c in self.run.call_args_list), 4) - barrier.wait() - with lock: - pushed.add(command[-1]) - self.remote.side_effect = remote - self.run.side_effect = run - self.assertEqual(len(self.publish()), 4) - self.assertEqual(len(pushed), 4) + def test_push_failure_stops_index_publication(self): + def fail(command, **kwargs): + if command[1] == 'push': raise subprocess.CalledProcessError(1, command) + self.run.side_effect = fail + with self.assertRaises(subprocess.CalledProcessError): self.publish() + self.assertFalse(any(c.args[0][1] == 'buildx' for c in self.run.call_args_list)) def test_registry_auth_failure_is_not_missing_image(self): - # Test the actual inspection function separately from the publication fixture. - with mock.patch.object(publisher.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "unauthorized")): - with self.assertRaisesRegex(RuntimeError, "Cannot inspect"): - REAL_REGISTRY_MANIFEST("ghcr.io/example/core:v1") + with mock.patch.object(publisher.subprocess, 'run', return_value=subprocess.CompletedProcess([], 1, '', 'unauthorized')): + with self.assertRaisesRegex(RuntimeError, 'Cannot inspect'): REAL_REGISTRY_MANIFEST('ghcr.io/example/core:v1') def test_registry_missing_manifest(self): - with mock.patch.object(publisher.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "manifest unknown")): - self.assertIsNone(REAL_REGISTRY_MANIFEST("ghcr.io/example/core:v1")) + with mock.patch.object(publisher.subprocess, 'run', return_value=subprocess.CompletedProcess([], 1, '', 'manifest unknown')): + self.assertIsNone(REAL_REGISTRY_MANIFEST('ghcr.io/example/core:v1')) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/services/core/cmd/oac/init.go b/services/core/cmd/oac/init.go index 7c8f6c8b4..432d9da60 100644 --- a/services/core/cmd/oac/init.go +++ b/services/core/cmd/oac/init.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "strings" - "syscall" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" @@ -45,7 +44,6 @@ func initCommand() error { log.Bg().Error("Initialization failed", "step", "validate_revision", "revision", revision, "error", err) return err } - syscall.Umask(0o077) release := releaseIdentity{revision} return initialize("/data", release, func() (map[string][]byte, error) { return readRelease("/opt/oac/node-payload", release) @@ -173,131 +171,129 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ } } nextStep("acquire_lock") - lock, err := os.OpenFile(filepath.Join(root, "secrets", ".init.lock"), os.O_CREATE|os.O_WRONLY, 0o600) - if err != nil { - return err - } - defer lock.Close() - if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { - return err - } - nextStep("verify_existing_installation") - marker := filepath.Join(root, "installation.json") - if raw, err := os.ReadFile(marker); err == nil { - var receipt installReceipt - if err := json.Unmarshal(raw, &receipt); err != nil { + return withLock(filepath.Join(root, "secrets", "init"), func() error { + + nextStep("verify_existing_installation") + marker := filepath.Join(root, "installation.json") + if raw, err := os.ReadFile(marker); err == nil { + var receipt installReceipt + if err := json.Unmarshal(raw, &receipt); err != nil { + return err + } + if receipt.SourceCommit != release.revision { + return errors.New("this data directory belongs to another release; create a new installation") + } + for name, checksum := range receipt.Files { + actual, err := fileDigest(filepath.Join(root, name)) + if err != nil { + return err + } + if actual != checksum { + return errors.New("installation files changed; restore the matching data directory") + } + } + if err := syncCoreKeyDigest(root); err != nil { + return err + } + logger.InfoContext(ctx, "Existing installation verified", "file_count", len(receipt.Files)) + return nil + } else if !errors.Is(err, fs.ErrNotExist) { return err } - if receipt.SourceCommit != release.revision { - return errors.New("this data directory belongs to another release; create a new installation") - } - for name, checksum := range receipt.Files { - actual, err := fileDigest(filepath.Join(root, name)) + nextStep("verify_empty_data") + for _, name := range []string{"database", "state"} { + entries, err := os.ReadDir(filepath.Join(root, name)) if err != nil { return err } - if actual != checksum { - return errors.New("installation files changed; restore the matching data directory") + if len(entries) > 0 { + return errors.New("existing data requires its original installation files") } } - logger.InfoContext(ctx, "Existing installation verified", "file_count", len(receipt.Files)) - return nil - } else if !errors.Is(err, fs.ErrNotExist) { - return err - } - nextStep("verify_empty_data") - for _, name := range []string{"database", "state"} { - entries, err := os.ReadDir(filepath.Join(root, name)) + nextStep("verify_bundled_metadata") + files, err := fetch() if err != nil { return err } - if len(entries) > 0 { - return errors.New("existing data requires its original installation files") + logger.InfoContext(ctx, "Bundled node installation metadata verified", "file_count", len(files)) + nextStep("publish_node_metadata") + prefix := "node-payload/releases/" + release.revision + "/" + names := []string{} + for _, name := range releaseMembers { + if err := writeOwned(filepath.Join(root, prefix+name), files[name]); err != nil { + return err + } + logger.InfoContext(ctx, "Node metadata file copied", "file", name) + names = append(names, prefix+name) } - } - nextStep("verify_bundled_metadata") - files, err := fetch() - if err != nil { - return err - } - logger.InfoContext(ctx, "Bundled node installation metadata verified", "file_count", len(files)) - nextStep("publish_node_metadata") - prefix := "node-payload/releases/" + release.revision + "/" - names := []string{} - for _, name := range releaseMembers { - if err := writeOwned(filepath.Join(root, prefix+name), files[name]); err != nil { + active, _ := json.Marshal(map[string]string{"source_commit": release.revision}) + if err := writeOwned(filepath.Join(root, "node-payload", "active.json"), active); err != nil { return err } - logger.InfoContext(ctx, "Node metadata file copied", "file", name) - names = append(names, prefix+name) - } - active, _ := json.Marshal(map[string]string{"source_commit": release.revision}) - if err := writeOwned(filepath.Join(root, "node-payload", "active.json"), active); err != nil { - return err - } - if err := filepath.WalkDir(filepath.Join(root, "node-payload"), func(path string, entry fs.DirEntry, err error) error { - if err != nil || !entry.IsDir() { + if err := filepath.WalkDir(filepath.Join(root, "node-payload"), func(path string, entry fs.DirEntry, err error) error { + if err != nil || !entry.IsDir() { + return err + } + if err := os.Chmod(path, 0o755); err != nil { + return err + } + return chown(path, 65532, 65532) + }); err != nil { return err } - if err := os.Chmod(path, 0o755); err != nil { + nextStep("prepare_credentials") + generators := []struct { + name string + generate func() string + }{ + {"secrets/web/core.key", func() string { + key, err := generateCoreKey() + if err != nil { + panic(err) + } + return key + }}, + {"secrets/database/password", func() string { return randomHex(32) }}, + {"secrets/core/credential.key", func() string { return base64.StdEncoding.EncodeToString(randomBytes(32)) }}, + {"secrets/core/installation.id", func() string { return uuid.NewString() }}, + } + for _, secret := range generators { + path := filepath.Join(root, secret.name) + if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { + if err := writeOwned(path, []byte(secret.generate()+"\n")); err != nil { + return err + } + logger.InfoContext(ctx, "Credential file generated", "file", secret.name) + } else if err != nil { + return err + } else { + logger.InfoContext(ctx, "Credential file retained", "file", secret.name) + } + if secret.name != "secrets/web/core.key" { + names = append(names, secret.name) + } + } + if err := syncCoreKeyDigest(root); err != nil { return err } - return chown(path, 65532, 65532) - }); err != nil { - return err - } - nextStep("prepare_credentials") - generators := []struct { - name string - generate func() string - }{ - {"secrets/web/core.key", func() string { - key, err := generateCoreKey() + + names = append(names, "node-payload/active.json") + nextStep("write_installation_receipt") + receipt := installReceipt{SourceCommit: release.revision, Files: map[string]string{}} + for _, name := range names { + checksum, err := fileDigest(filepath.Join(root, name)) if err != nil { - panic(err) - } - return key - }}, - {"secrets/database/password", func() string { return randomHex(32) }}, - {"secrets/core/credential.key", func() string { return base64.StdEncoding.EncodeToString(randomBytes(32)) }}, - {"secrets/core/installation.id", func() string { return uuid.NewString() }}, - } - for _, secret := range generators { - path := filepath.Join(root, secret.name) - if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { - if err := writeOwned(path, []byte(secret.generate()+"\n")); err != nil { return err } - logger.InfoContext(ctx, "Credential file generated", "file", secret.name) - } else if err != nil { - return err - } else { - logger.InfoContext(ctx, "Credential file retained", "file", secret.name) + receipt.Files[name] = checksum } - names = append(names, secret.name) - } - key, err := coreKey(root) - if err != nil { - return err - } - digests, _ := json.Marshal([]string{keyDigest(key)}) - if err := writeOwned(filepath.Join(root, "secrets", "core", "core-key-digests.json"), digests); err != nil { - return err - } - names = append(names, "secrets/core/core-key-digests.json", "node-payload/active.json") - nextStep("write_installation_receipt") - receipt := installReceipt{SourceCommit: release.revision, Files: map[string]string{}} - for _, name := range names { - if receipt.Files[name], err = fileDigest(filepath.Join(root, name)); err != nil { + raw, _ := json.Marshal(receipt) + if err := writeOwned(marker, raw); err != nil { return err } - } - raw, _ := json.Marshal(receipt) - if err := writeOwned(marker, raw); err != nil { - return err - } - logger.InfoContext(ctx, "Installation initialized", "sign_in_key_command", "docker compose exec web oac-web core-key") - return nil + logger.InfoContext(ctx, "Installation initialized", "sign_in_key_command", "docker compose exec web oac-web core-key") + return nil + }) } func randomBytes(n int) []byte { diff --git a/services/core/cmd/oac/init_test.go b/services/core/cmd/oac/init_test.go index b0164d682..c005a2fc3 100644 --- a/services/core/cmd/oac/init_test.go +++ b/services/core/cmd/oac/init_test.go @@ -12,6 +12,7 @@ import ( "maps" "os" "path/filepath" + "runtime" "strings" "testing" @@ -42,7 +43,7 @@ func snapshot(t *testing.T, root string) map[string]string { } data, err := os.ReadFile(path) relative, _ := filepath.Rel(root, path) - saved[relative] = string(data) + saved[filepath.ToSlash(relative)] = string(data) return err }); err != nil { t.Fatal(err) @@ -81,7 +82,7 @@ func TestInitializeKeepsIdentityAndKeysAcrossRestarts(t *testing.T) { } for _, name := range []string{"secrets/web/core.key", "secrets/database/password", "secrets/core/credential.key"} { info, err := os.Stat(filepath.Join(root, name)) - if err != nil || info.Mode().Perm() != 0o600 { + if err != nil || (runtime.GOOS != "windows" && info.Mode().Perm() != 0o600) { t.Fatalf("%s: %v %v", name, info.Mode(), err) } } @@ -301,3 +302,29 @@ func TestInitializationLogsFailureStep(t *testing.T) { t.Fatal("failure logged success or generated credentials") } } + +func TestInitializationRetainsRotatedKeyAndRepairsItsDerivedDigest(t *testing.T) { + root, release, files := initFixture(t) + if err := initialize(root, release, fixed(files)); err != nil { + t.Fatal(err) + } + key, err := generateCoreKey() + if err != nil { + t.Fatal(err) + } + // Simulate interruption after publishing the new key but before its digest. + if err := writeOwned(filepath.Join(root, "secrets", "web", "core.key"), []byte(key+"\n")); err != nil { + t.Fatal(err) + } + if err := initialize(root, release, refuseDownload(t)); err != nil { + t.Fatal(err) + } + after, _ := coreKey(root) + if after != key { + t.Fatal("rotated key replaced") + } + digest, _ := os.ReadFile(filepath.Join(root, "secrets", "core", "core-key-digests.json")) + if !strings.Contains(string(digest), keyDigest(key)) { + t.Fatal("derived digest not repaired") + } +} diff --git a/services/core/cmd/oac/install.go b/services/core/cmd/oac/install.go new file mode 100644 index 000000000..efe7d5955 --- /dev/null +++ b/services/core/cmd/oac/install.go @@ -0,0 +1,358 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +// Installation is host-independent. Only the launchers select a native binary; +// Docker owns the Linux filesystem, service identities and persistent data. +type installOptions struct { + dir, version, publicURL, host string + port int +} +type installer struct { + docker func(context.Context, string, ...string) ([]byte, error) + download func(context.Context, string, string) error + executable string +} + +func installCommand(ctx context.Context, args []string) error { + home, err := os.UserHomeDir() + if err != nil { + return err + } + flags := flag.NewFlagSet("oac install", flag.ContinueOnError) + var o installOptions + flags.StringVar(&o.dir, "install-dir", filepath.Join(home, ".oac", "core"), "absolute installation directory") + flags.StringVar(&o.version, "version", "latest", "release tag") + flags.StringVar(&o.publicURL, "public-url", "", "URL reachable by browsers and nodes") + flags.StringVar(&o.host, "host", "0.0.0.0", "Web bind address") + flags.IntVar(&o.port, "web-port", 8080, "Web port") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 { + return errors.New("unexpected installation arguments") + } + exe, err := os.Executable() + if err != nil { + return err + } + i := installer{docker: dockerOutput, download: downloadAsset, executable: exe} + return i.install(ctx, o) +} + +func dockerOutput(ctx context.Context, dir string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "docker", args...) + cmd.Dir = dir + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("docker %s: %w\n%s", args[0], err, output) + } + return output, nil +} + +func downloadAsset(ctx context.Context, address, path string) error { + client := &http.Client{Timeout: 2 * time.Minute, CheckRedirect: func(req *http.Request, via []*http.Request) error { + if req.URL.Scheme != "https" || len(via) >= 10 { + return errors.New("invalid release redirect") + } + return nil + }} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) + if err != nil { + return err + } + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: HTTP %d", address, res.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(res.Body, (1<<20)+1)) + if err != nil { + return err + } + if len(raw) > 1<<20 { + return errors.New("release configuration exceeds size limit") + } + return os.WriteFile(path, raw, 0o600) +} + +func (i installer) install(ctx context.Context, o installOptions) error { + if !filepath.IsAbs(o.dir) || filepath.Clean(o.dir) == filepath.VolumeName(o.dir)+string(filepath.Separator) { + return errors.New("--install-dir must name an absolute directory other than the filesystem root") + } + if o.port < 1 || o.port > 65535 { + return errors.New("--web-port must be between 1 and 65535") + } + if net.ParseIP(o.host) == nil { + return errors.New("--host must be an IP address") + } + if o.publicURL == "" { + o.publicURL = "http://localhost:" + strconv.Itoa(o.port) + if o.host == "0.0.0.0" { + // UDP connect selects a route without transmitting a packet. + if route, err := net.DialTimeout("udp4", "1.1.1.1:53", time.Second); err == nil { + address := route.LocalAddr().(*net.UDPAddr).IP + route.Close() + if address.IsPrivate() { + o.publicURL = "http://" + net.JoinHostPort(address.String(), strconv.Itoa(o.port)) + } + } + } + } + u, err := url.Parse(o.publicURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return errors.New("--public-url must be an HTTP(S) origin") + } + for _, value := range []string{o.publicURL, o.version} { + if strings.ContainsAny(value, "\r\n'\\") { + return errors.New("installation options contain invalid characters") + } + } + o.dir = filepath.Clean(o.dir) + if err := os.MkdirAll(filepath.Dir(o.dir), 0o700); err != nil { + return err + } + parent, err := filepath.EvalSymlinks(filepath.Dir(o.dir)) + if err != nil { + return err + } + o.dir = filepath.Join(parent, filepath.Base(o.dir)) + if info, err := os.Lstat(o.dir); err == nil && !info.IsDir() { + return errors.New("installation path must be a directory, not a symbolic link or file") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return withLock(o.dir, func() error { return i.installLocked(ctx, o) }) +} + +func (i installer) installLocked(ctx context.Context, o installOptions) error { + info, err := i.docker(ctx, "", "info", "--format", "{{.OSType}}/{{.Architecture}}") + if err != nil { + return fmt.Errorf("start Docker and check this account's access: %w", err) + } + switch strings.TrimSpace(string(info)) { + case "linux/x86_64", "linux/amd64", "linux/aarch64", "linux/arm64": + default: + return errors.New("Docker must run Linux amd64 or arm64 containers; on Windows select Linux containers in Docker Desktop") + } + version, err := i.docker(ctx, "", "compose", "version", "--short") + if err != nil { + return err + } + var major, minor int + if _, err := fmt.Sscanf(strings.TrimPrefix(strings.TrimSpace(string(version)), "v"), "%d.%d", &major, &minor); err != nil || major < 2 || (major == 2 && minor < 26) { + return errors.New("Docker Compose 2.26 or newer is required") + } + engine, err := i.docker(ctx, "", "version", "--format", "{{.Server.APIVersion}}") + if err != nil { + return err + } + if _, err := fmt.Sscanf(strings.TrimSpace(string(engine)), "%d.%d", &major, &minor); err != nil || major < 1 || (major == 1 && minor < 45) { + return errors.New("Docker Engine 26 or newer is required for data volume subdirectories") + } + stage := o.dir + ".staging" + if err := cleanStage(stage, o.dir); err != nil { + return err + } + entries, err := os.ReadDir(o.dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + resume := len(entries) > 0 + working := o.dir + if !resume { + listener, err := net.Listen("tcp", net.JoinHostPort(o.host, strconv.Itoa(o.port))) + if err != nil { + return fmt.Errorf("Web port unavailable; choose --web-port: %w", err) + } + listener.Close() + if err := os.Mkdir(stage, 0o700); err != nil { + return err + } + defer os.RemoveAll(stage) + // Directory creation publishes ownership atomically, including on Windows. + if err := os.Mkdir(filepath.Join(stage, stageMarker(o.dir)), 0o700); err != nil { + return err + } + working = stage + repository := os.Getenv("OAC_REPOSITORY") + if repository == "" { + repository = "MiniMax-AI/OpenAgentCore" + } + base := "https://github.com/" + repository + "/releases/latest/download/" + if o.version != "latest" { + base = "https://github.com/" + repository + "/releases/download/" + url.PathEscape(o.version) + "/" + } + fmt.Println("Downloading release configuration...") + for _, name := range []string{"compose-sha256sums.txt", "compose.yaml"} { + if err := i.download(ctx, base+name, filepath.Join(stage, name)); err != nil { + return err + } + } + contents := fmt.Sprintf("COMPOSE_PROJECT_NAME=oac-%s\nOAC_HOST='%s'\nOAC_WEB_PORT='%d'\nOAC_PUBLIC_URL='%s'\n", randomHex(5), o.host, o.port, o.publicURL) + if err := os.WriteFile(filepath.Join(stage, ".env"), []byte(contents), 0o600); err != nil { + return err + } + source, err := os.Open(i.executable) + if err != nil { + return err + } + defer source.Close() + target, err := os.OpenFile(filepath.Join(stage, cliName()), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700) + if err != nil { + return err + } + _, copyErr := io.Copy(target, source) + closeErr := target.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } else { + fmt.Println("Using saved settings and retaining existing data.") + } + for _, name := range []string{"compose.yaml", "compose-sha256sums.txt", ".env", cliName()} { + info, err := os.Lstat(filepath.Join(working, name)) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("incomplete installation: preserve %s and choose another directory", o.dir) + } + } + if err := verifyCompose(working); err != nil { + return err + } + compose := func(args ...string) ([]byte, error) { + return i.docker(ctx, working, append([]string{"compose"}, args...)...) + } + if _, err := compose("config", "--quiet"); err != nil { + return err + } + images, err := compose("config", "--images") + if err != nil { + return err + } + fmt.Println("Checking and downloading images...") + for _, image := range strings.Fields(string(images)) { + if resume { + if _, err := i.docker(ctx, working, "image", "inspect", image); err == nil { + continue + } + } + if _, err := i.docker(ctx, working, "pull", image); err != nil { + return err + } + } + if !resume { + if _, err := os.Stat(o.dir); err == nil { + if err := os.Remove(o.dir); err != nil { + return err + } + } + if err := os.Rename(stage, o.dir); err != nil { + return err + } + working = o.dir + _ = os.Remove(filepath.Join(o.dir, stageMarker(o.dir))) + } + fmt.Println("Starting services...") + if _, err := compose("up", "-d", "--wait", "--wait-timeout", "180", "--pull", "never", "--no-recreate"); err != nil { + return fmt.Errorf("startup failed; data retained, rerun the installer: %w", err) + } + key, err := compose("exec", "-T", "web", "/usr/local/bin/oac-web", "core-key") + if err != nil { + return err + } + environment, err := compose("config", "--environment") + if err != nil { + return err + } + for _, line := range strings.Split(string(environment), "\n") { + if strings.HasPrefix(line, "OAC_PUBLIC_URL=") { + o.publicURL = strings.TrimPrefix(line, "OAC_PUBLIC_URL=") + } + } + fmt.Printf("\nOpenAgentCore is running.\n\nConsole: %s\nCore key: %s\nCommand: %s\n", o.publicURL, strings.TrimSpace(string(key)), filepath.Join(o.dir, cliName())) + if u, _ := url.Parse(o.publicURL); u != nil && (u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1") { + fmt.Println("For remote access, set OAC_PUBLIC_URL in .env to a reachable origin and run oac apply.") + } + return nil +} + +func cliName() string { + if runtime.GOOS == "windows" { + return "oac.exe" + } + return "oac" +} + +func cleanStage(stage, owner string) error { + info, err := os.Lstat(stage) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return errors.New("staging path must be a directory, not a link or file") + } + entries, err := os.ReadDir(stage) + if err != nil { + return err + } + if len(entries) > 0 { + marker := filepath.Join(stage, stageMarker(owner)) + info, err := os.Lstat(marker) + if err != nil || !info.IsDir() { + return errors.New("unrecognized staging directory; preserve it and choose another installation directory") + } + contents, err := os.ReadDir(marker) + if err != nil || len(contents) != 0 { + return errors.New("invalid staging ownership marker") + } + + } + return os.RemoveAll(stage) +} + +func verifyCompose(dir string) error { + raw, err := os.ReadFile(filepath.Join(dir, "compose.yaml")) + if err != nil { + return err + } + sums, err := os.ReadFile(filepath.Join(dir, "compose-sha256sums.txt")) + if err != nil { + return err + } + digest := sha256.Sum256(raw) + if strings.TrimSpace(string(sums)) != hex.EncodeToString(digest[:])+" compose.yaml" { + return errors.New("Compose checksum mismatch; restore the matching release configuration") + } + return nil +} + +func stageMarker(owner string) string { + return fmt.Sprintf(".oac-installer-%x", sha256.Sum256([]byte(owner))) +} diff --git a/services/core/cmd/oac/install_test.go b/services/core/cmd/oac/install_test.go new file mode 100644 index 000000000..be48b1bfc --- /dev/null +++ b/services/core/cmd/oac/install_test.go @@ -0,0 +1,245 @@ +package main + +import ( + "context" + "crypto/sha256" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +type installFixture struct { + installer + options installOptions + calls []string + failure string + cached bool +} + +func newInstallFixture(t *testing.T) *installFixture { + t.Helper() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + listener.Close() + exe := filepath.Join(root, "source") + if err := os.WriteFile(exe, []byte("binary"), 0o700); err != nil { + t.Fatal(err) + } + f := &installFixture{options: installOptions{dir: filepath.Join(root, "installation with spaces"), version: "latest", host: "127.0.0.1", port: port}} + f.executable = exe + compose := []byte("services: {}\n") + f.download = func(_ context.Context, address, path string) error { + if f.failure == "download" { + return os.ErrPermission + } + raw := compose + if strings.HasSuffix(address, "sums.txt") { + raw = []byte(fmt.Sprintf("%x compose.yaml\n", sha256.Sum256(compose))) + } + return os.WriteFile(path, raw, 0o600) + } + f.docker = func(_ context.Context, dir string, args ...string) ([]byte, error) { + command := strings.Join(args, " ") + f.calls = append(f.calls, command) + if f.failure != "" && strings.HasPrefix(command, f.failure) { + return nil, os.ErrPermission + } + switch command { + case "info --format {{.OSType}}/{{.Architecture}}": + return []byte("linux/aarch64"), nil + case "compose version --short": + return []byte("v2.26.0"), nil + case "version --format {{.Server.APIVersion}}": + return []byte("1.45"), nil + case "compose config --images": + return []byte("example/core:latest\nexample/web:latest"), nil + case "compose config --environment": + return []byte("OAC_PUBLIC_URL=http://localhost:8080"), nil + } + if strings.HasPrefix(command, "image inspect") && !f.cached { + return nil, os.ErrNotExist + } + if strings.HasPrefix(command, "compose up") { + if _, err := os.Stat(filepath.Join(f.options.dir, ".env")); err != nil { + t.Fatal("services started before configuration publication") + } + } + return nil, nil + } + return f +} +func (f *installFixture) run() error { return f.install(context.Background(), f.options) } + +func TestInstallFailureBeforePublicationAndRetry(t *testing.T) { + for _, failure := range []string{"download", "compose config --quiet", "pull"} { + t.Run(failure, func(t *testing.T) { + f := newInstallFixture(t) + f.failure = failure + if err := f.run(); err == nil { + t.Fatal("expected failure") + } + for _, path := range []string{f.options.dir, f.options.dir + ".staging"} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("unpublished state remains: %s", path) + } + } + f.failure = "" + if err := f.run(); err != nil { + t.Fatal(err) + } + }) + } +} +func TestInstallResumePreservesConfigAndUsesCachedImages(t *testing.T) { + f := newInstallFixture(t) + f.failure = "compose up" + if err := f.run(); err == nil { + t.Fatal("expected startup failure") + } + saved, err := os.ReadFile(filepath.Join(f.options.dir, ".env")) + if err != nil { + t.Fatal(err) + } + f.failure = "" + f.cached = true + f.calls = nil + f.download = func(context.Context, string, string) error { + t.Fatal("resume downloaded replacement release") + return nil + } + f.options.publicURL = "https://replacement.invalid" + if err := f.run(); err != nil { + t.Fatal(err) + } + after, _ := os.ReadFile(filepath.Join(f.options.dir, ".env")) + if string(saved) != string(after) { + t.Fatal("saved configuration replaced") + } + for _, call := range f.calls { + if strings.HasPrefix(call, "pull") || strings.Contains(call, "down") { + t.Fatal(call) + } + } + if !strings.Contains(strings.Join(f.calls, "\n"), "--pull never --no-recreate") { + t.Fatal(f.calls) + } +} +func TestInstallRecoversRecognizedStageAndPreservesUnknownData(t *testing.T) { + for _, owned := range []bool{true, false} { + t.Run(fmt.Sprint(owned), func(t *testing.T) { + f := newInstallFixture(t) + stage := f.options.dir + ".staging" + os.Mkdir(stage, 0o700) + os.WriteFile(filepath.Join(stage, "partial"), []byte("untouched"), 0o600) + if owned { + os.Mkdir(filepath.Join(stage, stageMarker(f.options.dir)), 0o700) + } + err := f.run() + if owned && err != nil { + t.Fatal(err) + } + if !owned { + if err == nil { + t.Fatal("unrecognized staging accepted") + } + if raw, _ := os.ReadFile(filepath.Join(stage, "partial")); string(raw) != "untouched" { + t.Fatal("unknown contents removed") + } + } + }) + } + f := newInstallFixture(t) + os.Mkdir(f.options.dir, 0o700) + os.WriteFile(filepath.Join(f.options.dir, "user-file"), []byte("keep"), 0o600) + if err := f.run(); err == nil { + t.Fatal("unrelated directory accepted") + } + if raw, _ := os.ReadFile(filepath.Join(f.options.dir, "user-file")); string(raw) != "keep" { + t.Fatal("user data lost") + } +} +func TestInstallRejectsBusyPortAndLock(t *testing.T) { + f := newInstallFixture(t) + listener, err := net.Listen("tcp", net.JoinHostPort(f.options.host, fmt.Sprint(f.options.port))) + if err != nil { + t.Fatal(err) + } + if err := f.run(); err == nil { + t.Fatal("busy port accepted") + } + listener.Close() + if err := withLock(f.options.dir, func() error { + if err := f.run(); err == nil { + t.Fatal("concurrent installation accepted") + } + return nil + }); err != nil { + t.Fatal(err) + } +} +func TestInstallRefusesModifiedCompose(t *testing.T) { + f := newInstallFixture(t) + if err := f.run(); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(f.options.dir, "compose.yaml"), []byte("changed"), 0o600) + f.calls = nil + if err := f.run(); err == nil { + t.Fatal("changed compose accepted") + } + for _, call := range f.calls { + if strings.HasPrefix(call, "compose up") { + t.Fatal("started modified compose") + } + } +} +func TestInstallerInterruptedProcess(t *testing.T) { + if root := os.Getenv("OAC_TEST_INSTALL_KILL"); root != "" { + err := withLock(root, func() error { + stage := root + ".staging" + os.Mkdir(stage, 0o700) + os.Mkdir(filepath.Join(stage, stageMarker(root)), 0o700) + os.WriteFile(filepath.Join(stage, "ready"), nil, 0o600) + time.Sleep(time.Minute) + return nil + }) + if err != nil { + os.Exit(2) + } + return + } + f := newInstallFixture(t) + cmd := exec.Command(os.Args[0], "-test.run=^TestInstallerInterruptedProcess$") + cmd.Env = append(os.Environ(), "OAC_TEST_INSTALL_KILL="+f.options.dir) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + deadline := time.Now().Add(15 * time.Second) + for { + if _, err := os.Stat(filepath.Join(f.options.dir+".staging", "ready")); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("child never acquired lock") + } + time.Sleep(20 * time.Millisecond) + } + cmd.Process.Kill() + cmd.Wait() + if err := f.run(); err != nil { + t.Fatalf("killed installer left unrecoverable state: %v", err) + } +} diff --git a/services/core/cmd/oac/install_unix_test.go b/services/core/cmd/oac/install_unix_test.go new file mode 100644 index 000000000..73e875d08 --- /dev/null +++ b/services/core/cmd/oac/install_unix_test.go @@ -0,0 +1,46 @@ +//go:build unix + +package main + +import ( + "os" + "os/exec" + "os/signal" + "syscall" + "testing" +) + +func TestInstallWriteFailureLeavesRetryableState(t *testing.T) { + if os.Getenv("OAC_TEST_FILE_LIMIT") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestInstallWriteFailureLeavesRetryableState$") + cmd.Env = append(os.Environ(), "OAC_TEST_FILE_LIMIT=1") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("write failure probe: %v\n%s", err, output) + } + return + } + f := newInstallFixture(t) + var original syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_FSIZE, &original); err != nil { + t.Fatal(err) + } + limited := original + limited.Cur = 0 + signal.Ignore(syscall.SIGXFSZ) + if err := syscall.Setrlimit(syscall.RLIMIT_FSIZE, &limited); err != nil { + t.Fatal(err) + } + err := f.run() + if restore := syscall.Setrlimit(syscall.RLIMIT_FSIZE, &original); restore != nil { + t.Fatal(restore) + } + if err == nil { + t.Fatal("installation succeeded without writable files") + } + if _, err := os.Stat(f.options.dir + ".staging"); !os.IsNotExist(err) { + t.Fatal("failed installation retained staging") + } + if err := f.run(); err != nil { + t.Fatalf("installation did not recover after restoring writes: %v", err) + } +} diff --git a/services/core/cmd/oac/main.go b/services/core/cmd/oac/main.go index 0d1318c78..f2b3393d3 100644 --- a/services/core/cmd/oac/main.go +++ b/services/core/cmd/oac/main.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "flag" "fmt" "os" "os/signal" @@ -15,6 +16,7 @@ import ( "syscall" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimefs" ) func main() { @@ -28,6 +30,9 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() if err := run(ctx, os.Args[1], os.Args[2:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return + } if os.Args[1] != "init" { fmt.Fprintln(os.Stderr, err.Error()) } @@ -39,27 +44,35 @@ func main() { var buildRevision = "development" func usage() { - fmt.Fprintf(os.Stderr, "oac (%s)\nUsage: oac apply|core-key|rotate-core-key|init\n", buildRevision) + fmt.Fprintf(os.Stderr, "oac (%s)\nUsage: oac install|apply|core-key|rotate-core-key\n", buildRevision) } func run(ctx context.Context, command string, args []string) error { switch command { + case "install": + return installCommand(ctx, args) case "init": return initCommand() + case "rotate-volume-key": + return withLock("/data/secrets/init", func() error { return rotateVolumeKey("/data") }) } root, err := installDir() if err != nil { return err } - in := installation{root: root, data: dataDir(root)} runner := execRunner{dir: root} switch command { case "apply": return withLock(root, func() error { return apply(ctx, runner) }) case "core-key": - return coreKeyCommand(ctx, in, runner, args) + return coreKeyCommand(ctx, runner, args) case "rotate-core-key": - return withLock(root, func() error { return rotateCoreKey(ctx, in, runner) }) + return withLock(root, func() error { + if err := runner.Run(ctx, "run", "--rm", "--no-deps", "init", "/usr/local/bin/oac", "rotate-volume-key"); err != nil { + return err + } + return runner.Run(ctx, "restart", "core", "web") + }) default: usage() return errors.New("unknown command") @@ -81,25 +94,24 @@ func installDir() (string, error) { return "", errors.New("run oac from an installation directory that contains compose.yaml") } -type installation struct{ root, data string } - -func dataDir(root string) string { - if dir := os.Getenv("OAC_DATA_MOUNT"); dir != "" { - return dir - } - return filepath.Join(root, "data") -} - func withLock(root string, fn func() error) error { - file, err := os.OpenFile(filepath.Join(root, ".oac.lock"), os.O_CREATE|os.O_RDWR, 0o600) - if err != nil { + dir := root + ".lock" + if info, err := os.Lstat(dir); err == nil && !info.IsDir() { + return errors.New("invalid installation lock directory") + } + if err := os.MkdirAll(dir, 0o700); err != nil { return err } - defer file.Close() - if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX); err != nil { + handle, err := os.OpenRoot(dir) + if err != nil { return err } - defer syscall.Flock(int(file.Fd()), syscall.LOCK_UN) + defer handle.Close() + unlock, err := runtimefs.LockDirectory(handle) + if err != nil { + return fmt.Errorf("another operation is using this installation: %w", err) + } + defer unlock() return fn() } @@ -110,8 +122,8 @@ func apply(ctx context.Context, runner Runner) error { return runner.Run(ctx, "up", "-d", "--wait") } -func coreKeyCommand(ctx context.Context, in installation, runner Runner, args []string) error { - path := filepath.Join(in.data, "secrets", "web", "core.key") +func coreKeyCommand(ctx context.Context, runner Runner, args []string) error { + path := "Docker volume: secrets/web/core.key (use oac core-key --show)" if len(args) == 0 { fmt.Println(path) return nil @@ -130,24 +142,33 @@ func generateCoreKey() (string, error) { return "oac_admin_" + hex.EncodeToString(buf), nil } -func rotateCoreKey(ctx context.Context, in installation, runner Runner) error { +func rotateVolumeKey(data string) error { key, err := generateCoreKey() if err != nil { return err } - keyPath := filepath.Join(in.data, "secrets", "web", "core.key") - digestPath := filepath.Join(in.data, "secrets", "core", "core-key-digests.json") - if err := writeSecret(keyPath, key+"\n"); err != nil { + keyPath := filepath.Join(data, "secrets", "web", "core.key") + if err := writeOwned(keyPath, []byte(key+"\n")); err != nil { return err } - raw, err := json.Marshal([]string{keyDigest(key)}) + return syncCoreKeyDigest(data) +} + +func syncCoreKeyDigest(data string) error { + key, err := coreKey(data) if err != nil { return err } - if err := writeSecret(digestPath, string(raw)+"\n"); err != nil { - return err + value, err := hex.DecodeString(strings.TrimPrefix(key, "oac_admin_")) + if err != nil || !strings.HasPrefix(key, "oac_admin_") || len(value) != 32 { + return errors.New("invalid saved Core key") } - return runner.Run(ctx, "restart", "core", "web") + raw, _ := json.Marshal([]string{keyDigest(key)}) + path := filepath.Join(data, "secrets", "core", "core-key-digests.json") + if saved, err := os.ReadFile(path); err == nil && strings.TrimSpace(string(saved)) == string(raw) { + return nil + } + return writeOwned(path, raw) } func coreKey(data string) (string, error) { @@ -162,23 +183,3 @@ func keyDigest(key string) string { sum := sha256.Sum256([]byte(key)) return hex.EncodeToString(sum[:]) } - -func writeSecret(path, contents string) error { - info, err := os.Stat(path) - if err != nil { - return err - } - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - return os.ErrInvalid - } - temporary := path + ".tmp" - if err := os.WriteFile(temporary, []byte(contents), 0o600); err != nil { - return err - } - if err := os.Chown(temporary, int(stat.Uid), int(stat.Gid)); err != nil { - _ = os.Remove(temporary) - return err - } - return os.Rename(temporary, path) -} diff --git a/services/core/cmd/oac/oac_test.go b/services/core/cmd/oac/oac_test.go index ee865c213..d3ccd058d 100644 --- a/services/core/cmd/oac/oac_test.go +++ b/services/core/cmd/oac/oac_test.go @@ -28,40 +28,36 @@ func TestApplyDoesNotStartWhenTheConfigurationCheckFails(t *testing.T) { func TestRotateCoreKeyDigestDoesNotEchoTheKey(t *testing.T) { root := t.TempDir() - in := installation{root: root, data: filepath.Join(root, "data")} - if err := os.MkdirAll(filepath.Join(in.data, "secrets", "web"), 0o755); err != nil { + data := filepath.Join(root, "data") + if err := os.MkdirAll(filepath.Join(data, "secrets", "web"), 0o755); err != nil { t.Fatal(err) } - if err := os.MkdirAll(filepath.Join(in.data, "secrets", "core"), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(data, "secrets", "core"), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(in.data, "secrets", "web", "core.key"), []byte("old\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(data, "secrets", "web", "core.key"), []byte("old\n"), 0o600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(in.data, "secrets", "core", "core-key-digests.json"), []byte("[]\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(data, "secrets", "core", "core-key-digests.json"), []byte("[]\n"), 0o600); err != nil { t.Fatal(err) } - var restarted bool - runner := scriptedRunner{run: func(args ...string) error { - restarted = args[0] == "restart" - return nil - }} - if err := rotateCoreKey(context.Background(), in, runner); err != nil { + previous := chown + chown = func(string, int, int) error { return nil } + t.Cleanup(func() { chown = previous }) + if err := rotateVolumeKey(data); err != nil { t.Fatal(err) } - raw, err := os.ReadFile(filepath.Join(in.data, "secrets", "web", "core.key")) + + raw, err := os.ReadFile(filepath.Join(data, "secrets", "web", "core.key")) if err != nil { t.Fatal(err) } key := strings.TrimSpace(string(raw)) assertCoreKeyFormat(t, key) - digest, err := os.ReadFile(filepath.Join(in.data, "secrets", "core", "core-key-digests.json")) + digest, err := os.ReadFile(filepath.Join(data, "secrets", "core", "core-key-digests.json")) if err != nil || !strings.Contains(string(digest), keyDigest(key)) || strings.Contains(string(digest), key) { t.Fatalf("digest %s key leaked %v", digest, err) } - if !restarted { - t.Fatal("core was not restarted") - } } type scriptedRunner struct { diff --git a/services/core/tools/e2b-provider/Build.Dockerfile b/services/core/tools/e2b-provider/Build.Dockerfile index 446fa4cd2..84083cefd 100644 --- a/services/core/tools/e2b-provider/Build.Dockerfile +++ b/services/core/tools/e2b-provider/Build.Dockerfile @@ -1,5 +1,5 @@ # Fixed CPython and glibc baseline shared by native Core and the Debian Core image. -FROM python:3.12.12-slim-bookworm@sha256:2986c55feb36e6cae00fa1fefb454283e4b33f35e75ff8bdd123b134130be301 +FROM python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c RUN apt-get update && apt-get install -y --no-install-recommends binutils \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["python3", "/source/services/core/tools/e2b-provider/build.py"] diff --git a/services/core/tools/e2b-provider/build.py b/services/core/tools/e2b-provider/build.py index 8d6f9badf..8b69302d5 100644 --- a/services/core/tools/e2b-provider/build.py +++ b/services/core/tools/e2b-provider/build.py @@ -16,7 +16,7 @@ SOURCE = Path('/source/services/core/tools/e2b-provider') OUTPUT = Path('/output') NAME = 'oac-e2b-provider' -BASE = 'python:3.12.12-slim-bookworm@sha256:2986c55feb36e6cae00fa1fefb454283e4b33f35e75ff8bdd123b134130be301' +BASE = next(line.split()[1] for line in (SOURCE / 'Build.Dockerfile').read_text().splitlines() if line.startswith('FROM ')) def checked(args, **options): @@ -24,8 +24,9 @@ def checked(args, **options): def main(): - if sys.version_info[:3] != (3, 12, 12) or platform.system() != 'Linux' or platform.machine() != 'x86_64': - raise RuntimeError('Use the pinned Linux amd64 build image') + if sys.version_info[:3] != (3, 12, 12) or platform.system() != 'Linux' or platform.machine() not in ('x86_64', 'aarch64'): + raise RuntimeError('Use the pinned Linux amd64 or arm64 build image') + architecture = {'x86_64': 'amd64', 'aarch64': 'arm64'}[platform.machine()] os.umask(0o022) with tempfile.TemporaryDirectory(prefix='e2b-build-') as temporary: root = Path(temporary) @@ -56,7 +57,7 @@ def main(): if report != {'Version': PROTOCOL_VERSION, 'SDKVersion': SDK_VERSION}: raise RuntimeError('Unexpected helper readiness report') manifest = {'format_version': 1, 'sdk_version': report['SDKVersion'], 'python_version': platform.python_version(), - 'platform': 'linux-amd64', 'libc': platform.libc_ver(), 'build_image': BASE, + 'platform': 'linux-' + architecture, 'libc': platform.libc_ver(), 'build_image': BASE, 'entrypoint': NAME, 'source_sha256': {file.name: hashlib.sha256(file.read_bytes()).hexdigest() for file in sorted(source.glob('*.py'))}, @@ -66,7 +67,7 @@ def main(): if entry.is_symlink() or not (entry.is_file() or entry.is_dir()): raise RuntimeError('Unsupported artifact entry') entry.chmod(0o755 if entry.is_dir() or entry.stat().st_mode & 0o111 else 0o644) - destination = OUTPUT / (NAME + '-linux-amd64.tar.gz') + destination = OUTPUT / (NAME + '-linux-' + architecture + '.tar.gz') with tarfile.open(destination, 'w:gz', dereference=True) as archive: archive.add(exported, arcname=NAME) checksum = hashlib.sha256(destination.read_bytes()).hexdigest() diff --git a/services/web/Dockerfile b/services/web/Dockerfile index 642383d04..cea7c0df0 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -1,5 +1,5 @@ # The build context contains only oac-web and the existing Web dist. -FROM gcr.io/distroless/static-debian13:nonroot@sha256:e754765ad9e167b0677b41c617fd44afb7b9818a477f48f17bda08e12cfb98cb +FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3 COPY --chmod=0555 oac-web /usr/local/bin/oac-web COPY dist /www