Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/published-images.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
[
"agent-computer",
"agent-subscription-gateway",
"supervisor",
"agent-bot",
"agent-langgraph",
Expand Down
117 changes: 117 additions & 0 deletions .github/workflows/aws-deploy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
name: Deploy personal AWS stack

on:
workflow_dispatch:
inputs:
target:
description: Hosts to update
required: true
default: all
type: choice
options: [all, worker, control]

permissions:
contents: read
id-token: write

concurrency:
group: openbot-personal-aws
cancel-in-progress: false

jobs:
deploy:
runs-on: ubuntu-latest
timeout-minutes: 45
environment: production
env:
AWS_REGION: ${{ vars.AWS_REGION }}
CONTROL_REPOSITORY: ${{ vars.AWS_OPENBOT_ECR_REPOSITORY_URL }}
GATEWAY_REPOSITORY: ${{ vars.AWS_GATEWAY_ECR_REPOSITORY_URL }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check protected deployment settings
env:
ROLE_ARN: ${{ vars.AWS_ROLE_ARN }}
CONTROL_INSTANCE: ${{ vars.AWS_CONTROL_INSTANCE_ID }}
WORKER_INSTANCE: ${{ vars.AWS_WORKER_INSTANCE_ID }}
CONTROL_SECRET: ${{ vars.AWS_CONTROL_SECRET_ARN }}
WORKER_SECRET: ${{ vars.AWS_WORKER_SECRET_ARN }}
run: |
set -euo pipefail
for name in AWS_REGION ROLE_ARN CONTROL_REPOSITORY GATEWAY_REPOSITORY CONTROL_INSTANCE WORKER_INSTANCE CONTROL_SECRET WORKER_SECRET; do
test -n "${!name:-}" || { echo "::error::$name is not set in the production environment"; exit 1; }
done
test "${CONTROL_REPOSITORY%%/*}" = "${GATEWAY_REPOSITORY%%/*}"
- uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
role-to-assume: ${{ vars.AWS_ROLE_ARN }}
aws-region: ${{ vars.AWS_REGION }}
- uses: aws-actions/amazon-ecr-login@4625ce35226a7557230889aae2f52eb50ec3dcda # v2.0.1
- uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
- id: control-image
name: Build and push OpenBot
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
platforms: linux/amd64
push: true
tags: ${{ env.CONTROL_REPOSITORY }}:${{ github.sha }}
cache-from: type=gha,scope=aws-openbot-amd64
cache-to: type=gha,mode=max,scope=aws-openbot-amd64
provenance: true
sbom: true
- id: gateway-image
name: Build and push subscription gateway
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: agent-subscription-gateway/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ env.GATEWAY_REPOSITORY }}:${{ github.sha }}
cache-from: type=gha,scope=aws-gateway-amd64
cache-to: type=gha,mode=max,scope=aws-gateway-amd64
provenance: true
sbom: true
- uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1
- name: Sign and scan immutable images
env:
CONTROL_DIGEST: ${{ steps.control-image.outputs.digest }}
GATEWAY_DIGEST: ${{ steps.gateway-image.outputs.digest }}
run: |
set -euo pipefail
for ref in "$CONTROL_REPOSITORY@$CONTROL_DIGEST" "$GATEWAY_REPOSITORY@$GATEWAY_DIGEST"; do
cosign sign --yes "$ref"
docker run --rm \
-v "$HOME/.docker:/root/.docker:ro" \
aquasec/trivy@sha256:e2b22eac59c02003d8749f5b8d9bd073b62e30fefaef5b7c8371204e0a4b0c08 \
image --exit-code 1 --ignore-unfixed --severity CRITICAL "$ref"
done
- name: Pack the host files
run: tar -czf "$RUNNER_TEMP/openbot-aws-deploy.tgz" deploy/aws
- name: Deploy worker first
if: inputs.target == 'all' || inputs.target == 'worker'
env:
DEPLOY_BUNDLE: ${{ runner.temp }}/openbot-aws-deploy.tgz
WORKER_INSTANCE: ${{ vars.AWS_WORKER_INSTANCE_ID }}
WORKER_SECRET: ${{ vars.AWS_WORKER_SECRET_ARN }}
GATEWAY_DIGEST: ${{ steps.gateway-image.outputs.digest }}
run: |
deploy/aws/bin/deploy-via-ssm \
worker "$WORKER_INSTANCE" "$WORKER_SECRET" "$AWS_REGION" \
"${GATEWAY_REPOSITORY%%/*}" GATEWAY_IMAGE \
"$GATEWAY_REPOSITORY@$GATEWAY_DIGEST"
- name: Deploy control
if: inputs.target == 'all' || inputs.target == 'control'
env:
DEPLOY_BUNDLE: ${{ runner.temp }}/openbot-aws-deploy.tgz
CONTROL_INSTANCE: ${{ vars.AWS_CONTROL_INSTANCE_ID }}
CONTROL_SECRET: ${{ vars.AWS_CONTROL_SECRET_ARN }}
CONTROL_DIGEST: ${{ steps.control-image.outputs.digest }}
run: |
deploy/aws/bin/deploy-via-ssm \
control "$CONTROL_INSTANCE" "$CONTROL_SECRET" "$AWS_REGION" \
"${CONTROL_REPOSITORY%%/*}" OPENBOT_IMAGE \
"$CONTROL_REPOSITORY@$CONTROL_DIGEST"
16 changes: 10 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,12 @@ jobs:
exit 1
fi

# The two packages that are deployables in their own right rather than root workspaces. Root
# The three packages that are deployables in their own right rather than root workspaces. Root
# `typecheck` is `bun run --filter '*' typecheck`, and `--filter '*'` enumerates `workspaces`, which
# is app, server and worker. So both of these ship a `typecheck` script that nothing has ever run:
# `agent-computer` holds the only `spawn` in the deployment and `supervisor` is the only thing
# holding a Docker socket, which is a poor pair to leave untyped. Both are clean today, so this
# holding a Docker socket, and the subscription gateway holds provider processes. They are clean
# today, so this
# changes nothing about main and only stops the next change being the first one a type checker sees.
#
# Not by adding them to `workspaces`: each is built from its own lockfile and the Dockerfile depends
Expand All @@ -68,7 +69,7 @@ jobs:
# One red package must not hide whether the other is red too.
fail-fast: false
matrix:
package: [agent-computer, supervisor]
package: [agent-computer, supervisor, agent-subscription-gateway]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -86,6 +87,9 @@ jobs:
working-directory: ${{ matrix.package }}
- run: bun run typecheck
working-directory: ${{ matrix.package }}
- run: bun run test
if: matrix.package == 'agent-subscription-gateway'
working-directory: ${{ matrix.package }}

chart:
name: chart (${{ matrix.target }})
Expand Down Expand Up @@ -240,7 +244,7 @@ jobs:
working-directory: desktop
# Not the db:migrate script: that one loads ../.env, which does not exist in CI. DATABASE_URL
# comes from the job env instead, which drizzle.config.ts already reads.
- run: bunx drizzle-kit migrate --config=drizzle.config.ts
- run: bun x drizzle-kit migrate --config=drizzle.config.ts
working-directory: server
# A passing job must include the expected test floor. Import-time failures can otherwise skip
# files before their tests are registered.
Expand Down Expand Up @@ -317,15 +321,15 @@ jobs:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
# Collisions and gaps between the migration files themselves.
- run: bunx drizzle-kit check --config=drizzle.config.ts
- run: bun x drizzle-kit check --config=drizzle.config.ts
working-directory: server
# And the other direction: a schema change with no migration written for it. `generate` writes a
# file when it finds one, so the tree being dirty afterwards is the failure.
- name: Schema has no unwritten migration
working-directory: server
run: |
set -euo pipefail
bunx drizzle-kit generate --config=drizzle.config.ts --name=ci_drift_probe
bun x drizzle-kit generate --config=drizzle.config.ts --name=ci_drift_probe
if [ -n "$(git status --porcelain drizzle)" ]; then
echo "::error::The schema has changed without a migration. Run drizzle-kit generate and commit it."
git status --porcelain drizzle
Expand Down
28 changes: 16 additions & 12 deletions .github/workflows/publish-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
components: ${{ steps.components.outputs.components }}
image_prefix: ${{ steps.registry.outputs.image_prefix }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -101,13 +102,16 @@ jobs:
# set that is tested and the set that is published cannot drift apart.
- id: components
run: echo "components=$(jq -c . .github/published-images.json)" >> "$GITHUB_OUTPUT"
- id: registry
name: Select this fork's package namespace
run: echo "image_prefix=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/openbot" >> "$GITHUB_OUTPUT"

# The same checks CI runs, against the commit being published. This is the gate: nothing is built
# or tagged unless they pass here, on this exact tree.
checks:
needs: [metadata, verify]
if: needs.metadata.outputs.is_release == 'true'
uses: $/.github/workflows/ci.yml
uses: ./.github/workflows/ci.yml
permissions:
contents: read

Expand Down Expand Up @@ -146,20 +150,20 @@ jobs:
# The version, the commit, and a moving latest. The version tag is the one to deploy; the
# commit tag is how you find out what a running image actually contains.
tags: |
ghcr.io/copilotkit/openbot:${{ needs.metadata.outputs.version }}
ghcr.io/copilotkit/openbot:${{ github.sha }}
ghcr.io/copilotkit/openbot:latest
${{ needs.verify.outputs.image_prefix }}:${{ needs.metadata.outputs.version }}
${{ needs.verify.outputs.image_prefix }}:${{ github.sha }}
${{ needs.verify.outputs.image_prefix }}:latest
cache-from: type=gha
cache-to: type=gha,mode=max
provenance: true
sbom: true
# BuildKit's own attestations above travel inside the image. This one is the record GitHub
# holds, and it is what `gh attestation verify oci://ghcr.io/copilotkit/openbot:vX.Y.Z
# -R CopilotKit/OpenBot` checks before anybody deploys it. Bound to the digest, never a tag,
# holds, and it is what `gh attestation verify oci://<owner>/openbot:vX.Y.Z` checks before
# anybody deploys it. Bound to the digest, never a tag,
# because a tag can be moved to point at something else afterwards.
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
with:
subject-name: ghcr.io/copilotkit/openbot
subject-name: ${{ needs.verify.outputs.image_prefix }}
subject-digest: ${{ steps.push.outputs.digest }}
push-to-registry: true

Expand Down Expand Up @@ -226,7 +230,7 @@ jobs:
# `oci-mediatypes=true` because zstd layers have no Docker-schema2 media type to be
# described by. It is buildx's default for this exporter and is stated rather than
# assumed, since the push fails without it and the reason would not be obvious.
outputs: type=image,name=ghcr.io/copilotkit/openbot-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true,compression=zstd,force-compression=true
outputs: type=image,name=${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }},push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true,compression=zstd,force-compression=true
# Scoped per image and per architecture. One shared scope would have ten builds
# overwriting each other's cache and none of them reading their own.
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.platform.arch }}
Expand Down Expand Up @@ -292,7 +296,7 @@ jobs:
- id: merge
name: Write the tags over both architectures
env:
REPOSITORY: ghcr.io/copilotkit/openbot-${{ matrix.image }}
REPOSITORY: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }}
VERSION: ${{ needs.metadata.outputs.version }}
COMMIT: ${{ github.sha }}
run: |
Expand All @@ -319,15 +323,15 @@ jobs:
echo "digest=$digest" >> "$GITHUB_OUTPUT"
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
with:
subject-name: ghcr.io/copilotkit/openbot-${{ matrix.image }}
subject-name: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }}
subject-digest: ${{ steps.merge.outputs.digest }}
push-to-registry: true
# For the same reason the digests above travel as files: a matrix cannot hand a value to a
# later job.
- name: Record the manifest
env:
NAME: ${{ matrix.image }}
REPOSITORY: ghcr.io/copilotkit/openbot-${{ matrix.image }}
REPOSITORY: ${{ needs.verify.outputs.image_prefix }}-${{ matrix.image }}
DIGEST: ${{ steps.merge.outputs.digest }}
run: |
set -euo pipefail
Expand Down Expand Up @@ -396,7 +400,7 @@ jobs:
--arg version "$VERSION" \
--arg digest "$DIGEST" \
--arg commit "$COMMIT" \
--arg repository "ghcr.io/copilotkit/openbot" \
--arg repository "${{ needs.verify.outputs.image_prefix }}" \
--argjson components "$components" \
'{
version: $version,
Expand Down
60 changes: 60 additions & 0 deletions .github/workflows/subscription-gateway.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
name: Subscription gateway

on:
pull_request:
paths:
- agent-subscription-gateway/**
- deploy/aws/**
- shared/**
- .github/workflows/subscription-gateway.yml
push:
branches: [main]
paths:
- agent-subscription-gateway/**
- deploy/aws/**
- shared/**
- .github/workflows/subscription-gateway.yml
workflow_call:

permissions:
contents: read

concurrency:
group: subscription-gateway-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
- run: bun install --frozen-lockfile
working-directory: agent-subscription-gateway
- run: bun run typecheck
working-directory: agent-subscription-gateway
- run: bun run test
working-directory: agent-subscription-gateway
- uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
- name: Build the AWS architecture
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: .
file: agent-subscription-gateway/Dockerfile
platforms: linux/amd64
load: true
tags: openbot-subscription-gateway:ci
cache-from: type=gha,scope=subscription-gateway-amd64
cache-to: type=gha,mode=max,scope=subscription-gateway-amd64
- name: Scan the image
run: >-
docker run --rm
-v /var/run/docker.sock:/var/run/docker.sock
aquasec/trivy@sha256:e2b22eac59c02003d8749f5b8d9bd073b62e30fefaef5b7c8371204e0a4b0c08
image --exit-code 1 --ignore-unfixed --severity CRITICAL
openbot-subscription-gateway:ci
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ egress.env
# named node_modules is not a directory — so a sandbox that links its dependencies at the repo root
# leaves three links git will happily commit. That happened.
node_modules
.terraform/
**/dist/
app/src/lib/generated/application-config.ts
.logs/
Expand Down
40 changes: 40 additions & 0 deletions agent-subscription-gateway/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
FROM node:24.18.1-bookworm-slim@sha256:235600a8101ab264e117b1768e925532262668dc9b581ef1dd7d96ced463b8e7 AS node-toolchain
FROM oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS bun-toolchain

FROM node-toolchain AS runtime

ARG CODEX_VERSION=0.154.0
ARG CLAUDE_VERSION=2.1.271
ARG GROK_VERSION=1.0.30
ARG NPM_VERSION=12.0.2
ENV CODEX_VERSION=${CODEX_VERSION}
ENV CLAUDE_VERSION=${CLAUDE_VERSION}
ENV GROK_VERSION=${GROK_VERSION}
ENV HOME=/home/gateway

COPY --from=bun-toolchain /usr/local/bin/bun /usr/local/bin/bun
RUN ln -s bun /usr/local/bin/bunx \
&& apt-get update \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends ca-certificates git bubblewrap socat \
&& npm install --global "npm@${NPM_VERSION}" \
&& npm install --global --allow-scripts=@anthropic-ai/claude-code,@xai-official/grok "@openai/codex@${CODEX_VERSION}" "@anthropic-ai/claude-code@${CLAUDE_VERSION}" "@xai-official/grok@${GROK_VERSION}" \
&& npm cache clean --force \
&& rm -rf /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --create-home --uid 10001 --shell /usr/sbin/nologin gateway \
&& mkdir -p /app /workspaces /var/lib/openbot-gateway /home/gateway/.codex /home/gateway/.grok /etc/codex /etc/claude-code /etc/grok \
&& chmod 0755 /etc/codex /etc/claude-code /etc/grok \
&& chown -R 10001:10001 /app /workspaces /var/lib/openbot-gateway /home/gateway

WORKDIR /app
COPY --chown=10001:10001 agent-subscription-gateway/package.json agent-subscription-gateway/package.json
COPY --chown=10001:10001 agent-subscription-gateway/src agent-subscription-gateway/src
COPY --chown=10001:10001 shared shared
COPY --chmod=0444 deploy/aws/codex-config.toml /etc/codex/config.toml
COPY --chmod=0444 deploy/aws/codex-requirements.toml /etc/codex/requirements.toml
COPY --chmod=0444 deploy/aws/claude-managed-settings.json /etc/claude-code/managed-settings.json
COPY --chmod=0444 deploy/aws/grok-requirements.toml /etc/grok/requirements.toml

USER 10001:10001
CMD ["bun", "agent-subscription-gateway/src/main.ts"]
Loading