From 80b4869f9fc43d2d89cb9356668adbce5ff7b288 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 28 Jul 2026 13:22:12 -0700 Subject: [PATCH] Add connected-local Cloud support and Core runtime improvements ## Summary Add connected-local support to self-hosted Core so a Docker-deployed instance can link to Atomic Memory Cloud using Cloud-issued JWTs, content-free trace sync, and heartbeat reporting. This also delivers Core 1.2.0 runtime, performance, and reliability improvements while preserving local-only deployments and the existing `CORE_API_KEY` contract. ## Changes - Add dual authentication supporting the existing `CORE_API_KEY` and verified Cloud-issued RS256 JWTs, with user and project binding that rejects cross-user or cross-project access. - Add durable Cloud trace sync through a PostgreSQL outbox, allowlisted content-free envelopes, retry and dead-letter handling, backlog limits, and authentication recovery. - Add heartbeat reporting and a stable connected-local instance identity. - Auto-generate and persist a local `CORE_API_KEY` when needed, preserve explicit key overrides across restarts, apply connected-local Docker defaults, and provide a helper for retrieving the persisted key. - Add `EMBEDDING_DTYPE` configuration for quantized local embedding models. - Add bounded parallel reconciliation and a background scheduler for deferred AUDN work. - Enforce Anthropic JSON output using assistant prefill. - Restrict idempotent re-indexing to valid extraction-status transitions. - Add a repository-guarded prerelease Core image lane with per-platform verification, runner disk management, curated aliases, and structural policy enforcement. The lane remains disabled in this public repository. - Bump `@atomicmemory/core` to 1.2.0 and add `jose` for JWT verification. ## Why Connected-local mode lets operators link self-hosted Core to a Cloud project without sharing the local API key. Local SDK and CLI access remains unchanged, and outbound trace sync is explicitly enabled and excludes memory and query content. The additional Core improvements make larger local embedding models practical, reduce deferred-work latency, strengthen structured LLM output, and correct restart and re-indexing behavior. ## Validation - Added JWT configuration, dual-authentication, user-binding, heartbeat, trace sanitization, trace-sync, outbox, and deferred-scheduler tests. - Added JWKS-backed integration coverage for cross-user rejection and reconcile scoping. - Extended Docker smoke coverage for generated and operator-supplied API-key persistence. - Verified the Core package build, type checking, linting, packaging, metadata, security, repository hygiene, and Docker image build and boot. - Image-publisher policy suite passes 43/43 tests. - All 11 release-preparation and post-merge CI checks pass. --- .github/workflows/ci.yml | 9 + .../workflows/internal-core-docker-image.yml | 275 ++++++++++++ package.json | 3 +- packages/core/.env.example | 56 ++- packages/core/CHANGELOG.md | 28 ++ packages/core/README.md | 40 ++ packages/core/SECURITY.md | 43 ++ packages/core/docker-compose.yml | 3 + packages/core/package.json | 3 +- packages/core/scripts/docker-entrypoint.sh | 125 +++++- packages/core/scripts/docker-smoke-test.sh | 71 ++- packages/core/scripts/show-local-core-key.sh | 7 + .../src/__tests__/cloud-jwt-config.test.ts | 109 +++++ .../src/__tests__/deployment-config.test.ts | 24 +- .../src/__tests__/helpers/jwt-auth-headers.ts | 10 + packages/core/src/__tests__/setup.ts | 5 + .../src/__tests__/trace-sync-config.test.ts | 60 +++ .../src/app/__tests__/versioned-mount.test.ts | 122 +++++- packages/core/src/app/create-app.ts | 32 +- packages/core/src/app/runtime-container.ts | 2 + .../cloud/__tests__/heartbeat-client.test.ts | 50 +++ packages/core/src/cloud/cloud-api-client.ts | 25 ++ packages/core/src/cloud/env.ts | 25 ++ packages/core/src/cloud/heartbeat-client.ts | 41 ++ packages/core/src/cloud/instance-id.ts | 31 ++ packages/core/src/cloud/jwt-config.ts | 66 +++ packages/core/src/cloud/trace-sync-config.ts | 64 +++ packages/core/src/cloud/types.ts | 41 ++ packages/core/src/config.ts | 53 +-- .../cloud-trace-outbox-repository.test.ts | 75 ++++ .../db/__tests__/migration-backcompat.test.ts | 8 +- .../src/db/cloud-trace-outbox-repository.ts | 187 ++++++++ .../db/migrations/0005_cloud_trace_outbox.sql | 25 ++ .../__tests__/cloud-jwt-user-binding.test.ts | 20 + .../middleware/__tests__/dual-auth.test.ts | 337 ++++++++++++++ .../src/middleware/cloud-jwt-user-binding.ts | 89 ++++ packages/core/src/middleware/dual-auth.ts | 166 +++++++ .../core/src/middleware/require-bearer.ts | 26 +- packages/core/src/routes/documents.ts | 3 + packages/core/src/routes/memories.ts | 9 +- packages/core/src/server.ts | 36 ++ .../__tests__/cloud-trace-envelope.test.ts | 85 ++++ .../__tests__/cloud-trace-sync.test.ts | 22 + .../__tests__/deferred-audn-scheduler.test.ts | 68 +++ .../__tests__/memory-service-config.test.ts | 5 + .../src/services/audn-decision-executor.ts | 17 + .../core/src/services/cloud-trace-envelope.ts | 112 +++++ .../core/src/services/cloud-trace-sync.ts | 414 ++++++++++++++++++ .../src/services/deferred-audn-scheduler.ts | 66 +++ packages/core/src/services/deferred-audn.ts | 42 +- packages/core/src/services/embedding.ts | 8 +- packages/core/src/services/llm.ts | 12 +- packages/core/src/services/memory-crud.ts | 17 + packages/core/src/services/memory-ingest.ts | 51 ++- .../src/services/retrieval-side-effects.ts | 16 + pnpm-lock.yaml | 6 + scripts/ci/__tests__/release-policy.test.mjs | 230 ++++++++++ scripts/ci/release-policy.mjs | 267 ++++++++++- 58 files changed, 3749 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/internal-core-docker-image.yml create mode 100644 packages/core/scripts/show-local-core-key.sh create mode 100644 packages/core/src/__tests__/cloud-jwt-config.test.ts create mode 100644 packages/core/src/__tests__/helpers/jwt-auth-headers.ts create mode 100644 packages/core/src/__tests__/trace-sync-config.test.ts create mode 100644 packages/core/src/cloud/__tests__/heartbeat-client.test.ts create mode 100644 packages/core/src/cloud/cloud-api-client.ts create mode 100644 packages/core/src/cloud/env.ts create mode 100644 packages/core/src/cloud/heartbeat-client.ts create mode 100644 packages/core/src/cloud/instance-id.ts create mode 100644 packages/core/src/cloud/jwt-config.ts create mode 100644 packages/core/src/cloud/trace-sync-config.ts create mode 100644 packages/core/src/cloud/types.ts create mode 100644 packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts create mode 100644 packages/core/src/db/cloud-trace-outbox-repository.ts create mode 100644 packages/core/src/db/migrations/0005_cloud_trace_outbox.sql create mode 100644 packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts create mode 100644 packages/core/src/middleware/__tests__/dual-auth.test.ts create mode 100644 packages/core/src/middleware/cloud-jwt-user-binding.ts create mode 100644 packages/core/src/middleware/dual-auth.ts create mode 100644 packages/core/src/services/__tests__/cloud-trace-envelope.test.ts create mode 100644 packages/core/src/services/__tests__/cloud-trace-sync.test.ts create mode 100644 packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts create mode 100644 packages/core/src/services/cloud-trace-envelope.ts create mode 100644 packages/core/src/services/cloud-trace-sync.ts create mode 100644 packages/core/src/services/deferred-audn-scheduler.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 022beea..6fa8ea1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,15 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} + # release-policy.mjs parses workflow YAML with the root `yaml` + # devDependency, so this job needs an install (it previously ran on a + # bare checkout). + - name: Setup pnpm + run: | + corepack enable + corepack prepare pnpm@${PNPM_VERSION} --activate + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts - name: Run release-policy guardrails run: node scripts/ci/release-policy.mjs - name: Run release-policy unit tests diff --git a/.github/workflows/internal-core-docker-image.yml b/.github/workflows/internal-core-docker-image.yml new file mode 100644 index 0000000..70bac7b --- /dev/null +++ b/.github/workflows/internal-core-docker-image.yml @@ -0,0 +1,275 @@ +name: Internal Core Docker Image + +# Internal-only convenience lane. Builds packages/core/Dockerfile from any ref +# of atomicmemory-internal and pushes it to the PRIVATE package +# ghcr.io/atomicstrata/atomicmemory-core-internal for internal testing. +# +# This is NOT the release pipeline. The official image +# ghcr.io/atomicstrata/atomicmemory-core is published exclusively by +# publish-core-docker.yml running on the public repo, anchored to the +# published @atomicmemory/core npm package. Nothing here may ever push to the +# official package name, and the tag guard below refuses ref-derived tags that +# are release-shaped (semver with or without prerelease/build suffix) or that +# collide with a reserved alias, so an internal build can never masquerade as +# a release. Builds of main additionally publish the curated floating aliases +# `internal-latest` and `latest`; both live only on the private internal +# package and always point at the same digest. +# +# Naming contract with release-policy: scripts/ci/release-policy.mjs forbids +# workflow_dispatch on every `.github/workflows/publish-*.yml` because those +# are release lanes that must only be triggerable by the audited +# repository_dispatch release flow. This workflow is deliberately named +# OUTSIDE that prefix: it is operator-dispatchable by design and never touches +# release artifacts or release tag namespaces. If this file is ever renamed to +# publish-*.yml, release-policy will (correctly) fail it. +# +# Repo guard: like sync-to-private.yml, this file is mirrored into the public +# repo by the public export, so the job runs ONLY on +# atomicstrata/atomicmemory-internal. Keep this file free of anything +# confidential; it is public. + +on: + workflow_dispatch: + inputs: + ref: + description: "Branch, tag, or SHA of atomicmemory-internal to build" + required: false + default: main + type: string + platforms: + description: "Comma-separated buildx platforms" + required: false + default: linux/amd64,linux/arm64 + type: string + push: + branches: + - main + # Mirrors the core-image-affecting classification used by ci.yml's + # core-docker-smoke job (packages/core + root build-context inputs), + # plus this workflow itself. + paths: + - "packages/core/**" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "package.json" + - "turbo.json" + - ".github/workflows/internal-core-docker-image.yml" + +permissions: + contents: read + packages: write + +concurrency: + group: internal-core-docker-image-${{ inputs.ref || github.ref_name }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core-internal + DEFAULT_PLATFORMS: linux/amd64,linux/arm64 + +jobs: + publish: + name: build and push internal core image + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout requested ref + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + # The core image is several GB per platform and the verify step loads + # each platform's image into dockerd on top of the buildx cache; the + # standard runner's free disk is not enough (the first run died on + # ENOSPC six minutes in). Reclaim the runner's preinstalled bloat + # before building. + - name: Free runner disk space + run: | + set -euo pipefail + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/share/boost + sudo docker image prune --all --force >/dev/null + df -h / + + - name: Compute image tags + id: tags + env: + RAW_REF: ${{ inputs.ref || github.ref_name }} + run: | + set -euo pipefail + + sha="$(git rev-parse HEAD)" + short_sha="${sha:0:7}" + core_version="$(node -p "require('./packages/core/package.json').version")" + + # Slugify the ref into a valid docker tag: lowercase, restricted + # charset, no leading separator, bounded length. + slug="$(tr '[:upper:]' '[:lower:]' <<<"${RAW_REF}" | sed -e 's![^a-z0-9._-]!-!g' -e 's!^[.-]*!!' | cut -c1-100)" + + tags=("sha-${short_sha}") + + # Both guards below apply to the REF-DERIVED slug only. The curated + # aliases added afterwards are hardcoded and intentional; the slug is + # whatever a branch happens to be named, so it is the only untrusted + # tag source and must fail closed. + # + # Skip the slug tag when the ref was itself a commit sha; sha- + # already covers it. + if [[ -n "${slug}" && ! "${slug}" =~ ^[0-9a-f]{7,40}$ ]]; then + # Reserved namespaces: a branch named latest or internal-latest + # could hijack a floating main alias, and sha-*/verify-* could + # overwrite another commit's tag or a verify scratch tag. + if [[ "${slug}" == "latest" || "${slug}" == "internal-latest" || "${slug}" =~ ^(sha|verify)- ]]; then + echo "::error::Ref slug '${slug}' collides with a reserved tag namespace (latest, internal-latest, sha-*, verify-*). Rename the branch or dispatch with the commit sha instead." + exit 1 + fi + # Release shapes: bare majors (1 / v2, the official floating-alias + # shape), bare semver (1.2 / 1.2.3 / v1.2.3), and semver with any + # prerelease or build suffix (1.2.3-rc.1, v1.2.3-beta.2, 1.2.3+abc). + if [[ "${slug}" =~ ^v?[0-9]+$ || "${slug}" =~ ^v?[0-9]+(\.[0-9]+){1,2}([-+.].*)?$ ]]; then + echo "::error::Ref slug '${slug}' looks like a release tag; internal builds must not publish semver-shaped tags. Rename the ref or dispatch with a different ref." + exit 1 + fi + tags+=("${slug}") + fi + + # Curated floating aliases for main. `latest` exists so a bare + # `docker pull ${IMAGE_NAME}` (which resolves to :latest) works; it + # is published only on this PRIVATE internal package and always + # points at the same digest as internal-latest. The official + # ghcr.io/atomicstrata/atomicmemory-core:latest is published solely + # by publish-core-docker.yml on the public repo and is untouched. + if [[ "${slug}" == "main" ]]; then + tags+=("internal-latest" "latest") + fi + + tag_args=() + refs=() + for tag in "${tags[@]}"; do + tag_args+=("--tag" "${IMAGE_NAME}:${tag}") + refs+=("${IMAGE_NAME}:${tag}") + done + + { + echo "sha=${sha}" + echo "short_sha=${short_sha}" + echo "core_version=${core_version}" + echo "tag_args=${tag_args[*]}" + echo "refs=${refs[*]}" + } >>"${GITHUB_OUTPUT}" + + echo "Building @atomicmemory/core ${core_version} at ${sha}" + printf 'Will push:\n' + printf ' - %s\n' "${refs[@]}" + + - name: Login to GHCR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + + # Dynamic values reach the run scripts below ONLY via step env, never + # by interpolating ${{ ... }} into shell text: a dispatch input or a + # hostile branch's package.json version could otherwise inject shell. + # release-policy enforces this (no ${{ in run scripts of this file). + - name: Build and verify every requested platform + # Every platform that will be pushed is first built locally and + # verified (deployed package version readable via node). No platform + # selection can bypass verification; non-native platforms run under + # QEMU, and their build layers are reused by the push step via the + # shared buildx cache. + env: + PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }} + SHORT_SHA: ${{ steps.tags.outputs.short_sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + run: | + set -euo pipefail + + IFS=',' read -r -a platforms <<<"${PLATFORMS}" + for platform in "${platforms[@]}"; do + platform="$(tr -d '[:space:]' <<<"${platform}")" + verify_suffix="verify-${SHORT_SHA}-${platform//\//-}" + + docker buildx build \ + --platform "${platform}" \ + --file packages/core/Dockerfile \ + --load \ + --tag "${IMAGE_NAME}:${verify_suffix}" \ + . + + image_version="$(docker run --rm --platform "${platform}" --entrypoint node "${IMAGE_NAME}:${verify_suffix}" -p "require('./package.json').version")" + if [[ "${image_version}" != "${CORE_VERSION}" ]]; then + echo "::error::${platform} image package version is ${image_version}, expected ${CORE_VERSION}" + exit 1 + fi + echo "${platform}: verified @atomicmemory/core ${image_version}" + + # Free dockerd storage before the next platform loads; the push + # step rebuilds from the (kept) buildx cache, not from dockerd. + docker image rm --force "${IMAGE_NAME}:${verify_suffix}" >/dev/null + done + + - name: Build and push internal image + env: + PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }} + TAG_ARGS: ${{ steps.tags.outputs.tag_args }} + GIT_SHA: ${{ steps.tags.outputs.sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + run: | + set -euo pipefail + + read -r -a tag_args <<<"${TAG_ARGS}" + + docker buildx build \ + --platform "${PLATFORMS}" \ + --file packages/core/Dockerfile \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GIT_SHA}" \ + --label "org.opencontainers.image.version=${CORE_VERSION}" \ + --label "org.opencontainers.image.title=@atomicmemory/core (internal test build)" \ + "${tag_args[@]}" \ + --push \ + . + + - name: Report published image + env: + GIT_SHA: ${{ steps.tags.outputs.sha }} + SHORT_SHA: ${{ steps.tags.outputs.short_sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + REFS: ${{ steps.tags.outputs.refs }} + run: | + set -euo pipefail + + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:sha-${SHORT_SHA}" --format '{{ json .Manifest }}' | jq -r '.digest')" + + { + echo "## Internal core image published" + echo "" + echo "- revision: \`${GIT_SHA}\`" + echo "- package version: \`${CORE_VERSION}\`" + echo "- digest-pinned ref (immutable, prefer this for sharing): \`${IMAGE_NAME}@${digest}\`" + echo "- tags (mutable, a rebuild of the same ref moves them):" + for image_ref in ${REFS}; do + echo " - \`${image_ref}\`" + done + echo "" + echo "Ops integration-smoke usage (from the smoke harness directory, after docker login ghcr.io):" + echo "" + echo '```' + echo "CORE_SOURCE_MODE=image \\" + echo "CORE_IMAGE_REF=${IMAGE_NAME}@${digest} \\" + echo "CORE_PACKAGE_VERSION=${CORE_VERSION} \\" + echo " bash ci/run-required.sh" + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" diff --git a/package.json b/package.json index 8280400..d275e9c 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,8 @@ "@typescript-eslint/parser": "^8.59.3", "eslint": "9.39.1", "fallow": "2.75.0", - "turbo": "2.9.14" + "turbo": "2.9.14", + "yaml": "^2.8.3" }, "pnpm": { "overrides": { diff --git a/packages/core/.env.example b/packages/core/.env.example index 236e7aa..7c1310c 100644 --- a/packages/core/.env.example +++ b/packages/core/.env.example @@ -22,8 +22,42 @@ OPENAI_API_KEY= # Shared API key clients must send as `Authorization: Bearer `. # Rotate by restarting the server with a new value. # Generate with: openssl rand -hex 32 -# Docker image local mode defaults this to `local-dev-key` when omitted. -CORE_API_KEY=replace-with-a-strong-random-secret +# Docker image local mode: auto-generated and persisted to +# $CORE_STATE_DIR/core-api-key (default /var/lib/atomicmemory/state/core-api-key) +# when omitted. Required in production/staging (RAW_STORAGE_DEPLOYMENT_ENV). +# CORE_API_KEY=replace-with-a-strong-random-secret + +# Optional: verify Cloud-issued RS256 JWTs for connected-local console access. +# JWKS_URL + ISSUER + AUDIENCE must be set together; when unset, only +# CORE_API_KEY is accepted. +# CLOUD_JWKS_URL=https://api.dev.atomicstrata.ai/.well-known/atomic-core/jwks.json +# CLOUD_JWT_ISSUER=https://api.dev.atomicstrata.ai +# CLOUD_JWT_AUDIENCE=atomicmemory-core +# Optional project binding. When set, tokens for other projects are rejected. +# When omitted (single-key local), Core trusts the token's own project_id. +# CLOUD_PROJECT_ID=proj_your_connected_local_project +# Temporary until Cloud mint includes memory_user_id (dev PoC uses user "default"): +# CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default +# Optional — when false (default), CORE_API_KEY is rejected once JWT verify is enabled. +# CLOUD_JWT_STATIC_KEY_FALLBACK=false + +# --- Connected-local (Docker entrypoint defaults) --- +# Minimal self-hosted + Cloud dev console: set only these in docker run. +# OPENAI_API_KEY=sk-... +# ATOMICMEMORY_API_KEY=amc_live_... +# Mount atomic-memory-state (or equivalent) so the entrypoint can persist +# an auto-generated CORE_API_KEY at /var/lib/atomicmemory/state/core-api-key. +# Retrieve after start: docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key +# Optional override: CORE_API_KEY=your-local-secret (skips auto-generation) +# Setting ATOMICMEMORY_API_KEY turns connected-local on; the Docker entrypoint +# then applies tier defaults when unset (CLOUD_ENV=dev): +# ATOMICMEMORY_API_URL, CLOUD_TRACE_SYNC_ENABLED=true, +# CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, CLOUD_JWT_AUDIENCE=atomicmemory-core, +# ALLOWED_ORIGINS, CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default, +# CLOUD_JWT_STATIC_KEY_FALLBACK=true +# CLOUD_PROJECT_ID is optional: set it to pin one project; when omitted Core +# trusts the Cloud-minted token's own project_id. Override tier with +# CLOUD_ENV=staging|production # Trusted-proxy identity guard (Radar C4). The shared CORE_API_KEY above # authenticates the *caller process*, not the end user; `user_id` is @@ -206,3 +240,21 @@ EMBEDDING_DIMENSIONS=1536 # On Railway, DATABASE_URL is injected by the Postgres plugin. # Set OPENAI_API_KEY in Railway service variables. # PORT is injected automatically by Railway. + +# --- Optional Cloud trace sync (OSS Core → Cloud observability) --- +# Disabled when all vars are absent, or when CLOUD_TRACE_SYNC_ENABLED=false. +# Partial configuration (some vars set without ENABLED=true) fails startup. +# CLOUD_TRACE_SYNC_ENABLED=true +# ATOMICMEMORY_API_URL=https://api.atomicstrata.ai +# ATOMICMEMORY_API_KEY=amc_replace_me +# CORE_INSTANCE_ID=optional-stable-installation-id +# CLOUD_TRACE_SYNC_BATCH_SIZE=100 +# CLOUD_TRACE_SYNC_MAX_ATTEMPTS=8 +# CLOUD_TRACE_SYNC_BASE_RETRY_MS=1000 +# CLOUD_TRACE_SYNC_MAX_RETRY_MS=300000 +# CLOUD_TRACE_SYNC_POLL_INTERVAL_MS=5000 +# CLOUD_TRACE_SYNC_SENT_RETENTION_MS=86400000 +# CLOUD_TRACE_SYNC_SHUTDOWN_DRAIN_MS=10000 +# CLOUD_TRACE_SYNC_CLAIM_STALE_MS=300000 +# CLOUD_TRACE_SYNC_DEAD_LETTER_RETENTION_MS=604800000 +# CLOUD_TRACE_SYNC_MAX_PENDING=10000 diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 7ce4a1d..febdff3 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -6,6 +6,34 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [1.2.0] - 2026-07-28 + +### Added +- Connected-local Cloud support with optional Cloud-issued RS256 JWT + verification, user/project claim binding, and static API-key fallback. +- Durable, content-free memory trace synchronization through a PostgreSQL + outbox with bounded backlog, retry, dead-letter, and auth-recovery handling. +- Automatic generation and persistence of local `CORE_API_KEY` values and a + stable Core instance ID, including connected-local Docker defaults. +- `EMBEDDING_DTYPE` configuration for larger quantized local embedding models. +- A background scheduler for deferred AUDN reconciliation. + +### Changed +- Deferred AUDN reconciliation now runs with bounded parallelism. + +### Fixed +- Anthropic extraction requests use an assistant prefill to enforce JSON + responses. +- Idempotent re-indexing advances extraction status only through valid state + transitions. +- Local API-key overrides persist across container restarts. + +### Security +- Cloud JWT verification fails closed on issuer, audience, algorithm, user, and + project mismatches. +- Outbound Cloud trace envelopes use allowlisted summary fields and exclude + verbatim memory and query content. + ## [1.1.1] - 2026-06-15 ### Added diff --git a/packages/core/README.md b/packages/core/README.md index f3f1a71..3918dd6 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -112,6 +112,46 @@ EOF docker compose -f docker-compose.image.yml up ``` +### Connected to Atomic Memory Cloud + +Link a local Core instance to a Cloud **Local** project with a single `amc_…` API +key (traces + heartbeat). Apps on localhost use **`CORE_API_KEY`** (Local access +key) — never paste the Cloud key into SDK/CLI calls. + +1. In Cloud: create a Local project → create an API key (`amc_…`). +2. Copy the console **Connect** `docker run` snippet (or use the template below). +3. After Core starts, retrieve the auto-generated local key (or use your own override). +4. SDK/CLI: `Authorization: Bearer $CORE_API_KEY` → `http://127.0.0.1:17350`. +5. Console **Memories** use an automatic Cloud-minted JWT (no key paste). + +```bash +export OPENAI_API_KEY=sk-... +export ATOMICMEMORY_API_KEY=amc_... # Cloud connect only +export ATOMICMEMORY_API_URL=https://api.atomicstrata.ai + +docker run -d --name atomic-memory \ + --restart unless-stopped \ + -p 127.0.0.1:17350:17350 \ + -v atomic-memory-data:/var/lib/atomicmemory/postgres \ + -v atomic-memory-state:/var/lib/atomicmemory/state \ + -e OPENAI_API_KEY \ + -e ATOMICMEMORY_API_KEY \ + -e ATOMICMEMORY_API_URL \ + ghcr.io/atomicstrata/atomicmemory-core:stable + +# Retrieve the generated local access key for SDK/CLI (after /health is ok): +docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key +``` + +**Optional:** pass `-e CORE_API_KEY=your-secret` to pin a user-defined local key +instead of auto-generation. The state volume still persists whichever key is in use. + +When `ATOMICMEMORY_API_KEY` is set, Core enables outbound trace sync and +periodic heartbeat to `POST /v1/runtimes/heartbeat`. Invalid or revoked Cloud +credentials degrade observability only — local memory ingest/search keep working. + +Local-only (no Cloud): run with only `OPENAI_API_KEY` — same image, no Cloud env. + ### Docker from source ```bash diff --git a/packages/core/SECURITY.md b/packages/core/SECURITY.md index 9aa68be..93091e2 100644 --- a/packages/core/SECURITY.md +++ b/packages/core/SECURITY.md @@ -106,9 +106,52 @@ Non-boolean values for `TRUSTED_PROXY_MODE` are rejected at startup in all envs. To run a hosted deployment without the guard you must change the deployment env, not silently disable the guard. +### Cloud JWT verification (optional, connected-local) + +When `CLOUD_JWKS_URL`, `CLOUD_JWT_ISSUER`, and `CLOUD_JWT_AUDIENCE` are all set, +core accepts short-lived RS256 JWTs minted by Atomic Strata Cloud (instead of +`CORE_API_KEY` unless fallback is explicitly enabled). Verified claims inject +`X-AtomicMemory-Asserted-User` from `memory_user_id` (not `sub`) and workspace +from `project_id`. When a request carries wire `user_id` (body, query, or +`X-AtomicMemory-User-Id`), it must match the token's `memory_user_id` or the +request fails closed with 403. Routers that parse JSON after auth (for example +`/v1/documents`) re-check this binding once the body is available. JWT-scoped +mutating routes without an explicit `user_id` default to the asserted identity +(for example `POST /v1/memories/reconcile` never runs a global reconcile for a +JWT caller). When `CLOUD_PROJECT_ID` is set, token `project_id` must match it; +when omitted, Core trusts the token's own `project_id` claim (single-tenant +local posture). The JWKS trio must be set together; partial configuration fails +startup. + +`CLOUD_JWT_STATIC_KEY_FALLBACK=true` preserves the legacy `CORE_API_KEY` +acceptance path for non-JWT bearer tokens on connected-local deployments. When +false or unset, non-JWT bearer tokens are rejected once Cloud JWT verification +is enabled. + +Core prefetches JWKS at startup; JWT auth remains degraded until keys load. + +### Connected-local local access key (Docker) + +For self-hosted Docker (`RAW_STORAGE_DEPLOYMENT_ENV=local`), the entrypoint +generates a random `CORE_API_KEY` on first boot when the env var is unset, +persists it at `/var/lib/atomicmemory/state/core-api-key` (mode `600`), and +reuses it on restart when the state volume is mounted. Operators may override +with `-e CORE_API_KEY=…` for pinned secrets. Treat the state file like a +credential: restrict volume access, never log or commit it, and rotate by +deleting the state file (or volume) and restarting Core. + ### Deferred (not implemented) Per-user tokens / a multi-tenant auth layer in core (so a leaked credential compromises one user instead of all) are a larger future change and are **out of scope** here. The current contract assumes a trusted proxy in front of core for multi-user deployments. + +## Outbound Cloud Trace Sync + +When `CLOUD_TRACE_SYNC_ENABLED=true`, Core enqueues **redacted operation +metadata** (counts, ids, durations) to a local Postgres outbox and uploads +it to the configured Cloud observability endpoint. Envelopes **must not** +carry verbatim memory text, search queries, or fact content — only +allowlisted summary fields pass the outbound sanitizer. Treat the Cloud API +key (`amc_*`) like any other deployment secret. diff --git a/packages/core/docker-compose.yml b/packages/core/docker-compose.yml index ec50207..63eec80 100644 --- a/packages/core/docker-compose.yml +++ b/packages/core/docker-compose.yml @@ -33,6 +33,8 @@ services: - ${ENV_FILE:-.env} extra_hosts: - "host.docker.internal:host-gateway" + volumes: + - corestate:/var/lib/atomicmemory/state depends_on: postgres: condition: service_healthy @@ -50,3 +52,4 @@ services: volumes: pgdata: + corestate: diff --git a/packages/core/package.json b/packages/core/package.json index cedc928..11d30d7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@atomicmemory/core", - "version": "1.1.1", + "version": "1.2.0", "description": "Open-source memory engine for AI applications — semantic retrieval, AUDN mutation, and contradiction-safe claim versioning.", "type": "module", "license": "Apache-2.0", @@ -92,6 +92,7 @@ "@filoz/synapse-sdk": "^0.41.0", "@huggingface/transformers": "^3.8.1", "express": "^5.1.0", + "jose": "^6.2.3", "multiformats": "^13.4.1", "node-pg-migrate": "8.0.4", "openai": "^4.80.0", diff --git a/packages/core/scripts/docker-entrypoint.sh b/packages/core/scripts/docker-entrypoint.sh index 5cf9dd1..1c4a566 100644 --- a/packages/core/scripts/docker-entrypoint.sh +++ b/packages/core/scripts/docker-entrypoint.sh @@ -9,13 +9,122 @@ set -euo pipefail APP_PID="" POSTGRES_STARTED=false -LOCAL_DOCKER_CORE_API_KEY="local-dev-key" LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +CORE_STATE_DIR="${CORE_STATE_DIR:-/var/lib/atomicmemory/state}" +CORE_API_KEY_FILE="$CORE_STATE_DIR/core-api-key" log() { printf '[entrypoint] %s\n' "$*" } +is_hosted_deployment_env() { + case "${1:-local}" in + production|staging) return 0 ;; + *) return 1 ;; + esac +} + +generate_core_api_key() { + od -vN32 -An -tx1 /dev/urandom | tr -d ' \n' +} + +persist_core_api_key() { + local key="$1" + mkdir -p "$CORE_STATE_DIR" + printf '%s\n' "$key" > "$CORE_API_KEY_FILE" + chmod 600 "$CORE_API_KEY_FILE" +} + +resolve_core_api_key() { + if is_hosted_deployment_env "${RAW_STORAGE_DEPLOYMENT_ENV:-local}"; then + if [ -z "${CORE_API_KEY:-}" ]; then + log "CORE_API_KEY is required when RAW_STORAGE_DEPLOYMENT_ENV=${RAW_STORAGE_DEPLOYMENT_ENV:-local}" + exit 1 + fi + export CORE_API_KEY + log "CORE_API_KEY from environment (hosted — not persisted locally)" + return + fi + + if [ -n "${CORE_API_KEY:-}" ]; then + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY from environment and persisted to $CORE_API_KEY_FILE" + return + fi + + if [ -s "$CORE_API_KEY_FILE" ]; then + CORE_API_KEY="$(tr -d '[:space:]' < "$CORE_API_KEY_FILE")" + if [ -n "$CORE_API_KEY" ]; then + export CORE_API_KEY + log "CORE_API_KEY loaded from $CORE_API_KEY_FILE" + return + fi + fi + + CORE_API_KEY="$(generate_core_api_key)" + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY generated and persisted to $CORE_API_KEY_FILE" +} + +cloud_tier_api_url() { + case "${1:-dev}" in + dev) printf '%s' 'https://api.dev.atomicstrata.ai' ;; + staging) printf '%s' 'https://api.staging.atomicstrata.ai' ;; + production|prod) printf '%s' 'https://api.atomicstrata.ai' ;; + *) + log "Unknown CLOUD_ENV: ${1}" + exit 1 + ;; + esac +} + +cloud_tier_memory_origin() { + case "${1:-dev}" in + dev) printf '%s' 'https://memory.dev.atomicstrata.ai' ;; + staging) printf '%s' 'https://memory.staging.atomicstrata.ai' ;; + production|prod) printf '%s' 'https://memory.atomicstrata.ai' ;; + *) + log "Unknown CLOUD_ENV: ${1}" + exit 1 + ;; + esac +} + +# When running self-hosted Core for connected-local, apply tier defaults so +# operators only pass OPENAI_API_KEY + ATOMICMEMORY_API_KEY. The presence of +# ATOMICMEMORY_API_KEY is the single switch that turns connected-local on; +# CLOUD_PROJECT_ID is optional (Core trusts the token's project_id when unset). +apply_connected_local_defaults() { + if is_hosted_deployment_env "${RAW_STORAGE_DEPLOYMENT_ENV:-local}"; then + return + fi + + if [ -z "${ATOMICMEMORY_API_KEY:-}" ]; then + return + fi + + local tier="${CLOUD_ENV:-dev}" + local api_url memory_origin + + api_url="$(cloud_tier_api_url "$tier")" + memory_origin="$(cloud_tier_memory_origin "$tier")" + + export CLOUD_TRACE_SYNC_ENABLED="${CLOUD_TRACE_SYNC_ENABLED:-true}" + export ATOMICMEMORY_API_URL="${ATOMICMEMORY_API_URL:-$api_url}" + + export CLOUD_JWKS_URL="${CLOUD_JWKS_URL:-${api_url}/.well-known/atomic-core/jwks.json}" + export CLOUD_JWT_ISSUER="${CLOUD_JWT_ISSUER:-$api_url}" + export CLOUD_JWT_AUDIENCE="${CLOUD_JWT_AUDIENCE:-atomicmemory-core}" + export ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-$memory_origin}" + export CLOUD_JWT_STATIC_KEY_FALLBACK="${CLOUD_JWT_STATIC_KEY_FALLBACK:-true}" + if [ "$tier" = "dev" ]; then + export CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID="${CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID:-default}" + fi + log "Connected-local defaults applied (CLOUD_ENV=$tier, api=$ATOMICMEMORY_API_URL)" +} + stop_postgres() { if [ "$POSTGRES_STARTED" = "true" ]; then log "Stopping embedded Postgres..." @@ -43,18 +152,12 @@ configure_local_defaults() { local deployment_env="${RAW_STORAGE_DEPLOYMENT_ENV:-local}" export RAW_STORAGE_DEPLOYMENT_ENV="$deployment_env" - if [ -z "${CORE_API_KEY:-}" ]; then - if [ "$deployment_env" = "production" ]; then - log "CORE_API_KEY is required when RAW_STORAGE_DEPLOYMENT_ENV=production" - exit 1 - fi - export CORE_API_KEY="$LOCAL_DOCKER_CORE_API_KEY" - log "CORE_API_KEY not set; using local Docker default '$CORE_API_KEY'" - fi + apply_connected_local_defaults + resolve_core_api_key if [ -z "${STORAGE_KEY_HMAC_SECRET:-}" ]; then - if [ "$deployment_env" = "production" ]; then - log "STORAGE_KEY_HMAC_SECRET is required when RAW_STORAGE_DEPLOYMENT_ENV=production" + if is_hosted_deployment_env "$deployment_env"; then + log "STORAGE_KEY_HMAC_SECRET is required when RAW_STORAGE_DEPLOYMENT_ENV=$deployment_env" exit 1 fi export STORAGE_KEY_HMAC_SECRET="$LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET" diff --git a/packages/core/scripts/docker-smoke-test.sh b/packages/core/scripts/docker-smoke-test.sh index 694935a..58728a4 100755 --- a/packages/core/scripts/docker-smoke-test.sh +++ b/packages/core/scripts/docker-smoke-test.sh @@ -26,6 +26,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +USER_SUPPLIED_CORE_API_KEY="${CORE_API_KEY:-}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-}" COMPOSE_BASE_FILE="${COMPOSE_BASE_FILE:-docker-compose.yml}" APP_PORT="${APP_PORT:-}" @@ -34,6 +35,7 @@ SMOKE_ENV_FILE="$PROJECT_DIR/.env.docker-smoke-test" ENV_FILE_CREATED_BY_SCRIPT=0 SMOKE_CORE_API_KEY="test-core-api-key-do-not-leak" SMOKE_STORAGE_KEY_HMAC_SECRET="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +CORE_STATE_KEY_PATH="/var/lib/atomicmemory/state/core-api-key" HEALTH_TIMEOUT=90 HEALTH_INTERVAL=2 @@ -123,13 +125,21 @@ fi if [[ -z "${ENV_FILE:-}" && ! -f "$PROJECT_DIR/.env" ]]; then log "Creating stub smoke env file (compose env_file requires one)" - cat > "$SMOKE_ENV_FILE" < "$SMOKE_ENV_FILE" < "$SMOKE_ENV_FILE" </dev/null | tail -1 | cut -d= -f2- || true)" fi fi +if [[ -z "$CORE_API_KEY" ]]; then + log "No CORE_API_KEY in env — reading entrypoint-generated key from container" + CORE_API_KEY="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" +fi if [[ -z "$CORE_API_KEY" ]]; then fail "CORE_API_KEY is required for authenticated /v1 smoke checks" exit 1 fi AUTH_HEADER=("Authorization: Bearer ${CORE_API_KEY}") +saved_core_api_key="$CORE_API_KEY" +if [[ -n "${USER_SUPPLIED_CORE_API_KEY}" ]]; then + log "Test: explicit CORE_API_KEY override persisted to state volume" + state_key="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" + assert_ok "Explicit CORE_API_KEY override written to state volume" \ + '[ "$state_key" = "$saved_core_api_key" ]' + + log "Test: explicit CORE_API_KEY override survives recreate without env" + if (( ENV_FILE_CREATED_BY_SCRIPT == 1 )); then + cat > "$SMOKE_ENV_FILE" </dev/null +recreate_timeout=180 +elapsed=0 +while true; do + if curl -sf "http://localhost:${APP_PORT}/health" >/dev/null 2>&1; then + break + fi + if [[ $elapsed -ge $recreate_timeout ]]; then + fail "App did not become healthy after recreate within ${recreate_timeout}s" + warn "--- app container logs (recreate) ---" + docker compose -p "$COMPOSE_PROJECT" logs app 2>&1 | tail -40 + exit 1 + fi + sleep "$HEALTH_INTERVAL" + elapsed=$((elapsed + HEALTH_INTERVAL)) +done +recreated_key="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" +if [[ -n "${USER_SUPPLIED_CORE_API_KEY}" ]]; then + assert_ok "Explicit CORE_API_KEY override unchanged after recreate without env" \ + '[ "$recreated_key" = "$saved_core_api_key" ]' +else + assert_ok "CORE_API_KEY unchanged after container recreate (state volume)" \ + '[ "$recreated_key" = "$saved_core_api_key" ]' +fi +CORE_API_KEY="$recreated_key" +AUTH_HEADER=("Authorization: Bearer ${CORE_API_KEY}") + # --- Test 1: Root health --- log "Test: root /health endpoint" health_body=$(curl -sf "$BASE/health") diff --git a/packages/core/scripts/show-local-core-key.sh b/packages/core/scripts/show-local-core-key.sh new file mode 100644 index 0000000..ead1613 --- /dev/null +++ b/packages/core/scripts/show-local-core-key.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Print the persisted local CORE_API_KEY from a running Core container. +# Reads the state volume file written by docker-entrypoint.sh (generated or +# explicit override). Usage: ./scripts/show-local-core-key.sh [container_name] +set -euo pipefail +CONTAINER="${1:-atomic-memory}" +docker exec "$CONTAINER" cat /var/lib/atomicmemory/state/core-api-key diff --git a/packages/core/src/__tests__/cloud-jwt-config.test.ts b/packages/core/src/__tests__/cloud-jwt-config.test.ts new file mode 100644 index 0000000..4cbce1e --- /dev/null +++ b/packages/core/src/__tests__/cloud-jwt-config.test.ts @@ -0,0 +1,109 @@ +/** + * Cloud JWT env parsing — all-or-nothing validation. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const trackedEnvNames = [ + 'CLOUD_JWKS_URL', + 'CLOUD_JWT_ISSUER', + 'CLOUD_JWT_AUDIENCE', + 'CLOUD_PROJECT_ID', + 'CLOUD_JWT_STATIC_KEY_FALLBACK', + 'RAW_STORAGE_DEPLOYMENT_ENV', + 'DATABASE_URL', + 'CORE_API_KEY', + 'STORAGE_KEY_HMAC_SECRET', + 'EMBEDDING_DIMENSIONS', + 'EMBEDDING_PROVIDER', + 'OPENAI_API_KEY', +] as const; + +const originalEnv = Object.fromEntries( + trackedEnvNames.map((name) => [name, process.env[name]]), +) as Record<(typeof trackedEnvNames)[number], string | undefined>; + +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +beforeEach(() => { + process.env.DATABASE_URL = + 'postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory_test'; + process.env.CORE_API_KEY = 'test-core-api-key'; + process.env.STORAGE_KEY_HMAC_SECRET = + '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f'; + process.env.EMBEDDING_DIMENSIONS = '384'; + process.env.EMBEDDING_PROVIDER = 'openai'; + process.env.OPENAI_API_KEY = 'test-openai-key'; + process.env.RAW_STORAGE_DEPLOYMENT_ENV = 'local'; + delete process.env.CLOUD_JWKS_URL; + delete process.env.CLOUD_JWT_ISSUER; + delete process.env.CLOUD_JWT_AUDIENCE; + delete process.env.CLOUD_PROJECT_ID; + delete process.env.CLOUD_JWT_STATIC_KEY_FALLBACK; +}); + +afterEach(() => { + for (const name of trackedEnvNames) restoreEnv(name, originalEnv[name]); + vi.resetModules(); +}); + +async function loadConfig() { + vi.resetModules(); + return (await import('../config.js')).config; +} + +describe('parseCloudJwtConfig via config module', () => { + it('returns null when all CLOUD JWT vars are unset', async () => { + expect((await loadConfig()).cloudJwt).toBeNull(); + }); + + it('loads profile when all required Cloud JWT vars are set', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + process.env.CLOUD_PROJECT_ID = 'proj_test'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt).toEqual({ + jwksUrl: 'https://api.test/.well-known/atomic-core/jwks.json', + issuer: 'https://api.test', + audience: 'atomicmemory-core', + projectId: 'proj_test', + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }); + }); + + it('leaves projectId null when CLOUD_PROJECT_ID is unset (single-key local)', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt).toEqual({ + jwksUrl: 'https://api.test/.well-known/atomic-core/jwks.json', + issuer: 'https://api.test', + audience: 'atomicmemory-core', + projectId: null, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }); + }); + + it('parses explicit static-key fallback boolean', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + process.env.CLOUD_PROJECT_ID = 'proj_test'; + process.env.CLOUD_JWT_STATIC_KEY_FALLBACK = 'true'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt?.staticKeyFallbackEnabled).toBe(true); + }); + + it('fails startup when only some Cloud JWT vars are set', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/jwks.json'; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow(/must all be set together/); + }); +}); diff --git a/packages/core/src/__tests__/deployment-config.test.ts b/packages/core/src/__tests__/deployment-config.test.ts index a767a83..26ece68 100644 --- a/packages/core/src/__tests__/deployment-config.test.ts +++ b/packages/core/src/__tests__/deployment-config.test.ts @@ -186,9 +186,29 @@ describe('deployment configuration', () => { it('entrypoint provides local Docker auth and storage defaults', () => { const entrypoint = readFileSync(resolve(ROOT, 'scripts/docker-entrypoint.sh'), 'utf-8'); - expect(entrypoint).toContain('LOCAL_DOCKER_CORE_API_KEY="local-dev-key"'); + expect(entrypoint).toContain('CORE_API_KEY_FILE'); + expect(entrypoint).toContain('generate_core_api_key'); + expect(entrypoint).toContain('persist_core_api_key'); + expect(entrypoint).toContain('is_hosted_deployment_env'); + expect(entrypoint).toContain('apply_connected_local_defaults'); + expect(entrypoint).toContain('CLOUD_ENV'); expect(entrypoint).toContain('LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET='); - expect(entrypoint).toContain('RAW_STORAGE_DEPLOYMENT_ENV=production'); + expect(entrypoint).toContain('production|staging'); + expect(entrypoint).toContain('CORE_API_KEY is required'); + expect(entrypoint).toContain('CORE_API_KEY from environment and persisted to $CORE_API_KEY_FILE'); + expect(entrypoint).toContain('CORE_API_KEY from environment (hosted — not persisted locally)'); + expect(entrypoint).toContain('CORE_API_KEY loaded from $CORE_API_KEY_FILE'); + expect(entrypoint).toContain('CORE_API_KEY generated and persisted'); + }); + + it('show-local-core-key helper reads persisted state volume path', () => { + const helper = readFileSync(resolve(ROOT, 'scripts/show-local-core-key.sh'), 'utf-8'); + expect(helper).toContain('/var/lib/atomicmemory/state/core-api-key'); + }); + + it('compose mounts persistent Core state for generated local API keys', () => { + const compose = readFileSync(resolve(ROOT, 'docker-compose.yml'), 'utf-8'); + expect(compose).toContain('corestate:/var/lib/atomicmemory/state'); }); it('creates non-root user', () => { diff --git a/packages/core/src/__tests__/helpers/jwt-auth-headers.ts b/packages/core/src/__tests__/helpers/jwt-auth-headers.ts new file mode 100644 index 0000000..2beed09 --- /dev/null +++ b/packages/core/src/__tests__/helpers/jwt-auth-headers.ts @@ -0,0 +1,10 @@ +/** + * Bearer header helper for Cloud-issued JWT auth in integration tests. + */ + +export function jwtAuthHeader(accessToken: string): Record { + if (accessToken.length === 0) { + throw new Error('jwtAuthHeader: accessToken must be non-empty'); + } + return { Authorization: `Bearer ${accessToken}` }; +} diff --git a/packages/core/src/__tests__/setup.ts b/packages/core/src/__tests__/setup.ts index f897ef1..ebef260 100644 --- a/packages/core/src/__tests__/setup.ts +++ b/packages/core/src/__tests__/setup.ts @@ -15,6 +15,11 @@ process.env.CORE_API_KEY ??= 'test-core-api-key'; process.env.STORAGE_KEY_HMAC_SECRET ??= '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f'; process.env.DATABASE_URL ??= 'postgresql://atomicmem:atomicmem@localhost:5433/atomicmem_test'; process.env.EMBEDDING_DIMENSIONS ??= '1536'; +// Contributor shells may export EMBEDDING_PROVIDER=stub for other packages; +// core tests require a real provider name at config load time. +if (!process.env.EMBEDDING_PROVIDER || process.env.EMBEDDING_PROVIDER === 'stub') { + process.env.EMBEDDING_PROVIDER = 'openai'; +} process.env.RAW_STORAGE_DEPLOYMENT_ENV ??= 'local'; // Mirror .env.test.example for route seam tests when no local env file exists; // production config still defaults this flag to false in src/config.ts. diff --git a/packages/core/src/__tests__/trace-sync-config.test.ts b/packages/core/src/__tests__/trace-sync-config.test.ts new file mode 100644 index 0000000..01859b9 --- /dev/null +++ b/packages/core/src/__tests__/trace-sync-config.test.ts @@ -0,0 +1,60 @@ +/** + * Cloud trace sync env parsing. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { parseCloudTraceSyncConfig } from '../cloud/trace-sync-config.js'; + +const tracked = [ + 'CLOUD_TRACE_SYNC_ENABLED', + 'ATOMICMEMORY_API_URL', + 'ATOMICMEMORY_API_KEY', + 'CORE_INSTANCE_ID', +] as const; + +const original = Object.fromEntries(tracked.map((name) => [name, process.env[name]])) as Record< + (typeof tracked)[number], + string | undefined +>; + +function restore(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +beforeEach(() => { + for (const name of tracked) delete process.env[name]; +}); + +afterEach(() => { + for (const name of tracked) restore(name, original[name]); +}); + +describe('parseCloudTraceSyncConfig', () => { + it('returns null when all vars are absent', () => { + expect(parseCloudTraceSyncConfig()).toBeNull(); + }); + + it('returns null when explicitly disabled even with sibling vars present', () => { + process.env.CLOUD_TRACE_SYNC_ENABLED = 'false'; + process.env.ATOMICMEMORY_API_URL = 'https://api.test'; + process.env.ATOMICMEMORY_API_KEY = 'amc_test_key'; + expect(parseCloudTraceSyncConfig()).toBeNull(); + }); + + it('loads config when enabled with credentials', () => { + process.env.CLOUD_TRACE_SYNC_ENABLED = 'true'; + process.env.ATOMICMEMORY_API_URL = 'https://api.test/'; + process.env.ATOMICMEMORY_API_KEY = 'amc_test_key'; + const cfg = parseCloudTraceSyncConfig(); + expect(cfg?.enabled).toBe(true); + expect(cfg?.apiUrl).toBe('https://api.test'); + expect(cfg?.claimStaleMs).toBeGreaterThan(0); + expect(cfg?.maxPending).toBeGreaterThan(0); + }); + + it('throws on partial config without explicit disable', () => { + process.env.ATOMICMEMORY_API_URL = 'https://api.test'; + expect(() => parseCloudTraceSyncConfig()).toThrow(/Partial Cloud trace sync/); + }); +}); diff --git a/packages/core/src/app/__tests__/versioned-mount.test.ts b/packages/core/src/app/__tests__/versioned-mount.test.ts index 1bc2868..758223f 100644 --- a/packages/core/src/app/__tests__/versioned-mount.test.ts +++ b/packages/core/src/app/__tests__/versioned-mount.test.ts @@ -14,7 +14,8 @@ * re-test route logic, which is covered by the router-level test files. */ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { exportJWK, generateKeyPair, SignJWT } from 'jose'; import { pool } from '../../db/pool.js'; import { setupTestSchema } from '../../db/__tests__/test-fixtures.js'; @@ -22,9 +23,16 @@ import { createCoreRuntime } from '../runtime-container.js'; import { createApp } from '../create-app.js'; import { bindEphemeral, type BootedApp } from '../bind-ephemeral.js'; import { authHeader } from '../../__tests__/helpers/auth-headers.js'; +import { jwtAuthHeader } from '../../__tests__/helpers/jwt-auth-headers.js'; +import { config } from '../../config.js'; const TEST_USER = 'versioned-mount-user'; const TEST_AGENT = 'versioned-mount-agent'; +const JWKS_URL = 'https://cloud.test/.well-known/atomic-core/jwks.json'; +const JWT_ISSUER = 'https://api.test'; +const JWT_AUDIENCE = 'atomicmemory-core'; +const JWT_PROJECT = 'proj_mount'; +const JWT_USER = 'cloud-jwt-mount-user'; /** * Assert the response is HTTP 401 with `error_code: 'unauthenticated'`. @@ -39,14 +47,68 @@ async function expectUnauthenticatedJson(res: Response): Promise { describe('createApp /v1 mount coverage', () => { let booted: BootedApp; + let jwtBooted: BootedApp; + let jwtKeyPairPromise: ReturnType; + + async function mintMountTestJwt(memoryUserId: string = JWT_USER): Promise { + const { privateKey } = await jwtKeyPairPromise; + return new SignJWT({ + project_id: JWT_PROJECT, + memory_user_id: memoryUserId, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'mount-test-kid' }) + .setIssuer(JWT_ISSUER) + .setAudience(JWT_AUDIENCE) + .setSubject(memoryUserId) + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey); + } beforeAll(async () => { await setupTestSchema(pool); booted = await bindEphemeral(createApp(await createCoreRuntime({ pool }))); + + jwtKeyPairPromise = generateKeyPair('RS256'); + const { publicKey } = await jwtKeyPairPromise; + const jwk = await exportJWK(publicKey); + jwk.kid = 'mount-test-kid'; + jwk.alg = 'RS256'; + jwk.use = 'sig'; + + const realFetch = globalThis.fetch.bind(globalThis); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url === JWKS_URL) { + return new Response(JSON.stringify({ keys: [jwk] }), { status: 200 }); + } + return realFetch(input, init); + }), + ); + + const jwtRuntime = await createCoreRuntime({ + pool, + config: { + ...config, + cloudJwt: { + jwksUrl: JWKS_URL, + issuer: JWT_ISSUER, + audience: JWT_AUDIENCE, + projectId: JWT_PROJECT, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }, + }, + }); + jwtBooted = await bindEphemeral(createApp(jwtRuntime)); }); afterAll(async () => { await booted.close(); + await jwtBooted.close(); + vi.unstubAllGlobals(); await pool.end(); }); @@ -217,6 +279,64 @@ describe('createApp /v1 mount coverage', () => { }); expect(res.status).toBe(204); }); + + it('POST /v1/memories/ingest rejects Cloud JWT body user_id mismatch with 403', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/memories/ingest`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: 'other-user', + content: 'hello', + source_site: 'test', + }), + }); + expect(res.status).toBe(403); + const body = (await res.json()) as { error_code: string }; + expect(body.error_code).toBe('forbidden'); + }); + + it('POST /v1/documents rejects Cloud JWT body user_id mismatch with 403', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/documents`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: 'other-user', + source_site: 'drive', + provider: 'test', + external_id: 'doc-cross-user', + }), + }); + expect(res.status).toBe(403); + const body = (await res.json()) as { error_code: string }; + expect(body.error_code).toBe('forbidden'); + }); + + it('POST /v1/memories/reconcile scopes Cloud JWT to asserted user when body omits user_id', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/memories/reconcile`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + expect(res.status).toBe(200); + }); + + it('GET /v1/memories/list accepts a Cloud JWT via jwtAuthHeader', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch( + `${jwtBooted.baseUrl}/v1/memories/list?user_id=${JWT_USER}`, + { headers: jwtAuthHeader(token) }, + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(Array.isArray(body.memories)).toBe(true); + }); }); async function expectUnauthenticatedRawUpload( diff --git a/packages/core/src/app/create-app.ts b/packages/core/src/app/create-app.ts index ea62510..38c0c5c 100644 --- a/packages/core/src/app/create-app.ts +++ b/packages/core/src/app/create-app.ts @@ -18,6 +18,7 @@ import { embedText } from '../services/embedding.js'; import { createStorageRouter } from '../routes/storage.js'; import { createEntityRouter } from '../routes/entities.js'; import { MAX_INDEX_TEXT_BYTES } from '../schemas/documents.js'; +import { createAuthMiddleware } from '../middleware/dual-auth.js'; import { requireBearer } from '../middleware/require-bearer.js'; import { assertedUserGuard } from '../middleware/asserted-user.js'; import { rejectNulInRequestTarget, rejectNulInBody } from '../middleware/reject-nul-bytes.js'; @@ -27,6 +28,7 @@ import { CORE_CAPABILITIES } from './capabilities-descriptor.js'; import { verifyCapabilitiesDescriptor } from './verify-capabilities.js'; import { SearchBodySchema } from '../schemas/memories.js'; import type { CoreRuntime } from './runtime-container.js'; +import { getCloudTraceHealthSnapshotAsync, toCloudTracePublicHealth } from '../services/cloud-trace-sync.js'; /** Default JSON-body cap for non-document routers. */ const DEFAULT_JSON_BODY_LIMIT = '1mb'; @@ -80,7 +82,14 @@ export function createApp(runtime: CoreRuntime): ReturnType { // expected-key buffer is captured a single time (timingSafeEqual // wants matching-length buffers; cheap but allocation-stable). // `/health` and `/openapi.json` stay outside this scope. - const auth = requireBearer(runtime.config.coreApiKey); + const authBundle = createAuthMiddleware({ + coreApiKey: runtime.config.coreApiKey, + cloudJwt: runtime.config.cloudJwt, + }); + const auth = authBundle.middleware; + if (authBundle.prefetchJwks) { + app.set('cloudJwtPrefetch', authBundle.prefetchJwks); + } // defense-in-depth: when TRUSTED_PROXY_MODE is on, the trusted // caller must restate the user it authenticated in the @@ -100,17 +109,17 @@ export function createApp(runtime: CoreRuntime): ReturnType { verifyCapabilitiesDescriptor(CORE_CAPABILITIES, memoryRouter, SearchBodySchema); app.use( '/v1/memories', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, assertUser, memoryRouter, ); app.use( '/v1/agents', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, assertUser, createAgentRouter(runtime.repos.trust), ); @@ -181,9 +190,9 @@ export function createApp(runtime: CoreRuntime): ReturnType { app.use( '/v1/entities', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, createEntityRouter({ pool: runtime.pool, memory: runtime.repos.memory, @@ -260,8 +269,19 @@ export function createApp(runtime: CoreRuntime): ReturnType { // `/health` is intentionally unversioned — it is an infrastructure // liveness probe (load balancers, Docker, Railway), not part of the // versioned application API. Versioned endpoints live under `/v1/*`. - app.get('/health', (_req, res) => { - res.json({ status: 'ok' }); + app.get('/health', async (_req, res) => { + const cloudTraceSync = runtime.config.cloudTraceSync?.enabled + ? toCloudTracePublicHealth( + await getCloudTraceHealthSnapshotAsync(runtime.pool, runtime.config.cloudTraceSync), + ) + : undefined; + const status = cloudTraceSync?.status === 'degraded' || cloudTraceSync?.status === 'paused' + ? 'degraded' + : 'ok'; + res.json({ + status, + ...(cloudTraceSync ? { cloud_trace_sync: cloudTraceSync } : {}), + }); }); // `GET /openapi.json` serves the committed OpenAPI spec. Like diff --git a/packages/core/src/app/runtime-container.ts b/packages/core/src/app/runtime-container.ts index 5adba5f..41b7791 100644 --- a/packages/core/src/app/runtime-container.ts +++ b/packages/core/src/app/runtime-container.ts @@ -167,6 +167,8 @@ export interface CoreRuntimeConfig { eventChainPackagingEnabled: boolean; /** Reflect channel gate (BEAM-0.85 Phase 1, Task 1.9). */ reflectEnabled: boolean; + /** Opt-in OSS Core → Cloud trace upload profile. Null when disabled. */ + cloudTraceSync: import('../config.js').CloudTraceSyncConfig | null; /** Top-K reflections to fetch when reflect retrieval is enabled. */ reflectRetrievalTopK: number; /** Anthropic model used by the reflect worker. */ diff --git a/packages/core/src/cloud/__tests__/heartbeat-client.test.ts b/packages/core/src/cloud/__tests__/heartbeat-client.test.ts new file mode 100644 index 0000000..9a1ae16 --- /dev/null +++ b/packages/core/src/cloud/__tests__/heartbeat-client.test.ts @@ -0,0 +1,50 @@ +/** + * Heartbeat client unit tests. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { sendRuntimeHeartbeat } from '../heartbeat-client.js'; + +describe('sendRuntimeHeartbeat', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns ok on 200', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ ok: true, status: 200 }), + ); + const result = await sendRuntimeHeartbeat({ + apiUrl: 'https://api.test', + apiKey: 'amc_test_key', + payload: { + core_instance_id: 'inst-1', + core_version: '1.0.0', + connector_version: '1.0.0', + capabilities: ['memory.read'], + }, + }); + expect(result.ok).toBe(true); + expect(result.errorCode).toBeNull(); + }); + + it('maps auth failures', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ ok: false, status: 401 }), + ); + const result = await sendRuntimeHeartbeat({ + apiUrl: 'https://api.test', + apiKey: 'amc_test_key', + payload: { + core_instance_id: 'inst-1', + core_version: '1.0.0', + connector_version: '1.0.0', + capabilities: [], + }, + }); + expect(result.ok).toBe(false); + expect(result.errorCode).toBe('auth_401'); + }); +}); diff --git a/packages/core/src/cloud/cloud-api-client.ts b/packages/core/src/cloud/cloud-api-client.ts new file mode 100644 index 0000000..5ea5425 --- /dev/null +++ b/packages/core/src/cloud/cloud-api-client.ts @@ -0,0 +1,25 @@ +/** + * Shared outbound POST helper for Cloud API calls (traces, heartbeat). + */ + +export interface CloudApiPostOptions { + apiUrl: string; + apiKey: string; + path: string; + body: unknown; + timeoutMs?: number; +} + +export async function cloudApiPost(options: CloudApiPostOptions): Promise { + const base = options.apiUrl.replace(/\/+$/, ''); + const path = options.path.startsWith('/') ? options.path : `/${options.path}`; + return fetch(`${base}${path}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${options.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(options.body), + signal: AbortSignal.timeout(options.timeoutMs ?? 30_000), + }); +} diff --git a/packages/core/src/cloud/env.ts b/packages/core/src/cloud/env.ts new file mode 100644 index 0000000..8d074ae --- /dev/null +++ b/packages/core/src/cloud/env.ts @@ -0,0 +1,25 @@ +/** + * Small env helpers for Cloud config parsers (isolated from monolithic config.ts). + */ + +export function optionalEnv(name: string): string | undefined { + return process.env[name] || undefined; +} + +export function parsePositiveIntEnv(name: string, fallback: number): number { + const raw = optionalEnv(name); + if (!raw) return fallback; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + +export function parseStrictBoolEnv(name: string, fallback: boolean): boolean { + const raw = optionalEnv(name); + if (raw === undefined) return fallback; + if (raw === 'true') return true; + if (raw === 'false') return false; + throw new Error(`${name} must be 'true' or 'false' (got '${raw}')`); +} diff --git a/packages/core/src/cloud/heartbeat-client.ts b/packages/core/src/cloud/heartbeat-client.ts new file mode 100644 index 0000000..15a6ab8 --- /dev/null +++ b/packages/core/src/cloud/heartbeat-client.ts @@ -0,0 +1,41 @@ +/** + * Outbound runtime heartbeat to Cloud (amc_ auth + core_instance_id). + */ + +import { cloudApiPost } from './cloud-api-client.js'; +import type { CloudRuntimeHeartbeatPayload } from './types.js'; + +export interface SendRuntimeHeartbeatInput { + apiUrl: string; + apiKey: string; + payload: CloudRuntimeHeartbeatPayload; +} + +export interface SendRuntimeHeartbeatResult { + ok: boolean; + status: number; + errorCode: string | null; +} + +export async function sendRuntimeHeartbeat( + input: SendRuntimeHeartbeatInput, +): Promise { + try { + const response = await cloudApiPost({ + apiUrl: input.apiUrl, + apiKey: input.apiKey, + path: '/v1/runtimes/heartbeat', + body: input.payload, + }); + if (response.ok) { + return { ok: true, status: response.status, errorCode: null }; + } + return { + ok: false, + status: response.status, + errorCode: response.status === 401 || response.status === 403 ? `auth_${response.status}` : `http_${response.status}`, + }; + } catch { + return { ok: false, status: 0, errorCode: 'network_error' }; + } +} diff --git a/packages/core/src/cloud/instance-id.ts b/packages/core/src/cloud/instance-id.ts new file mode 100644 index 0000000..ac048c6 --- /dev/null +++ b/packages/core/src/cloud/instance-id.ts @@ -0,0 +1,31 @@ +/** + * Stable Core installation id for Cloud runtime identity and trace envelopes. + */ + +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const DEFAULT_STATE_DIR = join(homedir(), '.atomicmemory', 'state'); + +function resolveInstanceIdPath(): string { + const stateDir = process.env.CORE_STATE_DIR?.trim() || DEFAULT_STATE_DIR; + return join(stateDir, 'core-instance-id'); +} + +export async function resolveCoreInstanceId(configuredId: string): Promise { + if (configuredId.trim()) return configuredId.trim(); + const instanceIdPath = resolveInstanceIdPath(); + try { + const existing = await readFile(instanceIdPath, 'utf8'); + const trimmed = existing.trim(); + if (trimmed) return trimmed; + } catch { + // first boot — generate below + } + const generated = randomUUID(); + await mkdir(dirname(instanceIdPath), { recursive: true }); + await writeFile(instanceIdPath, `${generated}\n`, 'utf8'); + return generated; +} diff --git a/packages/core/src/cloud/jwt-config.ts b/packages/core/src/cloud/jwt-config.ts new file mode 100644 index 0000000..694b4fb --- /dev/null +++ b/packages/core/src/cloud/jwt-config.ts @@ -0,0 +1,66 @@ +/** + * Parse Cloud JWT verification env (inbound console/SDK tokens). + */ + +import type { CloudJwtConfig } from './types.js'; +import { optionalEnv, parseStrictBoolEnv } from './env.js'; + +function requireCloudJwtFields( + jwksUrl: string | undefined, + issuer: string | undefined, + audience: string | undefined, +): asserts jwksUrl is string { + if (jwksUrl && issuer && audience) return; + throw new Error( + 'CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, and CLOUD_JWT_AUDIENCE must all be set together when enabling Cloud JWT verification', + ); +} + +function buildCloudJwtConfig( + jwksUrl: string, + issuer: string, + audience: string, + projectId: string | undefined, + staticKeyFallbackRaw: string | undefined, +): CloudJwtConfig { + const staticKeyFallbackEnabled = + staticKeyFallbackRaw === undefined + ? false + : parseStrictBoolEnv('CLOUD_JWT_STATIC_KEY_FALLBACK', false); + const legacyDefaultMemoryUserId = + optionalEnv('CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID')?.trim() || null; + return { + jwksUrl, + issuer, + audience, + projectId: projectId?.trim() || null, + staticKeyFallbackEnabled, + legacyDefaultMemoryUserId, + }; +} + +export function parseCloudJwtConfig(): CloudJwtConfig | null { + const jwksUrl = optionalEnv('CLOUD_JWKS_URL'); + const issuer = optionalEnv('CLOUD_JWT_ISSUER'); + const audience = optionalEnv('CLOUD_JWT_AUDIENCE'); + const projectId = optionalEnv('CLOUD_PROJECT_ID'); + const staticKeyFallbackRaw = optionalEnv('CLOUD_JWT_STATIC_KEY_FALLBACK'); + if ( + jwksUrl === undefined && + issuer === undefined && + audience === undefined && + projectId === undefined && + staticKeyFallbackRaw === undefined + ) { + return null; + } + + requireCloudJwtFields(jwksUrl, issuer, audience); + try { + void new URL(jwksUrl); + } catch { + throw new Error(`CLOUD_JWKS_URL must be a valid URL, got '${jwksUrl}'`); + } + + return buildCloudJwtConfig(jwksUrl, issuer!, audience!, projectId, staticKeyFallbackRaw); +} diff --git a/packages/core/src/cloud/trace-sync-config.ts b/packages/core/src/cloud/trace-sync-config.ts new file mode 100644 index 0000000..40f0375 --- /dev/null +++ b/packages/core/src/cloud/trace-sync-config.ts @@ -0,0 +1,64 @@ +/** + * Parse Cloud trace sync env (outbound amc_ uploads). + */ + +import type { CloudTraceSyncConfig } from './types.js'; +import { optionalEnv, parsePositiveIntEnv } from './env.js'; + +function requireCloudTraceSyncCredentials( + apiUrl: string | undefined, + apiKey: string | undefined, +): asserts apiUrl is string { + if (apiUrl && apiKey) return; + throw new Error( + 'Cloud trace sync requires ATOMICMEMORY_API_URL and ATOMICMEMORY_API_KEY when CLOUD_TRACE_SYNC_ENABLED=true', + ); +} + +function buildCloudTraceSyncConfig( + apiUrl: string, + apiKey: string, + instanceId: string | undefined, +): CloudTraceSyncConfig { + if (!apiKey.startsWith('amc_')) { + throw new Error('ATOMICMEMORY_API_KEY must be a project-scoped amc_ Cloud API key'); + } + + return { + enabled: true, + apiUrl: apiUrl.replace(/\/+$/, ''), + apiKey, + instanceId: instanceId?.trim() || '', + batchSize: parsePositiveIntEnv('CLOUD_TRACE_SYNC_BATCH_SIZE', 100), + maxAttempts: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_ATTEMPTS', 8), + baseRetryMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_BASE_RETRY_MS', 1_000), + maxRetryMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_RETRY_MS', 300_000), + pollIntervalMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_POLL_INTERVAL_MS', 5_000), + sentRetentionMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_SENT_RETENTION_MS', 86_400_000), + shutdownDrainMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_SHUTDOWN_DRAIN_MS', 10_000), + claimStaleMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_CLAIM_STALE_MS', 300_000), + deadLetterRetentionMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_DEAD_LETTER_RETENTION_MS', 604_800_000), + maxPending: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_PENDING', 10_000), + }; +} + +export function parseCloudTraceSyncConfig(): CloudTraceSyncConfig | null { + const enabledRaw = optionalEnv('CLOUD_TRACE_SYNC_ENABLED'); + const apiUrl = optionalEnv('ATOMICMEMORY_API_URL'); + const apiKey = optionalEnv('ATOMICMEMORY_API_KEY'); + const instanceId = optionalEnv('CORE_INSTANCE_ID'); + const partial = Boolean(enabledRaw || apiUrl || apiKey || instanceId); + + if (!partial) return null; + if (enabledRaw === 'false') return null; + + const enabled = enabledRaw === 'true'; + if (!enabled) { + throw new Error( + 'Partial Cloud trace sync configuration detected. Set CLOUD_TRACE_SYNC_ENABLED=true with ATOMICMEMORY_API_URL and ATOMICMEMORY_API_KEY, or remove all Cloud trace sync env vars.', + ); + } + + requireCloudTraceSyncCredentials(apiUrl, apiKey); + return buildCloudTraceSyncConfig(apiUrl, apiKey!, instanceId); +} diff --git a/packages/core/src/cloud/types.ts b/packages/core/src/cloud/types.ts new file mode 100644 index 0000000..7d537bf --- /dev/null +++ b/packages/core/src/cloud/types.ts @@ -0,0 +1,41 @@ +/** + * Cloud-connected local configuration types (trace sync + JWT verify). + */ + +export interface CloudTraceSyncConfig { + enabled: boolean; + apiUrl: string; + apiKey: string; + instanceId: string; + batchSize: number; + maxAttempts: number; + baseRetryMs: number; + maxRetryMs: number; + pollIntervalMs: number; + sentRetentionMs: number; + shutdownDrainMs: number; + claimStaleMs: number; + deadLetterRetentionMs: number; + maxPending: number; +} + +export interface CloudJwtConfig { + jwksUrl: string; + issuer: string; + audience: string; + /** + * Optional bound Cloud project for connected-local JWT verification. + * When null, Core trusts the token's own `project_id` claim. + */ + projectId: string | null; + staticKeyFallbackEnabled: boolean; + legacyDefaultMemoryUserId: string | null; +} + +export interface CloudRuntimeHeartbeatPayload { + core_instance_id: string; + core_version: string; + connector_version: string; + capabilities: string[]; + local_url?: string; +} diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index b36e109..b176428 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -22,6 +22,11 @@ import { parseFilecoinProviderConfig, type FilecoinProviderConfig, } from './storage/providers/filecoin/config.js'; +import { parseCloudJwtConfig } from './cloud/jwt-config.js'; +import { parseCloudTraceSyncConfig } from './cloud/trace-sync-config.js'; +import { optionalEnv, parsePositiveIntEnv, parseStrictBoolEnv } from './cloud/env.js'; +import type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; +export type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; export type EmbeddingProviderName = 'openai' | 'ollama' | 'openai-compatible' | 'transformers' | 'voyage'; export type LLMProviderName = EmbeddingProviderName | 'groq' | 'anthropic' | 'google-genai' | 'claude-code' | 'codex'; @@ -75,6 +80,12 @@ export interface RuntimeConfig { * restarting the server with a new value. */ coreApiKey: string; + /** + * Optional Cloud-issued JWT verification (JWKS). When set, `/v1/*` accepts + * either `CORE_API_KEY` or a valid RS256 JWT with `aud` matching + * `CLOUD_JWT_AUDIENCE`. All three env vars must be set together. + */ + cloudJwt: CloudJwtConfig | null; /** * Trusted-proxy identity guard. When `true`, every * user-scoped request carrying a `user_id` MUST also carry the same @@ -168,6 +179,8 @@ export interface RuntimeConfig { retrievalTraceEnabled: boolean; ingestTraceDir: string; ingestTraceEnabled: boolean; + /** Opt-in OSS Core → Cloud trace upload profile. Null when disabled. */ + cloudTraceSync: CloudTraceSyncConfig | null; extractionCacheEnabled: boolean; extractionCacheDir: string; embeddingCacheEnabled: boolean; @@ -225,6 +238,9 @@ export interface RuntimeConfig { temporalQueryConstraintBoost: number; deferredAudnEnabled: boolean; deferredAudnBatchSize: number; + deferredAudnConcurrency: number; + deferredAudnAutoReconcile: boolean; + deferredAudnReconcileIntervalMs: number; compositeGroupingEnabled: boolean; compositeMinClusterSize: number; compositeMaxClusterSize: number; @@ -720,10 +736,6 @@ function requireEnv(name: string): string { return value; } -function optionalEnv(name: string): string | undefined { - return process.env[name] || undefined; -} - function parseEmbeddingProvider( value: string | undefined, fallback: EmbeddingProviderName, @@ -802,31 +814,6 @@ function parseRegexEnv(name: string): string | undefined { } } -function parsePositiveIntEnv(name: string, fallback: number): number { - const raw = optionalEnv(name); - if (!raw) return fallback; - const parsed = parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed < 1) { - throw new Error(`${name} must be a positive integer`); - } - return parsed; -} - -/** - * Strict boolean env parser. Unlike the widespread `(env ?? 'false') === - * 'true'` idiom, this rejects any value other than the literal `'true'` / - * `'false'` rather than silently treating a typo (e.g. `TRUSTED_PROXY_MODE= - * ture`) as `false`. Used for security-relevant gates where a silent - * mis-parse would disable a guard. Fails closed at startup. - */ -function parseStrictBoolEnv(name: string, fallback: boolean): boolean { - const raw = optionalEnv(name); - if (raw === undefined) return fallback; - if (raw === 'true') return true; - if (raw === 'false') return false; - throw new Error(`${name} must be 'true' or 'false' (got '${raw}')`); -} - function parseVectorBackend(value: string | undefined): VectorBackendName { if (!value) return 'pgvector'; if (value === 'pgvector' || value === 'ruvector-mock' || value === 'zvec-mock') return value; @@ -1251,6 +1238,7 @@ const embeddingProvider = parseEmbeddingProvider(optionalEnv('EMBEDDING_PROVIDER const llmProvider = parseLlmProvider(optionalEnv('LLM_PROVIDER'), 'openai'); const rawStorageDeploymentEnv = parseRawStorageDeploymentEnv(optionalEnv('RAW_STORAGE_DEPLOYMENT_ENV')); const trustedProxyMode = resolveTrustedProxyMode(rawStorageDeploymentEnv); +const cloudJwt = parseCloudJwtConfig(); const retrievalProfile = parseRetrievalProfile(optionalEnv('RETRIEVAL_PROFILE')); const retrievalProfileSettings = getRetrievalProfile(retrievalProfile); const DEFAULT_SIMILARITY_THRESHOLD = 0.3; @@ -1300,6 +1288,7 @@ export const config: RuntimeConfig = { databaseUrl: requireEnv('DATABASE_URL'), openaiApiKey, coreApiKey: requireEnv('CORE_API_KEY'), + cloudJwt, trustedProxyMode, coreAdminApiKey: optionalEnv('CORE_ADMIN_API_KEY'), coreTestScopeAllowPattern: parseRegexEnv('CORE_TEST_SCOPE_ALLOW_PATTERN'), @@ -1368,6 +1357,7 @@ export const config: RuntimeConfig = { retrievalTraceEnabled: (optionalEnv('RETRIEVAL_TRACE_ENABLED') ?? 'false') === 'true', ingestTraceDir: optionalEnv('INGEST_TRACE_DIR') ?? './.traces/ingest', ingestTraceEnabled: (optionalEnv('INGEST_TRACE_ENABLED') ?? 'false') === 'true', + cloudTraceSync: parseCloudTraceSyncConfig(), extractionCacheEnabled: (optionalEnv('EXTRACTION_CACHE_ENABLED') ?? 'false') === 'true', extractionCacheDir: optionalEnv('EXTRACTION_CACHE_DIR') ?? './.eval-cache', embeddingCacheEnabled: (optionalEnv('EMBEDDING_CACHE_ENABLED') ?? 'false') === 'true', @@ -1425,6 +1415,9 @@ export const config: RuntimeConfig = { temporalQueryConstraintBoost: parseFloat(optionalEnv('TEMPORAL_QUERY_CONSTRAINT_BOOST') ?? '2'), deferredAudnEnabled: (optionalEnv('DEFERRED_AUDN_ENABLED') ?? 'false') === 'true', deferredAudnBatchSize: parseInt(optionalEnv('DEFERRED_AUDN_BATCH_SIZE') ?? '20', 10), + deferredAudnConcurrency: parseInt(optionalEnv('DEFERRED_AUDN_CONCURRENCY') ?? '8', 10), + deferredAudnAutoReconcile: (optionalEnv('DEFERRED_AUDN_AUTO_RECONCILE') ?? 'false') === 'true', + deferredAudnReconcileIntervalMs: parseInt(optionalEnv('DEFERRED_AUDN_RECONCILE_INTERVAL_MS') ?? '5000', 10), compositeGroupingEnabled: (optionalEnv('COMPOSITE_GROUPING_ENABLED') ?? 'true') === 'true', compositeMinClusterSize: parseInt(optionalEnv('COMPOSITE_MIN_CLUSTER_SIZE') ?? '2', 10), compositeMaxClusterSize: parseInt(optionalEnv('COMPOSITE_MAX_CLUSTER_SIZE') ?? '3', 10), @@ -1665,6 +1658,8 @@ export const INTERNAL_POLICY_CONFIG_FIELDS = [ 'fastAudnEnabled', 'fastAudnDuplicateThreshold', // Observation / deferred 'observationNetworkEnabled', 'deferredAudnEnabled', 'deferredAudnBatchSize', + 'deferredAudnConcurrency', + 'deferredAudnAutoReconcile', 'deferredAudnReconcileIntervalMs', // Composite grouping 'compositeGroupingEnabled', 'compositeMinClusterSize', 'compositeMaxClusterSize', 'compositeSimilarityThreshold', diff --git a/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts b/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts new file mode 100644 index 0000000..27368f2 --- /dev/null +++ b/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts @@ -0,0 +1,75 @@ +/** + * Integration tests for Cloud trace outbox repository. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { pool } from '../pool.js'; +import { setupTestSchema } from './test-fixtures.js'; +import { + claimCloudTraceBatch, + countPendingCloudTraces, + enqueueCloudTraceOutbox, + markCloudTraceSent, +} from '../cloud-trace-outbox-repository.js'; +import { buildCloudTraceEnvelope } from '../../services/cloud-trace-envelope.js'; + +function sampleEnvelope(eventId: string = randomUUID()) { + return buildCloudTraceEnvelope({ + eventId, + coreInstanceId: 'core-test-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 1, + summary: { user_id: 'test-user', operation_detail: 'test ingest' }, + }); +} + +beforeAll(async () => { + await setupTestSchema(pool); +}); + +beforeEach(async () => { + await pool.query('DELETE FROM cloud_trace_outbox'); +}); + +afterAll(async () => { + await pool.end(); +}); + +describe('cloud trace outbox repository', () => { + it('enqueue and claim a pending row', async () => { + const envelope = sampleEnvelope(); + await enqueueCloudTraceOutbox(pool, envelope); + const claimed = await claimCloudTraceBatch(pool, 10, 300_000); + expect(claimed).toHaveLength(1); + expect(claimed[0]?.eventId).toBe(envelope.event_id); + const state = await pool.query( + `SELECT delivery_state FROM cloud_trace_outbox WHERE event_id = $1`, + [envelope.event_id], + ); + expect(state.rows[0]?.delivery_state).toBe('claimed'); + }); + + it('reclaims stale claimed rows after crash', async () => { + const envelope = sampleEnvelope(); + await enqueueCloudTraceOutbox(pool, envelope); + const first = await claimCloudTraceBatch(pool, 10, 300_000); + expect(first).toHaveLength(1); + + await pool.query( + `UPDATE cloud_trace_outbox + SET claimed_at = now() - interval '10 minutes' + WHERE event_id = $1`, + [envelope.event_id], + ); + + const reclaimed = await claimCloudTraceBatch(pool, 10, 60_000); + expect(reclaimed).toHaveLength(1); + expect(reclaimed[0]?.eventId).toBe(envelope.event_id); + + await markCloudTraceSent(pool, envelope.event_id); + expect(await countPendingCloudTraces(pool)).toBe(0); + }); +}); diff --git a/packages/core/src/db/__tests__/migration-backcompat.test.ts b/packages/core/src/db/__tests__/migration-backcompat.test.ts index 2be5450..397c922 100644 --- a/packages/core/src/db/__tests__/migration-backcompat.test.ts +++ b/packages/core/src/db/__tests__/migration-backcompat.test.ts @@ -32,7 +32,13 @@ const pool = useMigrationTestPool({ beforeEach, afterAll }); // no legacy table. Allowlisting it covers its PK, indexes, CHECK and the FK to // memories, since isAllowedNewIndex/isAllowedNewConstraint pass any object whose // table is a new allowed table. -const ALLOWED_NEW_TABLES = new Set(['pgmigrations', 'schema_version', 'entity_settings', 'entity_edges']); +const ALLOWED_NEW_TABLES = new Set([ + 'pgmigrations', + 'schema_version', + 'entity_settings', + 'entity_edges', + 'cloud_trace_outbox', +]); const ALLOWED_NEW_INDEXES = new Set([ // schema_version primary key is auto-generated by Postgres on `applied_at`. 'schema_version_pkey', diff --git a/packages/core/src/db/cloud-trace-outbox-repository.ts b/packages/core/src/db/cloud-trace-outbox-repository.ts new file mode 100644 index 0000000..1eb841e --- /dev/null +++ b/packages/core/src/db/cloud-trace-outbox-repository.ts @@ -0,0 +1,187 @@ +/** + * Durable Postgres outbox for Cloud trace upload. + */ + +import type pg from 'pg'; +import type { CloudTraceEnvelopeV2 } from '../services/cloud-trace-envelope.js'; + +export type CloudTraceDeliveryState = 'pending' | 'claimed' | 'sent' | 'dead_letter'; + +export interface CloudTraceOutboxRow { + eventId: string; + schemaVersion: number; + payload: CloudTraceEnvelopeV2; + deliveryState: CloudTraceDeliveryState; + attemptCount: number; + nextAttemptAt: Date; + lastErrorCode: string | null; + createdAt: Date; + claimedAt: Date | null; + sentAt: Date | null; + deadLetterAt: Date | null; +} + +export async function enqueueCloudTraceOutbox( + pool: pg.Pool, + envelope: CloudTraceEnvelopeV2, +): Promise { + await pool.query( + `INSERT INTO cloud_trace_outbox (event_id, schema_version, payload, delivery_state) + VALUES ($1, $2, $3::jsonb, 'pending') + ON CONFLICT (event_id) DO NOTHING`, + [envelope.event_id, envelope.schema_version, JSON.stringify(envelope)], + ); +} + +export async function claimCloudTraceBatch( + pool: pg.Pool, + limit: number, + staleAfterMs: number, +): Promise { + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const result = await client.query( + `SELECT event_id, schema_version, payload, delivery_state, attempt_count, + next_attempt_at, last_error_code, created_at, claimed_at, sent_at, dead_letter_at + FROM cloud_trace_outbox + WHERE ( + delivery_state = 'pending' + AND next_attempt_at <= now() + ) + OR ( + delivery_state = 'claimed' + AND claimed_at < now() - ($2::bigint * interval '1 millisecond') + ) + ORDER BY created_at ASC + FOR UPDATE SKIP LOCKED + LIMIT $1`, + [limit, staleAfterMs], + ); + const ids = result.rows.map((row) => row.event_id as string); + if (ids.length > 0) { + await client.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'claimed', claimed_at = now() + WHERE event_id = ANY($1::uuid[])`, + [ids], + ); + } + await client.query('COMMIT'); + return result.rows.map(mapRow); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} + +export async function markCloudTraceSent(pool: pg.Pool, eventId: string): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'sent', sent_at = now(), last_error_code = NULL + WHERE event_id = $1`, + [eventId], + ); +} + +export async function scheduleCloudTraceRetry( + pool: pg.Pool, + eventId: string, + errorCode: string, + nextAttemptAt: Date, +): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'pending', + attempt_count = attempt_count + 1, + next_attempt_at = $2, + last_error_code = $3, + claimed_at = NULL + WHERE event_id = $1`, + [eventId, nextAttemptAt, errorCode], + ); +} + +export async function countPendingCloudTraces(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT COUNT(*)::int AS count + FROM cloud_trace_outbox + WHERE delivery_state IN ('pending', 'claimed')`, + ); + return result.rows[0]?.count ?? 0; +} + +export async function countDeadLetterCloudTraces(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT COUNT(*)::int AS count + FROM cloud_trace_outbox + WHERE delivery_state = 'dead_letter'`, + ); + return result.rows[0]?.count ?? 0; +} + +export async function getOldestPendingCloudTraceAgeMs(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) * 1000 AS age_ms + FROM cloud_trace_outbox + WHERE delivery_state IN ('pending', 'claimed')`, + ); + const age = result.rows[0]?.age_ms; + if (age == null) return null; + const parsed = Number(age); + return Number.isFinite(parsed) ? Math.round(parsed) : null; +} + +export async function markCloudTraceDeadLetter( + pool: pg.Pool, + eventId: string, + errorCode: string, +): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'dead_letter', + dead_letter_at = now(), + last_error_code = $2, + claimed_at = NULL + WHERE event_id = $1`, + [eventId, errorCode], + ); +} + +export async function purgeSentCloudTraces(pool: pg.Pool, retentionMs: number): Promise { + await pool.query( + `DELETE FROM cloud_trace_outbox + WHERE delivery_state = 'sent' + AND sent_at IS NOT NULL + AND sent_at < now() - ($1::bigint * interval '1 millisecond')`, + [retentionMs], + ); +} + +export async function purgeDeadLetterCloudTraces(pool: pg.Pool, retentionMs: number): Promise { + await pool.query( + `DELETE FROM cloud_trace_outbox + WHERE delivery_state = 'dead_letter' + AND dead_letter_at IS NOT NULL + AND dead_letter_at < now() - ($1::bigint * interval '1 millisecond')`, + [retentionMs], + ); +} + +function mapRow(row: pg.QueryResultRow): CloudTraceOutboxRow { + return { + eventId: row.event_id, + schemaVersion: row.schema_version, + payload: row.payload as CloudTraceEnvelopeV2, + deliveryState: row.delivery_state, + attemptCount: row.attempt_count, + nextAttemptAt: row.next_attempt_at, + lastErrorCode: row.last_error_code, + createdAt: row.created_at, + claimedAt: row.claimed_at, + sentAt: row.sent_at, + deadLetterAt: row.dead_letter_at, + }; +} diff --git a/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql b/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql new file mode 100644 index 0000000..df2676f --- /dev/null +++ b/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql @@ -0,0 +1,25 @@ +-- Durable outbox for connected-local Cloud trace upload. + +CREATE TABLE IF NOT EXISTS cloud_trace_outbox ( + event_id UUID PRIMARY KEY, + schema_version INTEGER NOT NULL, + payload JSONB NOT NULL, + delivery_state TEXT NOT NULL CHECK ( + delivery_state IN ('pending', 'claimed', 'sent', 'dead_letter') + ), + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + claimed_at TIMESTAMPTZ, + sent_at TIMESTAMPTZ, + dead_letter_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_pending_idx + ON cloud_trace_outbox (delivery_state, next_attempt_at) + WHERE delivery_state IN ('pending', 'claimed'); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_sent_retention_idx + ON cloud_trace_outbox (sent_at) + WHERE delivery_state = 'sent'; diff --git a/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts b/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts new file mode 100644 index 0000000..65bc66a --- /dev/null +++ b/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts @@ -0,0 +1,20 @@ +/** + * Unit tests for Cloud JWT user binding helpers. + */ + +import { describe, expect, it } from 'vitest'; +import { resolveReconcileUserId } from '../cloud-jwt-user-binding.js'; + +describe('resolveReconcileUserId', () => { + it('uses body user_id when present', () => { + expect(resolveReconcileUserId('bob', 'alice')).toBe('bob'); + }); + + it('defaults to asserted JWT user when body user_id is absent', () => { + expect(resolveReconcileUserId(undefined, 'alice')).toBe('alice'); + }); + + it('returns undefined for static-key callers with no body user_id', () => { + expect(resolveReconcileUserId(undefined, null)).toBeUndefined(); + }); +}); diff --git a/packages/core/src/middleware/__tests__/dual-auth.test.ts b/packages/core/src/middleware/__tests__/dual-auth.test.ts new file mode 100644 index 0000000..966a522 --- /dev/null +++ b/packages/core/src/middleware/__tests__/dual-auth.test.ts @@ -0,0 +1,337 @@ +/** + * Unit tests for dual-auth middleware (static CORE_API_KEY + optional Cloud JWT). + */ + +import { exportJWK, generateKeyPair, SignJWT } from 'jose'; +import type { Request, Response } from 'express'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CloudJwtConfig } from '../../config.js'; +import { ASSERTED_USER_HEADER } from '../asserted-user.js'; +import { createAuthMiddleware } from '../dual-auth.js'; + +const CORE_KEY = 'test-shared-secret-do-not-leak'; +const JWKS_URL = 'https://cloud.test/.well-known/atomic-core/jwks.json'; +const ISSUER = 'https://api.test'; +const AUDIENCE = 'atomicmemory-core'; +const PROJECT_ID = 'proj_abc'; +const MEMORY_USER_ID = 'tenant-user-1'; + +function cloudJwtConfig(overrides: Partial = {}): CloudJwtConfig { + return { + jwksUrl: JWKS_URL, + issuer: ISSUER, + audience: AUDIENCE, + projectId: PROJECT_ID, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + ...overrides, + }; +} + +function buildRes(): { + res: Response; + statusCode: number; + body: unknown; +} { + const stub: { + res: Response; + status: (code: number) => Response; + json: (body: unknown) => Response; + statusCode: number; + body: unknown; + } = { statusCode: 0, body: undefined } as never; + stub.status = vi.fn((code: number) => { + stub.statusCode = code; + return stub.res; + }); + stub.json = vi.fn((body: unknown) => { + stub.body = body; + return stub.res; + }); + stub.res = { status: stub.status, json: stub.json } as unknown as Response; + return stub; +} + +async function signTestJwt(claims: Record): Promise { + const { privateKey } = await keyPair(); + return new SignJWT(claims) + .setProtectedHeader({ alg: 'RS256', kid: 'test-kid' }) + .setIssuer(ISSUER) + .setAudience(AUDIENCE) + .setSubject('api_key:key_test') + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey); +} + +let keyPairPromise: ReturnType | undefined; + +async function keyPair() { + keyPairPromise ??= generateKeyPair('RS256'); + return keyPairPromise; +} + +async function stubJwksFetch(): Promise { + const { publicKey } = await keyPair(); + const jwk = await exportJWK(publicKey); + jwk.kid = 'test-kid'; + jwk.alg = 'RS256'; + jwk.use = 'sig'; + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url === JWKS_URL) { + return new Response(JSON.stringify({ keys: [jwk] }), { status: 200 }); + } + throw new Error(`unexpected fetch: ${url}`); + }), + ); +} + +async function runHandler( + handler: ReturnType['middleware'], + req: Request, +): Promise<{ stub: ReturnType; next: ReturnType }> { + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { stub, next }; +} + +afterEach(() => { + vi.unstubAllGlobals(); + keyPairPromise = undefined; +}); + +describe('createAuthMiddleware', () => { + it('delegates to static bearer when cloudJwt is disabled', () => { + const { middleware: handler } = createAuthMiddleware({ coreApiKey: CORE_KEY, cloudJwt: null }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('accepts a valid Cloud JWT and injects memory-user headers', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(req.headers[ASSERTED_USER_HEADER.toLowerCase()]).toBe(MEMORY_USER_ID); + expect(stub.statusCode).toBe(0); + }); + + it('rejects request user_id different from memory_user_id with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: 'other-user' }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect((stub.body as { error_code: string }).error_code).toBe('forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects token project_id different from configured project with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: 'proj_other', + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect(next).not.toHaveBeenCalled(); + }); + + it('trusts the token project_id when no project is bound (single-key local)', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ projectId: null }), + }); + const token = await signTestJwt({ + project_id: 'proj_inferred_from_token', + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('rejects body user_id different from memory_user_id with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + body: { user_id: 'other-user' }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect((stub.body as { error_code: string }).error_code).toBe('forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('accepts legacy Cloud mint without memory_user_id when legacy default is configured', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ legacyDefaultMemoryUserId: MEMORY_USER_ID }), + }); + const token = await signTestJwt({ project_id: PROJECT_ID }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('rejects JWT missing memory_user_id with 401', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ project_id: PROJECT_ID }); + const req = { headers: { authorization: `Bearer ${token}` } } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects an expired Cloud JWT', async () => { + await stubJwksFetch(); + const { privateKey } = await keyPair(); + const token = await new SignJWT({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'test-kid' }) + .setIssuer(ISSUER) + .setAudience(AUDIENCE) + .setSubject('api_key:key_test') + .setIssuedAt(Math.floor(Date.now() / 1000) - 600) + .setExpirationTime(Math.floor(Date.now() / 1000) - 300) + .sign(privateKey); + + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const req = { headers: { authorization: `Bearer ${token}` } } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects non-JWT bearer when static fallback is disabled', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ staticKeyFallbackEnabled: false }), + }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('falls back to CORE_API_KEY only when static fallback is explicitly enabled', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ staticKeyFallbackEnabled: true }), + }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('prefetch marks JWKS ready and cached verify survives fetch outage', async () => { + await stubJwksFetch(); + const bundle = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + expect(await bundle.prefetchJwks!()).toBe(true); + expect(bundle.isJwksReady!()).toBe(true); + + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const warmReq = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + await runHandler(bundle.middleware, warmReq); + + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('jwks unavailable'); + }), + ); + + const { stub, next } = await runHandler(bundle.middleware, warmReq); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); +}); diff --git a/packages/core/src/middleware/cloud-jwt-user-binding.ts b/packages/core/src/middleware/cloud-jwt-user-binding.ts new file mode 100644 index 0000000..82fa496 --- /dev/null +++ b/packages/core/src/middleware/cloud-jwt-user-binding.ts @@ -0,0 +1,89 @@ +/** + * Cloud JWT user binding helpers shared by dual-auth and routers that + * parse JSON after auth (e.g. `/v1/documents`). + */ + +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { ASSERTED_USER_HEADER } from './asserted-user.js'; + +const ASSERTED_USER_HEADER_LOWER = ASSERTED_USER_HEADER.toLowerCase(); + +/** Read wire `user_id` from parsed body, query, or direct-storage header. */ +function readRequestUserId(req: Request): string | null { + const fromBody = pickUserId(req.body); + if (fromBody !== null) return fromBody; + const fromQuery = pickUserId(req.query); + if (fromQuery !== null) return fromQuery; + const raw = req.headers['x-atomicmemory-user-id']; + const value = Array.isArray(raw) ? raw[0] : raw; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function pickUserId(source: unknown): string | null { + if (typeof source !== 'object' || source === null) return null; + const value = (source as Record)['user_id']; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function readAssertedUserHeader(req: Request): string | null { + const raw = req.headers[ASSERTED_USER_HEADER_LOWER]; + const value = Array.isArray(raw) ? raw[0] : raw; + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function respondForbidden(res: Response, reason: string): void { + res.status(403).json({ error_code: 'forbidden', error: reason }); +} + +/** + * When a JWT authenticated the request, wire `user_id` must match the + * verified `memory_user_id` injected as {@link ASSERTED_USER_HEADER}. + */ +export function enforceMemoryUserBinding( + req: Request, + memoryUserId: string, + res: Response, +): boolean { + const requestUserId = readRequestUserId(req); + if (requestUserId !== null && requestUserId !== memoryUserId) { + respondForbidden(res, 'request user_id does not match token memory_user_id'); + return false; + } + return true; +} + +/** Re-check binding after a router-owned body parser runs post-auth. */ +function enforceCloudJwtUserBindingFromHeader(req: Request, res: Response): boolean { + const assertedUser = readAssertedUserHeader(req); + if (assertedUser === null) { + return true; + } + return enforceMemoryUserBinding(req, assertedUser, res); +} + +/** Express guard for routers that parse JSON after global auth. */ +export function cloudJwtUserBindingGuard(): RequestHandler { + return (req: Request, res: Response, next: NextFunction): void => { + if (enforceCloudJwtUserBindingFromHeader(req, res)) { + next(); + } + }; +} + +/** JWT-authenticated reconcile and similar routes: never run global scope. */ +export function readAssertedUserId(req: Request): string | null { + return readAssertedUserHeader(req); +} + +/** Prefer explicit body user_id; otherwise scope JWT callers to asserted identity. */ +export function resolveReconcileUserId( + bodyUserId: string | undefined, + assertedUserId: string | null, +): string | undefined { + if (bodyUserId) { + return bodyUserId; + } + return assertedUserId ?? undefined; +} diff --git a/packages/core/src/middleware/dual-auth.ts b/packages/core/src/middleware/dual-auth.ts new file mode 100644 index 0000000..83ee4c7 --- /dev/null +++ b/packages/core/src/middleware/dual-auth.ts @@ -0,0 +1,166 @@ +/** + * Dual authentication for SDK-facing `/v1/*` routes. + * + * Accepts either the deployment-wide `CORE_API_KEY` (when explicitly enabled + * as a fallback) or a Cloud-issued RS256 JWT verified against a remote JWKS. + * On JWT success the verified `memory_user_id` / `project_id` claims are + * injected as asserted-user headers so downstream scope enforcement sees a + * stable end-user identity. + */ + +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { createRemoteJWKSet, jwtVerify, type JWTVerifyGetKey } from 'jose'; +import type { CloudJwtConfig } from '../config.js'; +import { ASSERTED_USER_HEADER } from './asserted-user.js'; +import { enforceMemoryUserBinding } from './cloud-jwt-user-binding.js'; +import { readBearerToken, requireBearer, respondUnauthenticated } from './require-bearer.js'; +const JWKS_PREFETCH_TIMEOUT_MS = 5_000; + +export interface DualAuthOptions { + coreApiKey: string; + cloudJwt: CloudJwtConfig | null; +} + +export interface AuthMiddlewareBundle { + middleware: RequestHandler; + prefetchJwks?: () => Promise; + isJwksReady?: () => boolean; +} + +export interface CloudJwtVerifier { + config: CloudJwtConfig; + jwks: JWTVerifyGetKey; + prefetch: () => Promise; + isReady: () => boolean; + markReady: () => void; +} + +function looksLikeJwt(token: string): boolean { + const parts = token.split('.'); + return parts.length === 3 && parts.every((part) => part.length > 0); +} + +function respondForbidden(res: Response, reason: string): void { + res.status(403).json({ error_code: 'forbidden', error: reason }); +} + +function injectAssertedIdentity(req: Request, memoryUserId: string): void { + req.headers[ASSERTED_USER_HEADER.toLowerCase()] = memoryUserId; +} + +/** + * True when a project binding is configured and the token's `project_id` + * claim does not match it. A null binding means single-tenant local mode: + * Core trusts whatever project the Cloud-minted token carries. + */ +function tokenProjectMismatch(boundProjectId: string | null, tokenProjectId: string): boolean { + return boundProjectId !== null && tokenProjectId !== boundProjectId; +} + +function readRequiredStringClaim(payload: Record, claim: string): string | null { + const value = payload[claim]; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +/** + * Create a shared JWKS verifier for Cloud JWT auth. The remote JWK set is + * constructed once and reused across requests. + */ +function createCloudJwtVerifier(cloudJwt: CloudJwtConfig): CloudJwtVerifier { + const jwks = createRemoteJWKSet(new URL(cloudJwt.jwksUrl)); + let jwksReady = false; + + return { + config: cloudJwt, + jwks, + isReady: () => jwksReady, + markReady: () => { + jwksReady = true; + }, + async prefetch(): Promise { + try { + const response = await fetch(cloudJwt.jwksUrl, { + signal: AbortSignal.timeout(JWKS_PREFETCH_TIMEOUT_MS), + }); + if (response.ok) { + jwksReady = true; + } + } catch { + // Degraded until a verify path loads keys into the jose cache. + } + return jwksReady; + }, + }; +} + +/** + * Build auth middleware: static bearer and/or Cloud JWT (when configured). + */ +export function createAuthMiddleware(options: DualAuthOptions): AuthMiddlewareBundle { + const staticBearer = requireBearer(options.coreApiKey); + if (!options.cloudJwt) { + return { middleware: staticBearer }; + } + + const verifier = createCloudJwtVerifier(options.cloudJwt); + + const middleware: RequestHandler = (req: Request, res: Response, next: NextFunction): void => { + const token = readBearerToken(req); + if (token === null) { + respondUnauthenticated(res, 'missing or malformed Authorization header'); + return; + } + + if (!looksLikeJwt(token)) { + if (options.cloudJwt!.staticKeyFallbackEnabled) { + staticBearer(req, res, next); + return; + } + respondUnauthenticated(res, 'invalid api key'); + return; + } + + void (async () => { + try { + const { payload } = await jwtVerify(token, verifier.jwks, { + algorithms: ['RS256'], + issuer: options.cloudJwt!.issuer, + audience: options.cloudJwt!.audience, + }); + verifier.markReady(); + + const principalId = typeof payload.sub === 'string' ? payload.sub : null; + let memoryUserId = readRequiredStringClaim(payload as Record, 'memory_user_id'); + if (!memoryUserId) { + memoryUserId = options.cloudJwt!.legacyDefaultMemoryUserId; + } + const projectId = readRequiredStringClaim(payload as Record, 'project_id'); + + if (!principalId || !memoryUserId || !projectId) { + respondUnauthenticated(res, 'invalid api key'); + return; + } + + if (tokenProjectMismatch(options.cloudJwt!.projectId, projectId)) { + respondForbidden(res, 'token project_id does not match configured Cloud project'); + return; + } + + if (!enforceMemoryUserBinding(req, memoryUserId, res)) { + return; + } + + injectAssertedIdentity(req, memoryUserId); + next(); + } catch { + respondUnauthenticated(res, 'invalid api key'); + } + })(); + }; + + return { + middleware, + prefetchJwks: () => verifier.prefetch(), + isJwksReady: () => verifier.isReady(), + }; +} diff --git a/packages/core/src/middleware/require-bearer.ts b/packages/core/src/middleware/require-bearer.ts index dbff639..b431909 100644 --- a/packages/core/src/middleware/require-bearer.ts +++ b/packages/core/src/middleware/require-bearer.ts @@ -22,6 +22,19 @@ import type { Request, RequestHandler, Response, NextFunction } from 'express'; const BEARER_PREFIX = 'Bearer '; +/** Read the bearer token from `Authorization`, or null when missing/malformed. */ +export function readBearerToken(req: Request): string | null { + const raw = req.headers['authorization']; + if (typeof raw !== 'string' || !raw.startsWith(BEARER_PREFIX)) return null; + const token = raw.slice(BEARER_PREFIX.length).trim(); + return token.length > 0 ? token : null; +} + +/** Standard 401 envelope for bearer/JWT auth failures. */ +export function respondUnauthenticated(res: Response, reason: string): void { + res.status(401).json({ error_code: 'unauthenticated', error: reason }); +} + /** * Build a middleware that admits a request only when its * `Authorization` header carries the configured shared key. The @@ -35,7 +48,7 @@ export function requireBearer(expectedApiKey: string): RequestHandler { } const expectedBuffer = Buffer.from(expectedApiKey, 'utf8'); return (req: Request, res: Response, next: NextFunction): void => { - const headerValue = readAuthorizationHeader(req); + const headerValue = readBearerToken(req); if (headerValue === null) { respondUnauthenticated(res, 'missing or malformed Authorization header'); return; @@ -52,14 +65,3 @@ export function requireBearer(expectedApiKey: string): RequestHandler { next(); }; } - -function readAuthorizationHeader(req: Request): string | null { - const raw = req.headers['authorization']; - if (typeof raw !== 'string' || !raw.startsWith(BEARER_PREFIX)) return null; - const token = raw.slice(BEARER_PREFIX.length).trim(); - return token.length > 0 ? token : null; -} - -function respondUnauthenticated(res: Response, reason: string): void { - res.status(401).json({ error_code: 'unauthenticated', error: reason }); -} diff --git a/packages/core/src/routes/documents.ts b/packages/core/src/routes/documents.ts index f128354..9ee71bd 100644 --- a/packages/core/src/routes/documents.ts +++ b/packages/core/src/routes/documents.ts @@ -35,6 +35,7 @@ import { handleRouteError } from './route-errors.js'; import { validateBody, validateQuery, validateParams } from '../middleware/validate.js'; import { validateResponse } from '../middleware/validate-response.js'; import { rejectNulInBody } from '../middleware/reject-nul-bytes.js'; +import { cloudJwtUserBindingGuard } from '../middleware/cloud-jwt-user-binding.js'; import { DOCUMENT_RESPONSE_SCHEMAS } from './response-schema-map.js'; import { DocumentByIdQuerySchema, @@ -209,6 +210,7 @@ export function createDocumentRouter( // raw-upload route's Buffer body is skipped by the guard. router.use(express.json({ limit: ROUTER_JSON_BODY_LIMIT })); router.use(rejectNulInBody); + router.use(cloudJwtUserBindingGuard()); // Step 5 — JSON-body routes. The Phase C failure-marker routes // (`/:id/extraction-failure`, `/:id/index-failure`) live with the @@ -383,6 +385,7 @@ function registerIndexRoute(router: Router, service: DocumentService): void { '/:id/index', indexJsonParser, rejectNulInBody, + cloudJwtUserBindingGuard(), validateParams(DocumentIdParamSchema), validateBody(IndexDocumentBodySchema), async (req: Request, res: Response) => { diff --git a/packages/core/src/routes/memories.ts b/packages/core/src/routes/memories.ts index 119be5d..125dc45 100644 --- a/packages/core/src/routes/memories.ts +++ b/packages/core/src/routes/memories.ts @@ -48,6 +48,10 @@ import { import type { AgentScope, WorkspaceContext } from '../db/repository-types.js'; import { handleRouteError } from './route-errors.js'; import { validateBody, validateQuery, validateParams } from '../middleware/validate.js'; +import { + readAssertedUserId, + resolveReconcileUserId, +} from '../middleware/cloud-jwt-user-binding.js'; import { CORS_ALLOWED_HEADERS_VALUE } from '../app/cors-headers.js'; import { validateResponse } from '../middleware/validate-response.js'; import { MEMORY_RESPONSE_SCHEMAS } from './response-schema-map.js'; @@ -757,8 +761,9 @@ function registerReconcileRoute(router: Router, service: MemoryService): void { router.post('/reconcile', validateBody(ReconcileBodySchema), async (req: Request, res: Response) => { try { const { userId } = req.body as { userId: string | undefined }; - const result = userId - ? await service.reconcileDeferred(userId) + const effectiveUserId = resolveReconcileUserId(userId, readAssertedUserId(req)); + const result = effectiveUserId + ? await service.reconcileDeferred(effectiveUserId) : await service.reconcileDeferredAll(); res.json(formatReconciliationResponse(result)); } catch (err) { diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 7f8a1b2..1becfbc 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -15,17 +15,32 @@ import { pool } from './db/pool.js'; import { createCoreRuntime, type CoreRuntime } from './app/runtime-container.js'; import { createApp } from './app/create-app.js'; import { checkEmbeddingDimensions } from './app/startup-checks.js'; +import { startDeferredAudnScheduler, type DeferredAudnScheduler } from './services/deferred-audn-scheduler.js'; +import { startCloudTraceSync, stopCloudTraceSync } from './services/cloud-trace-sync.js'; // Process-lifecycle signal handlers reference `runtime` via a closure // captured AFTER `bootstrap()` resolves — wired below. Reconciler // startup stays disabled until `buildReconcilerDeps` returns a // non-null bundle backed by the active storage provider. let runtime: CoreRuntime | null = null; +let deferredAudnScheduler: DeferredAudnScheduler | null = null; async function bootstrap(): Promise { runtime = await createCoreRuntime({ pool }); const app = createApp(runtime); + const prefetchCloudJwt = app.get('cloudJwtPrefetch') as (() => Promise) | undefined; + if (prefetchCloudJwt) { + const ready = await prefetchCloudJwt(); + if (!ready) { + console.warn( + '[startup] Cloud JWKS prefetch did not complete; JWT auth remains degraded until keys load', + ); + } else { + console.log('[startup] Cloud JWKS prefetch ok'); + } + } + const check = await checkEmbeddingDimensions(runtime.pool, runtime.config); if (!check.ok) { console.error(`[startup] FATAL: ${check.message}`); @@ -33,9 +48,28 @@ async function bootstrap(): Promise { } console.log(`[startup] ${check.message}`); + if (runtime.config.cloudTraceSync?.enabled) { + await startCloudTraceSync(runtime.pool, runtime.config.cloudTraceSync); + console.log('[startup] Cloud trace sync uploader started'); + } + app.listen(runtime.config.port, () => { console.log(`AtomicMemory Core running on http://localhost:${runtime!.config.port}`); }); + + // Drain the deferred-AUDN queue in the background so ingest stays fast while + // AUDN is still applied. Opt-in: only when deferred AUDN is enabled. + if (runtime.config.deferredAudnEnabled && runtime.config.deferredAudnAutoReconcile) { + const memory = runtime.services.memory; + deferredAudnScheduler = startDeferredAudnScheduler({ + reconcile: () => memory.reconcileDeferredAll(), + intervalMs: runtime.config.deferredAudnReconcileIntervalMs, + onError: (err) => console.error('[deferred-audn-scheduler] reconcile tick failed:', err), + }); + console.log( + `[startup] deferred-AUDN auto-reconcile every ${runtime.config.deferredAudnReconcileIntervalMs}ms`, + ); + } } bootstrap().catch((err) => { @@ -54,6 +88,8 @@ process.on('unhandledRejection', (reason) => { async function shutdown(signal: string): Promise { console.log(`[shutdown] Received ${signal}, closing...`); + if (deferredAudnScheduler) await deferredAudnScheduler.stop(); + await stopCloudTraceSync(); const closing = runtime ? runtime.pool.end() : pool.end(); await closing; process.exit(0); diff --git a/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts b/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts new file mode 100644 index 0000000..1508d44 --- /dev/null +++ b/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts @@ -0,0 +1,85 @@ +/** + * Unit tests for Cloud trace envelope redaction. + */ + +import { describe, expect, it } from 'vitest'; +import { buildCloudTraceEnvelope } from '../cloud-trace-envelope.js'; + +describe('buildCloudTraceEnvelope', () => { + it('builds a v2 envelope with allowlisted summary fields', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 42, + summary: { + user_id: 'tenant-user-1', + operation_detail: 'consensus ingest (2 fact(s))', + new_memory_id: 'mem_abc', + content_len: 42, + }, + evidence: { source: 'core' }, + }); + + expect(envelope.schema_version).toBe(2); + expect(envelope.summary.user_id).toBe('tenant-user-1'); + expect(envelope.summary.operation_detail).toBe('consensus ingest (2 fact(s))'); + expect(envelope.summary.content_len).toBe(42); + expect(envelope).not.toHaveProperty('actor'); + }); + + it('drops disallowed summary keys including raw content previews', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.search', + outcome: 'success', + durationMs: 10, + summary: { + user_id: 'tenant-user-1', + input_summary: 'secret query text', + query_len: 17, + }, + }); + + expect(envelope.summary).toEqual({ user_id: 'tenant-user-1', query_len: 17 }); + }); + + it('allowlists only bounded evidence string fields', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.search', + outcome: 'success', + durationMs: 10, + evidence: { + mode: 'verbatim', + source_site: 'chatgpt', + as_of: '2026-01-01T00:00:00Z', + api_key: 'secret', + nested: { route: 'search' }, + tokens: ['leak'], + }, + }); + + expect(envelope.evidence).toEqual({ mode: 'verbatim', source_site: 'chatgpt' }); + }); + + it('drops oversized source_site evidence strings', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 1, + evidence: { source_site: 'x'.repeat(121) }, + }); + + expect(envelope.evidence).toEqual({}); + }); +}); diff --git a/packages/core/src/services/__tests__/cloud-trace-sync.test.ts b/packages/core/src/services/__tests__/cloud-trace-sync.test.ts new file mode 100644 index 0000000..9f8028a --- /dev/null +++ b/packages/core/src/services/__tests__/cloud-trace-sync.test.ts @@ -0,0 +1,22 @@ +/** + * Unit tests for Cloud trace sync auth pause recovery. + */ + +import { describe, expect, it } from 'vitest'; +import { resetCloudTraceAuthPause } from '../cloud-trace-sync.js'; + +describe('resetCloudTraceAuthPause', () => { + it('clears paused and auth-related lastErrorCode', () => { + const state = { paused: true, lastErrorCode: 'auth_401' }; + resetCloudTraceAuthPause(state); + expect(state.paused).toBe(false); + expect(state.lastErrorCode).toBeNull(); + }); + + it('clears non-auth lastErrorCode after a successful upload', () => { + const state = { paused: true, lastErrorCode: 'network_error' }; + resetCloudTraceAuthPause(state); + expect(state.paused).toBe(false); + expect(state.lastErrorCode).toBeNull(); + }); +}); diff --git a/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts b/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts new file mode 100644 index 0000000..99d32c5 --- /dev/null +++ b/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts @@ -0,0 +1,68 @@ +/** + * Tests for the deferred-AUDN background scheduler: it polls reconcile() on the + * interval, never overlaps ticks, surfaces errors via onError, and stop() awaits + * the in-flight drain. Uses fake timers to stay deterministic (no real sleeps). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + startDeferredAudnScheduler, + type DeferredAudnSchedulerOptions, +} from '../deferred-audn-scheduler.js'; +import type { ReconciliationResult } from '../deferred-audn.js'; + +const EMPTY: ReconciliationResult = { + processed: 0, resolved: 0, noops: 0, updates: 0, + supersedes: 0, deletes: 0, adds: 0, errors: 0, durationMs: 0, +}; + +function makeOptions(over: Partial = {}): DeferredAudnSchedulerOptions { + return { + reconcile: vi.fn(async () => EMPTY), + intervalMs: 1000, + onError: vi.fn(), + ...over, + }; +} + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe('startDeferredAudnScheduler', () => { + it('is idle before the first tick', () => { + const s = startDeferredAudnScheduler(makeOptions()); + expect(s.isRunning).toBe(false); + }); + + it('calls reconcile on each interval tick', async () => { + const opts = makeOptions(); + startDeferredAudnScheduler(opts); + await vi.advanceTimersByTimeAsync(2500); + expect(opts.reconcile).toHaveBeenCalledTimes(2); + }); + + it('does not overlap ticks while a reconcile is in flight', async () => { + let release!: () => void; + const reconcile = vi.fn(() => new Promise((r) => { release = () => r(EMPTY); })); + startDeferredAudnScheduler(makeOptions({ reconcile })); + await vi.advanceTimersByTimeAsync(3000); + expect(reconcile).toHaveBeenCalledTimes(1); + release(); + }); + + it('routes a rejected reconcile to onError instead of throwing', async () => { + const onError = vi.fn(); + const reconcile = vi.fn(async () => { throw new Error('boom'); }); + startDeferredAudnScheduler(makeOptions({ reconcile, onError })); + await vi.advanceTimersByTimeAsync(1000); + expect(onError).toHaveBeenCalledOnce(); + }); + + it('stop() clears the interval and awaits the in-flight drain', async () => { + const opts = makeOptions(); + const s = startDeferredAudnScheduler(opts); + await vi.advanceTimersByTimeAsync(1000); + await s.stop(); + await vi.advanceTimersByTimeAsync(5000); + expect(opts.reconcile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/src/services/__tests__/memory-service-config.test.ts b/packages/core/src/services/__tests__/memory-service-config.test.ts index 80d08d3..1f6dd32 100644 --- a/packages/core/src/services/__tests__/memory-service-config.test.ts +++ b/packages/core/src/services/__tests__/memory-service-config.test.ts @@ -125,6 +125,7 @@ describe('MemoryService config seam', () => { '', undefined, undefined, + false, ); }); @@ -152,6 +153,7 @@ describe('MemoryService config seam', () => { 'https://example.test/thread', sessionTimestamp, 'thread-1', + false, ); }); @@ -193,6 +195,7 @@ describe('MemoryService config seam', () => { '', undefined, undefined, + false, ); }); @@ -220,6 +223,7 @@ describe('MemoryService config seam', () => { 'https://example.test/quick', sessionTimestamp, 'thread-quick', + false, ); }); @@ -300,6 +304,7 @@ describe('MemoryService config seam', () => { workspace, sessionTimestamp, 'thread-workspace', + false, ); }); diff --git a/packages/core/src/services/audn-decision-executor.ts b/packages/core/src/services/audn-decision-executor.ts index 3b94b73..4ccab08 100644 --- a/packages/core/src/services/audn-decision-executor.ts +++ b/packages/core/src/services/audn-decision-executor.ts @@ -10,6 +10,7 @@ import { type AUDNDecision } from './extraction.js'; import { emitAuditEvent } from './audit-events.js'; import { recordContradictionLesson } from './lesson-service.js'; import { emitLineageEvent } from './memory-lineage.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; import { applyOpinionSignal, audnActionToOpinionSignal } from './memory-network.js'; import { buildAtomicFactProjection, buildForesightProjections } from './memcell-projection.js'; import { @@ -241,6 +242,22 @@ async function updateCanonicalFact( throw new Error(`AUDN UPDATE failed: missing successor canonical object for "${target.memoryId}"`); } await deps.stores.memory.updateMemoryMetadata(userId, target.memoryId, { cmo_id: lineage.cmoId }); + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.update', + durationMs: 0, + userId, + summary: { + fact_len: fact.fact.length, + previous_memory_id: target.memoryId, + new_memory_id: target.memoryId, + }, + evidence: { source: 'audn-update' }, + }, + ); return { outcome: 'updated', memoryId: target.memoryId }; } diff --git a/packages/core/src/services/cloud-trace-envelope.ts b/packages/core/src/services/cloud-trace-envelope.ts new file mode 100644 index 0000000..16be1f5 --- /dev/null +++ b/packages/core/src/services/cloud-trace-envelope.ts @@ -0,0 +1,112 @@ +/** + * Redacted Cloud trace envelope builder for OSS Core outbound sync. + * + * Keeps secrets and verbatim memory/query text out of the durable outbox payload. + * Summary and evidence fields are allowlisted; unknown keys are dropped. + */ + +const CLOUD_TRACE_SCHEMA_VERSION = 2 as const; +const MAX_EVIDENCE_STRING_LEN = 120; + +export type CloudTraceOperation = + | 'memory.ingest' + | 'memory.update' + | 'memory.delete' + | 'memory.search'; + +export type CloudTraceOutcome = 'success' | 'error'; + +export interface BuildCloudTraceEnvelopeInput { + eventId: string; + coreInstanceId: string; + occurredAt: string; + operation: CloudTraceOperation; + outcome: CloudTraceOutcome; + durationMs: number; + summary?: Record; + evidence?: Record; +} + +export interface CloudTraceEnvelopeV2 { + schema_version: typeof CLOUD_TRACE_SCHEMA_VERSION; + event_id: string; + core_instance_id: string; + occurred_at: string; + operation: CloudTraceOperation; + outcome: CloudTraceOutcome; + duration_ms: number; + summary: Record; + evidence: Record; +} + +/** Summary keys permitted on outbound Cloud trace envelopes. */ +const ALLOWED_SUMMARY_KEYS = new Set([ + 'user_id', + 'result_count', + 'new_memory_id', + 'previous_memory_id', + 'episode_id', + 'memories_stored', + 'memories_updated', + 'memories_deleted', + 'content_len', + 'query_len', + 'fact_len', + 'operation_detail', +]); + +/** Evidence keys permitted on outbound Cloud trace envelopes. */ +const ALLOWED_EVIDENCE_KEYS = new Set(['mode', 'source', 'source_site']); + +/** + * Build a version-2 envelope suitable for Cloud trace ingest. + * Allowlists summary and evidence fields; drops unknown keys and arrays. + */ +export function buildCloudTraceEnvelope( + input: BuildCloudTraceEnvelopeInput, +): CloudTraceEnvelopeV2 { + if (!input.eventId || !input.coreInstanceId) { + throw new Error('cloud trace envelope requires eventId and coreInstanceId'); + } + if (input.durationMs < 0) { + throw new Error('durationMs must be non-negative'); + } + + return { + schema_version: CLOUD_TRACE_SCHEMA_VERSION, + event_id: input.eventId, + core_instance_id: input.coreInstanceId, + occurred_at: input.occurredAt, + operation: input.operation, + outcome: input.outcome, + duration_ms: input.durationMs, + summary: allowlistSummary(input.summary ?? {}), + evidence: allowlistEvidence(input.evidence ?? {}), + }; +} + +function allowlistSummary(value: Record): Record { + const out: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!ALLOWED_SUMMARY_KEYS.has(key)) continue; + if (typeof entry === 'number' && Number.isFinite(entry)) { + out[key] = entry; + continue; + } + if (typeof entry === 'string' && entry.length > 0 && entry.length <= MAX_EVIDENCE_STRING_LEN) { + out[key] = entry; + } + } + return out; +} + +function allowlistEvidence(value: Record): Record { + const out: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!ALLOWED_EVIDENCE_KEYS.has(key)) continue; + if (typeof entry === 'string' && entry.length > 0 && entry.length <= MAX_EVIDENCE_STRING_LEN) { + out[key] = entry; + } + } + return out; +} diff --git a/packages/core/src/services/cloud-trace-sync.ts b/packages/core/src/services/cloud-trace-sync.ts new file mode 100644 index 0000000..98a69c7 --- /dev/null +++ b/packages/core/src/services/cloud-trace-sync.ts @@ -0,0 +1,414 @@ +/** + * OSS Core → Cloud trace sync runtime: enqueue hooks, background uploader, health. + */ + +import { randomUUID } from 'node:crypto'; +import type pg from 'pg'; +import { cloudApiPost } from '../cloud/cloud-api-client.js'; +import { resolveCoreInstanceId } from '../cloud/instance-id.js'; +import { sendRuntimeHeartbeat } from '../cloud/heartbeat-client.js'; +import type { CloudTraceSyncConfig } from '../config.js'; +import { readPackageVersion } from '../db/migration-schema.js'; +import { + claimCloudTraceBatch, + countDeadLetterCloudTraces, + countPendingCloudTraces, + enqueueCloudTraceOutbox, + getOldestPendingCloudTraceAgeMs, + markCloudTraceDeadLetter, + markCloudTraceSent, + purgeDeadLetterCloudTraces, + purgeSentCloudTraces, + scheduleCloudTraceRetry, +} from '../db/cloud-trace-outbox-repository.js'; +import { + buildCloudTraceEnvelope, + type CloudTraceOperation, + type CloudTraceOutcome, +} from './cloud-trace-envelope.js'; + +const RUNTIME_CAPABILITIES = ['memory.read', 'memory.write', 'trace.stream'] as const; +const HEALTH_CACHE_TTL_MS = 5_000; + +export interface CloudTraceHealthSnapshot { + status: 'healthy' | 'degraded' | 'paused' | 'disabled'; + pendingCount: number; + oldestPendingAgeMs: number | null; + lastSuccessAt: string | null; + deadLetterCount: number; + lastErrorCode: string | null; + lastHeartbeatAt: string | null; + heartbeatStatus: 'ok' | 'failed' | 'auth_paused' | 'skipped'; + enqueueFailures: number; +} + +/** Public liveness view — no backlog counts or error codes. */ +export interface CloudTracePublicHealth { + status: 'healthy' | 'degraded' | 'paused' | 'disabled'; +} + +export interface RecordCloudTraceOperationInput { + operation: CloudTraceOperation; + outcome?: CloudTraceOutcome; + durationMs: number; + userId?: string; + summary?: Record; + evidence?: Record; +} + +interface UploaderRuntime { + pool: pg.Pool; + config: CloudTraceSyncConfig; + instanceId: string; + timer: NodeJS.Timeout | null; + draining: boolean; + paused: boolean; + inFlight: Promise; + lastSuccessAt: Date | null; + lastErrorCode: string | null; + enqueueFailures: number; + lastHeartbeatAt: Date | null; + heartbeatStatus: CloudTraceHealthSnapshot['heartbeatStatus']; + healthCache: { expiresAt: number; snapshot: CloudTraceHealthSnapshot } | null; +} + +let uploaderRuntime: UploaderRuntime | null = null; + +export function recordCloudTraceOperation( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig | null | undefined, + instanceId: string | null | undefined, + input: RecordCloudTraceOperationInput, +): void { + if (!syncConfig?.enabled || !instanceId) return; + void enqueueCloudTraceFromInput(pool, syncConfig, instanceId, input).catch((error) => { + if (uploaderRuntime) uploaderRuntime.enqueueFailures += 1; + console.error('[cloud-trace-sync] enqueue failed:', error); + }); +} + +async function enqueueCloudTraceFromInput( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig, + instanceId: string, + input: RecordCloudTraceOperationInput, +): Promise { + const pending = await countPendingCloudTraces(pool); + if (pending >= syncConfig.maxPending) { + if (uploaderRuntime) uploaderRuntime.enqueueFailures += 1; + throw new Error(`cloud trace outbox backlog at cap (${syncConfig.maxPending})`); + } + const summary = { + user_id: input.userId ?? 'default', + ...(input.summary ?? {}), + }; + const envelope = buildCloudTraceEnvelope({ + eventId: randomUUID(), + coreInstanceId: instanceId, + occurredAt: new Date().toISOString(), + operation: input.operation, + outcome: input.outcome ?? 'success', + durationMs: Math.max(0, Math.round(input.durationMs)), + summary, + evidence: input.evidence, + }); + await enqueueCloudTraceOutbox(pool, envelope); +} + +export async function startCloudTraceSync( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig, +): Promise { + if (uploaderRuntime) return; + const instanceId = await resolveCoreInstanceId(syncConfig.instanceId); + syncConfig.instanceId = instanceId; + uploaderRuntime = { + pool, + config: syncConfig, + instanceId, + timer: null, + draining: false, + paused: false, + inFlight: Promise.resolve(), + lastSuccessAt: null, + lastErrorCode: null, + enqueueFailures: 0, + lastHeartbeatAt: null, + heartbeatStatus: 'skipped', + healthCache: null, + }; + scheduleUploaderTick(); +} + +export async function stopCloudTraceSync(): Promise { + const runtime = uploaderRuntime; + if (!runtime) return; + runtime.draining = true; + if (runtime.timer) clearTimeout(runtime.timer); + runtime.timer = null; + const deadline = Date.now() + runtime.config.shutdownDrainMs; + while (Date.now() < deadline) { + await runtime.inFlight; + const pending = await countPendingCloudTraces(runtime.pool); + if (pending === 0) break; + if (runtime.paused) break; + const claimed = await claimCloudTraceBatch( + runtime.pool, + runtime.config.batchSize, + runtime.config.claimStaleMs, + ); + if (claimed.length === 0) break; + for (const row of claimed) { + await uploadRow(runtime, row.eventId, row.payload, row.attemptCount); + } + await sleep(Math.min(runtime.config.pollIntervalMs, deadline - Date.now())); + } + uploaderRuntime = null; +} + +function sleep(ms: number): Promise { + if (ms <= 0) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function getCloudTraceHealthSnapshot(): CloudTraceHealthSnapshot { + const runtime = uploaderRuntime; + if (!runtime) { + return { + status: 'disabled', + pendingCount: 0, + oldestPendingAgeMs: null, + lastSuccessAt: null, + deadLetterCount: 0, + lastErrorCode: null, + lastHeartbeatAt: null, + heartbeatStatus: 'skipped', + enqueueFailures: 0, + }; + } + return { + status: runtime.paused ? 'paused' : runtime.lastErrorCode ? 'degraded' : 'healthy', + pendingCount: 0, + oldestPendingAgeMs: null, + lastSuccessAt: runtime.lastSuccessAt?.toISOString() ?? null, + deadLetterCount: 0, + lastErrorCode: runtime.lastErrorCode, + lastHeartbeatAt: runtime.lastHeartbeatAt?.toISOString() ?? null, + heartbeatStatus: runtime.heartbeatStatus, + enqueueFailures: runtime.enqueueFailures, + }; +} + +export async function getCloudTraceHealthSnapshotAsync( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig | null | undefined, +): Promise { + if (!syncConfig?.enabled) { + return getCloudTraceHealthSnapshot(); + } + const runtime = uploaderRuntime; + const now = Date.now(); + if (runtime?.healthCache && runtime.healthCache.expiresAt > now) { + return runtime.healthCache.snapshot; + } + const [pendingCount, oldestPendingAgeMs, deadLetterCount] = await Promise.all([ + countPendingCloudTraces(pool), + getOldestPendingCloudTraceAgeMs(pool), + countDeadLetterCloudTraces(pool), + ]); + let status: CloudTraceHealthSnapshot['status'] = 'healthy'; + if (runtime?.paused) status = 'paused'; + else if (deadLetterCount > 0 || (oldestPendingAgeMs ?? 0) > 300_000) status = 'degraded'; + const snapshot: CloudTraceHealthSnapshot = { + status, + pendingCount, + oldestPendingAgeMs, + lastSuccessAt: runtime?.lastSuccessAt?.toISOString() ?? null, + deadLetterCount, + lastErrorCode: runtime?.lastErrorCode ?? null, + lastHeartbeatAt: runtime?.lastHeartbeatAt?.toISOString() ?? null, + heartbeatStatus: runtime?.heartbeatStatus ?? 'skipped', + enqueueFailures: runtime?.enqueueFailures ?? 0, + }; + if (runtime) { + runtime.healthCache = { expiresAt: now + HEALTH_CACHE_TTL_MS, snapshot }; + } + return snapshot; +} + +export function toCloudTracePublicHealth( + snapshot: CloudTraceHealthSnapshot, +): CloudTracePublicHealth { + return { status: snapshot.status }; +} + +function scheduleUploaderTick(): void { + const runtime = uploaderRuntime; + if (!runtime || runtime.draining) return; + runtime.timer = setTimeout(() => { + runtime.inFlight = uploadOnce(runtime) + .catch((error) => { + runtime.lastErrorCode = 'upload_loop_error'; + console.error('[cloud-trace-sync] upload tick failed:', error); + }) + .finally(() => { + purgeSentCloudTraces(runtime.pool, runtime.config.sentRetentionMs).catch(() => {}); + purgeDeadLetterCloudTraces(runtime.pool, runtime.config.deadLetterRetentionMs).catch( + () => {}, + ); + scheduleUploaderTick(); + }); + }, runtime.config.pollIntervalMs); +} + +async function uploadOnce(runtime: UploaderRuntime): Promise { + if (!runtime.draining) { + await sendHeartbeatOnce(runtime); + } + if (runtime.paused) return; + const batch = await claimCloudTraceBatch( + runtime.pool, + runtime.config.batchSize, + runtime.config.claimStaleMs, + ); + for (const row of batch) { + await uploadRow(runtime, row.eventId, row.payload, row.attemptCount); + } +} + +function resolveLocalUrl(): string { + const port = process.env.PORT ?? '17350'; + return `http://127.0.0.1:${port}`; +} + +function clearAuthPause(runtime: UploaderRuntime): void { + resetCloudTraceAuthPause(runtime); +} + +/** Clear sticky auth pause after a successful heartbeat or upload. */ +export function resetCloudTraceAuthPause(state: { + paused: boolean; + lastErrorCode: string | null; +}): void { + state.paused = false; + state.lastErrorCode = null; +} + +async function sendHeartbeatOnce(runtime: UploaderRuntime): Promise { + const packageVersion = readPackageVersion(); + const result = await sendRuntimeHeartbeat({ + apiUrl: runtime.config.apiUrl, + apiKey: runtime.config.apiKey, + payload: { + core_instance_id: runtime.instanceId, + core_version: packageVersion, + connector_version: packageVersion, + capabilities: [...RUNTIME_CAPABILITIES], + local_url: resolveLocalUrl(), + }, + }); + if (result.ok) { + runtime.lastHeartbeatAt = new Date(); + runtime.heartbeatStatus = 'ok'; + clearAuthPause(runtime); + return; + } + runtime.heartbeatStatus = result.errorCode?.startsWith('auth_') ? 'auth_paused' : 'failed'; + if (result.errorCode?.startsWith('auth_')) { + runtime.paused = true; + runtime.lastErrorCode = result.errorCode; + } +} + +async function uploadRow( + runtime: UploaderRuntime, + eventId: string, + payload: unknown, + attemptCount: number, +): Promise { + let response: Response; + try { + response = await cloudApiPost({ + apiUrl: runtime.config.apiUrl, + apiKey: runtime.config.apiKey, + path: '/v1/observability/traces', + body: payload, + }); + } catch { + await handleRetry(runtime, eventId, attemptCount, 'network_error', null); + return; + } + + await handleUploadResponse(runtime, eventId, response, attemptCount); +} + +async function handleUploadResponse( + runtime: UploaderRuntime, + eventId: string, + response: Response, + attemptCount: number, +): Promise { + const status = response.status; + + if (status === 401 || status === 403) { + runtime.paused = true; + runtime.lastErrorCode = `auth_${status}`; + await scheduleCloudTraceRetry( + runtime.pool, + eventId, + runtime.lastErrorCode, + new Date(Date.now() + runtime.config.maxRetryMs), + ); + return; + } + + if (status === 200 || status === 201) { + await markCloudTraceSent(runtime.pool, eventId); + runtime.lastSuccessAt = new Date(); + clearAuthPause(runtime); + return; + } + + if (status === 408 || status === 429 || status >= 500) { + const retryAfter = parseRetryAfterMs(response.headers.get('retry-after')); + await handleRetry(runtime, eventId, attemptCount, `http_${status}`, retryAfter); + return; + } + + await markCloudTraceDeadLetter(runtime.pool, eventId, `http_${status}`); + runtime.lastErrorCode = `http_${status}`; +} + +async function handleRetry( + runtime: UploaderRuntime, + eventId: string, + attemptCount: number, + errorCode: string, + retryAfterMs: number | null, +): Promise { + runtime.lastErrorCode = errorCode; + if (attemptCount + 1 >= runtime.config.maxAttempts) { + await markCloudTraceDeadLetter(runtime.pool, eventId, errorCode); + return; + } + const backoff = Math.min( + runtime.config.maxRetryMs, + runtime.config.baseRetryMs * 2 ** attemptCount, + ); + const delayMs = retryAfterMs ?? backoff; + await scheduleCloudTraceRetry( + runtime.pool, + eventId, + errorCode, + new Date(Date.now() + delayMs), + ); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const seconds = Number.parseInt(value, 10); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const dateMs = Date.parse(value); + if (Number.isFinite(dateMs)) return Math.max(0, dateMs - Date.now()); + return null; +} \ No newline at end of file diff --git a/packages/core/src/services/deferred-audn-scheduler.ts b/packages/core/src/services/deferred-audn-scheduler.ts new file mode 100644 index 0000000..496bbee --- /dev/null +++ b/packages/core/src/services/deferred-audn-scheduler.ts @@ -0,0 +1,66 @@ +/** + * Background scheduler that drains the deferred-AUDN queue off the ingest path. + * + * With `DEFERRED_AUDN_ENABLED=true`, ingest stores facts immediately and queues + * the per-fact AUDN decision for later, so a write returns fast. Without a + * drainer, that queue only empties when something calls `/v1/memories/reconcile`. + * This scheduler polls `reconcile()` on an interval so AUDN is applied + * automatically in the background — making ingest latency low while keeping the + * supersede/dedup/contradiction decisions that AUDN provides. + * + * It mirrors `raw-storage-reconciler-scheduler.ts`: one in-flight tick at a time, + * `stop()` clears the interval and awaits the in-flight drain. `createCoreRuntime` + * does NOT start it — the process lifecycle (server.ts) opts in via config so + * tests can drive `reconcile()` directly and stay deterministic. + */ +import type { ReconciliationResult } from './deferred-audn.js'; + +export interface DeferredAudnSchedulerOptions { + /** Drains one batch of the deferred-AUDN queue (e.g. MemoryService.reconcileDeferredAll). */ + reconcile: () => Promise; + /** Polling interval — `setInterval(tick, intervalMs)`. */ + intervalMs: number; + /** Called when a tick's reconcile rejects, so the error is surfaced, not swallowed. */ + onError: (err: unknown) => void; +} + +export interface DeferredAudnScheduler { + /** Clears the interval and awaits the in-flight `reconcile()`. */ + stop(): Promise; + /** True while a `reconcile()` invocation is in flight. */ + readonly isRunning: boolean; +} + +/** Start polling `reconcile()` every `intervalMs`; returns a stop handle. */ +export function startDeferredAudnScheduler( + options: DeferredAudnSchedulerOptions, +): DeferredAudnScheduler { + let isRunning = false; + let stopped = false; + let inFlight: Promise = Promise.resolve(); + + const tick = (): void => { + if (stopped || isRunning) return; + isRunning = true; + inFlight = options + .reconcile() + .then(() => undefined) + .catch((err) => options.onError(err)) + .finally(() => { + isRunning = false; + }); + }; + + const handle = setInterval(tick, options.intervalMs); + + return { + get isRunning() { + return isRunning; + }, + async stop() { + stopped = true; + clearInterval(handle); + await inFlight; + }, + }; +} diff --git a/packages/core/src/services/deferred-audn.ts b/packages/core/src/services/deferred-audn.ts index 76163d4..2231ee9 100644 --- a/packages/core/src/services/deferred-audn.ts +++ b/packages/core/src/services/deferred-audn.ts @@ -102,6 +102,17 @@ export async function getReconciliationStatus( return { pending, enabled: config.deferredAudnEnabled }; } +/** Tally a single resolved AUDN action into the running result. */ +function tallyReconcileAction(result: ReconciliationResult, action: string): void { + switch (action) { + case 'NOOP': result.noops++; break; + case 'UPDATE': result.updates++; break; + case 'SUPERSEDE': result.supersedes++; break; + case 'DELETE': result.deletes++; break; + case 'ADD': result.adds++; break; + } +} + async function processReconciliationBatch( pool: pg.Pool, repo: MemoryStore, @@ -113,21 +124,24 @@ async function processReconciliationBatch( supersedes: 0, deletes: 0, adds: 0, errors: 0, durationMs: 0, }; - for (const memory of deferred) { - result.processed++; - try { - const action = await reconcileSingleMemory(pool, repo, memory); - result.resolved++; - switch (action) { - case 'NOOP': result.noops++; break; - case 'UPDATE': result.updates++; break; - case 'SUPERSEDE': result.supersedes++; break; - case 'DELETE': result.deletes++; break; - case 'ADD': result.adds++; break; + // Each deferred memory re-fetches its own candidates and is independent, so the + // per-memory LLM AUDN decisions run concurrently (capped) instead of serially — + // the reconcile pass is LLM-bound, so this is a ~Nx wall-clock win. + const concurrency = Math.max(1, config.deferredAudnConcurrency); + for (let i = 0; i < deferred.length; i += concurrency) { + const chunk = deferred.slice(i, i + concurrency); + const settled = await Promise.allSettled( + chunk.map((memory) => reconcileSingleMemory(pool, repo, memory)), + ); + for (const outcome of settled) { + result.processed++; + if (outcome.status === 'fulfilled') { + result.resolved++; + tallyReconcileAction(result, outcome.value); + } else { + result.errors++; + console.error('[deferred-audn] Error reconciling memory:', outcome.reason); } - } catch (err) { - result.errors++; - console.error(`[deferred-audn] Error reconciling memory ${memory.id}:`, err); } } diff --git a/packages/core/src/services/embedding.ts b/packages/core/src/services/embedding.ts index 21d6630..106e7c5 100644 --- a/packages/core/src/services/embedding.ts +++ b/packages/core/src/services/embedding.ts @@ -257,9 +257,13 @@ type TransformersPipeline = (texts: string | string[], options: Record { const { pipeline } = await import('@huggingface/transformers'); - console.log(`[embedding] Loading local WASM model: ${model}`); + // EMBEDDING_DTYPE lets larger local models (e.g. bge-base) load as quantized + // (q8) instead of fp32 — fp32 of a big model is a ~400MB download + slow WASM + // init that can hang. Small models (MiniLM) stay fine on the fp32 default. + const dtype = (process.env.EMBEDDING_DTYPE ?? 'fp32') as 'fp32' | 'fp16' | 'q8' | 'q4'; + console.log(`[embedding] Loading local model: ${model} (dtype=${dtype})`); const start = performance.now(); - const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' }); + const extractor = await pipeline('feature-extraction', model, { dtype }); console.log(`[embedding] Model loaded in ${(performance.now() - start).toFixed(0)}ms`); return extractor as unknown as TransformersPipeline; } diff --git a/packages/core/src/services/llm.ts b/packages/core/src/services/llm.ts index a2f117c..489f685 100644 --- a/packages/core/src/services/llm.ts +++ b/packages/core/src/services/llm.ts @@ -348,12 +348,19 @@ class AnthropicLLM implements LLMProvider { async chat(messages: ChatMessage[], options: ChatOptions = {}): Promise { const split = splitAnthropicMessages(messages); + // Anthropic has no response_format; enforce JSON via an assistant prefill ('{'). + // Without this, jsonMode callers (extraction/AUDN) receive conversational prose + // and extract zero facts. The prefill character is prepended back to the result. + const prefillJson = options.jsonMode === true; + const reqMessages = prefillJson + ? [...split.messages, { role: 'assistant' as const, content: '{' }] + : split.messages; const request = () => this.client.messages.create({ model: this.model, max_tokens: options.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, temperature: options.temperature ?? 0, ...(split.system !== undefined ? { system: split.system } : {}), - messages: split.messages, + messages: reqMessages, }); const started = performance.now(); const response = await retryOnRateLimit(request); @@ -363,7 +370,8 @@ class AnthropicLLM implements LLMProvider { null, ); recordChatCost(this.model, usage, started); - return extractAnthropicText(response); + const text = extractAnthropicText(response); + return prefillJson ? `{${text}` : text; } } diff --git a/packages/core/src/services/memory-crud.ts b/packages/core/src/services/memory-crud.ts index df5f50c..53284ef 100644 --- a/packages/core/src/services/memory-crud.ts +++ b/packages/core/src/services/memory-crud.ts @@ -21,6 +21,7 @@ import { markCleanupFailedAndSyncArtifact, markCleanupSuccessAndSyncArtifact, } from '../db/raw-doc-artifact-sync.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; export interface ClaimSlotBackfillResult { scanned: number; @@ -77,12 +78,28 @@ export async function expandMemoriesInWorkspace( } export async function deleteMemory(deps: MemoryServiceDeps, id: string, userId: string) { + const existing = await deps.stores.memory.getMemory(id, userId); + if (!existing) return; const version = await deps.stores.claim.getClaimVersionByMemoryId(userId, id); const target = version ? { claimId: version.claim_id, versionId: version.id } : null; await deps.stores.memory.softDeleteMemory(userId, id); if (config.auditLoggingEnabled) { emitAuditEvent('memory:delete', userId, {}, { memoryId: id }); } + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.delete', + durationMs: 0, + userId, + summary: { + operation_detail: 'memory.delete', + previous_memory_id: id, + }, + }, + ); if (!target) return; await deps.stores.claim.supersedeClaimVersion(userId, target.versionId, null); await deps.stores.claim.invalidateClaim(userId, target.claimId); diff --git a/packages/core/src/services/memory-ingest.ts b/packages/core/src/services/memory-ingest.ts index d3ea6f3..ffb2544 100644 --- a/packages/core/src/services/memory-ingest.ts +++ b/packages/core/src/services/memory-ingest.ts @@ -24,6 +24,7 @@ import type { MemoryServiceDeps, } from './memory-service-types.js'; import type { ReflectionJobsRepository } from '../db/reflection-jobs-repository.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; /** Enqueue a reflection job for (userId, conversationId) when the feature gate is on. */ async function maybeEnqueueReflectionJob( @@ -86,6 +87,8 @@ function buildIngestResult(episodeId: string, factsCount: number, acc: IngestAcc } function finalizeIngestResult( + deps: MemoryServiceDeps, + ingestStart: number, episodeId: string, factsCount: number, acc: IngestAccumulator, @@ -94,10 +97,30 @@ function finalizeIngestResult( traceCollector: IngestTraceCollector, traceMetadata: Parameters[0], ): IngestResult { - return { + const result = { ...buildIngestResult(episodeId, factsCount, acc, linksCreated, compositesCreated), ingestTraceId: traceCollector.finalize(traceMetadata), }; + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.ingest', + durationMs: performance.now() - ingestStart, + userId: traceMetadata.userId, + summary: { + operation_detail: `${traceMetadata.mode} ingest (${factsCount} fact(s))`, + result_count: acc.counters.stored + acc.counters.updated, + episode_id: episodeId, + memories_stored: acc.counters.stored, + memories_updated: acc.counters.updated, + memories_deleted: acc.counters.deleted, + }, + evidence: { mode: traceMetadata.mode, source_site: traceMetadata.sourceSite }, + }, + ); + return result; } /** Full consensus-based ingest pipeline. */ @@ -186,6 +209,8 @@ export async function performIngest( await maybeEnqueueReflectionJob(deps.reflectionJobs, deps.reflectEnabled, userId, episodeId); console.log(`[timing] ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${facts.length} facts, ${postWrite.compositesCreated} composites, topic=${postWrite.topicAbstractionApplied})`); return finalizeIngestResult( + deps, + ingestStart, episodeId, facts.length, acc, @@ -236,6 +261,8 @@ export async function performQuickIngest( console.log(`[timing] quick-ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${extractedFacts.length} facts, ${acc.counters.stored} stored, ${acc.counters.skipped} skipped)`); return finalizeIngestResult( + deps, + ingestStart, episodeId, extractedFacts.length, acc, @@ -286,6 +313,7 @@ export async function performStoreVerbatim( metadata?: MemoryMetadata, sessionId?: string, ): Promise { + const ingestStart = performance.now(); const episodeId = await deps.stores.episode.storeEpisode({ userId, content, sourceSite, sourceUrl, sessionId }); const embedding = await embedText(content); const writeSecurity = assessWriteSecurity(content, sourceSite, deps.config); @@ -329,7 +357,7 @@ export async function performStoreVerbatim( memoryId, }); - return { + const result: IngestResult = { episodeId, factsExtracted: 1, memoriesStored: isUpdate ? 0 : 1, @@ -343,6 +371,23 @@ export async function performStoreVerbatim( compositesCreated: 0, ingestTraceId: traceCollector.finalize({ mode: 'verbatim', userId, sourceSite, sourceUrl, episodeId, factsExtracted: 1 }), }; + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: isUpdate ? 'memory.update' : 'memory.ingest', + durationMs: performance.now() - ingestStart, + userId, + summary: { + content_len: content.length, + new_memory_id: memoryId, + previous_memory_id: isUpdate ? memoryId : undefined, + }, + evidence: { mode: 'verbatim', source_site: sourceSite }, + }, + ); + return result; } /** @@ -462,6 +507,8 @@ export async function performWorkspaceIngest( console.log(`[timing] ws-ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${facts.length} facts, workspace=${workspace.workspaceId})`); return finalizeIngestResult( + deps, + ingestStart, episodeId, facts.length, acc, diff --git a/packages/core/src/services/retrieval-side-effects.ts b/packages/core/src/services/retrieval-side-effects.ts index 34671f5..f7470ac 100644 --- a/packages/core/src/services/retrieval-side-effects.ts +++ b/packages/core/src/services/retrieval-side-effects.ts @@ -12,6 +12,7 @@ import type { SearchResult } from '../db/repository-types.js'; import type { MemoryServiceDeps } from './memory-service-types.js'; import { emitAuditEvent } from './audit-events.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; /** Run post-search side effects. Swallows per-memory touch failures. */ export function recordSearchSideEffects( @@ -34,4 +35,19 @@ export function recordSearchSideEffects( topScore: outputMemories[0]?.score ?? 0, }, { sourceSite }); } + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.search', + durationMs: 0, + userId, + summary: { + query_len: query.length, + result_count: outputMemories.length, + }, + evidence: { source_site: sourceSite }, + }, + ); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a710ef5..c632df6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: turbo: specifier: 2.9.14 version: 2.9.14 + yaml: + specifier: ^2.8.3 + version: 2.9.0 adapters/langchain: dependencies: @@ -189,6 +192,9 @@ importers: express: specifier: ^5.1.0 version: 5.2.1 + jose: + specifier: ^6.2.3 + version: 6.2.3 multiformats: specifier: ^13.4.1 version: 13.4.2 diff --git a/scripts/ci/__tests__/release-policy.test.mjs b/scripts/ci/__tests__/release-policy.test.mjs index 8ec1944..46475f1 100644 --- a/scripts/ci/__tests__/release-policy.test.mjs +++ b/scripts/ci/__tests__/release-policy.test.mjs @@ -10,6 +10,8 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { checkCodeownersText, + checkCompositeActionText, + checkImagePublisherText, checkPackageManifest, checkWorkflowText, } from "../release-policy.mjs"; @@ -73,6 +75,234 @@ test("CODEOWNERS with only an unrelated rule does not cover the workflow", () => assert.ok(failures.some((failure) => /no rule covers/.test(failure))); }); +const VALID_INTERNAL_IMAGE_YAML = [ + "env:", + " IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core-internal", + "jobs:", + " publish:", + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " steps:", + ' - run: docker buildx build --tag "${IMAGE_NAME}:sha-abc1234" --push .', + "", +].join("\n"); + +test("workflow without image pushes is not held to image-publisher policy", () => { + assert.deepEqual(checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker build .\n", ".github/workflows/ci.yml"), []); +}); + +test("non-enumerated workflow pushing images is rejected", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --push .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane nor the enumerated internal image publisher/.test(failure))); +}); + +test("internal image publisher without the repository guard is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace(/^.*github\.repository.*\n/m, ""); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /repository guard|github\.repository/.test(failure))); +}); + +test("internal image publisher pinned to the wrong image name is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace("atomicmemory-core-internal", "atomicmemory-core"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /assign env IMAGE_NAME exactly once/.test(failure))); +}); + +test("internal image publisher tagging a literal registry path is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', '--tag "ghcr.io/atomicstrata/atomicmemory-core:latest"'); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("internal image publisher pushing through another variable is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', '--tag "${ALT_IMAGE}:latest"'); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("internal image publisher overriding IMAGE_NAME via quoted key or flow mapping is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + ' steps:\n - env: { "IMAGE_NAME": "ghcr.io/atomicstrata/atomicmemory-core" }', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("workflow splitting --output type=registry across continuation lines is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - run: |\n docker buildx build --output \\\n type=registry .\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher using docker push or imagetools create is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + ' - run: docker push "${IMAGE_NAME}:sha-abc1234"\n'; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /forbidden here; they publish without a --tag/.test(failure))); +}); + +test("valid internal image publisher fixture passes", () => { + assert.deepEqual(checkImagePublisherText(VALID_INTERNAL_IMAGE_YAML, ".github/workflows/internal-core-docker-image.yml"), []); +}); + +test("workflow pushing via --output=type=registry is treated as a publisher", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --output=type=registry,name=ghcr.io/x/y .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("workflow pushing via --output type=image,push=true is treated as a publisher", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --output type=image,name=ghcr.io/x/y,push=true .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher with a step-level IMAGE_NAME override is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + " steps:\n - env:\n IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core", + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("internal image publisher reassigning IMAGE_NAME in shell is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + ' steps:\n - run: echo "IMAGE_NAME=ghcr.io/atomicstrata/atomicmemory-core" >> "$GITHUB_ENV"', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not be reassigned/.test(failure))); +}); + +test("internal image publisher using raw output exporters is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace("--push .", "--output=type=registry ."); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /raw buildx output exporters/.test(failure))); +}); + +test("internal image publisher overriding IMAGE_NAME via YAML anchor/alias is rejected", () => { + const yaml = [ + "env:", + " IMAGE_NAME: &pin ghcr.io/atomicstrata/atomicmemory-core-internal", + "jobs:", + " publish:", + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " steps:", + " - env:", + " IMAGE_NAME: *pin", + ' - run: docker buildx build --tag "${IMAGE_NAME}:sha-abc1234" --push .', + "", + ].join("\n"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("workflow pushing via docker/build-push-action is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - uses: docker/build-push-action@v6\n with:\n push: true\n tags: ghcr.io/x/y:latest\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher using docker/build-push-action is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " - uses: docker/build-push-action@v6\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /docker\/build-push-action is forbidden/.test(failure))); +}); + +test("unparseable workflow YAML fails the image-publisher policy closed", () => { + const failures = checkImagePublisherText("jobs: [unclosed\n {bad", ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /failed to parse/.test(failure))); +}); + +test("internal image publisher using compact -tVALUE tagging is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', "-tghcr.io/atomicstrata/atomicmemory-core:latest"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("workflow pushing via compact -otype=registry is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - run: docker buildx build -otype=registry,name=ghcr.io/x/y .\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("workflow using a case-variant build-push-action ref is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - uses: Docker/build-push-action@v6\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher with a compound OR repository guard is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " if: github.repository == 'atomicstrata/atomicmemory-internal' || github.repository == 'atomicstrata/atomicmemory'", + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /guarded with exactly/.test(failure))); +}); + +test("internal image publisher interpolating GitHub expressions into run is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + '--tag "${IMAGE_NAME}:sha-abc1234"', + '--tag "${IMAGE_NAME}:sha-abc1234" --platform "${{ inputs.platforms }}"', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not interpolate/.test(failure))); +}); + +test("internal image publisher using a non-allowlisted action is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " - uses: actions/setup-node@v4\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /action allowlist/.test(failure))); +}); + +test("workflow pushing via docker image push or skopeo copy is treated as a publisher", () => { + for (const cmd of ["docker image push ghcr.io/x/y", "skopeo copy oci:img docker://ghcr.io/x/y", "podman push ghcr.io/x/y", "docker compose push"]) { + const failures = checkImagePublisherText(`jobs:\n x:\n steps:\n - run: ${cmd}\n`, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure)), `expected publisher classification for: ${cmd}`); + } +}); + +test("a publish-*.yaml (yaml extension) pushing images is still held to image-publisher policy", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --push .\n", ".github/workflows/publish-evil.yaml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("steps inherited through a YAML merge key are still scanned", () => { + const yaml = [ + "x: &tpl", + " steps:", + " - run: docker buildx build --push .", + "jobs:", + " evil:", + " <<: *tpl", + "", + ].join("\n"); + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("non-publish workflow calling a publishing reusable workflow is rejected", () => { + const yaml = "jobs:\n x:\n uses: ./.github/workflows/publish-core-docker.yml\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /calls publishing reusable workflow/.test(failure))); +}); + +test("internal image publisher delegating to any reusable workflow is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " delegate:\n if: github.repository == 'atomicstrata/atomicmemory-internal'\n uses: ./.github/workflows/other.yml\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not call a reusable workflow/.test(failure))); +}); + +test("composite action containing a push sink is rejected", () => { + const yaml = "name: helper\nruns:\n using: composite\n steps:\n - run: crane cp img ghcr.io/x/y\n shell: bash\n"; + const failures = checkCompositeActionText(yaml, ".github/actions/helper/action.yml"); + assert.ok(failures.some((failure) => /composite actions must not push/.test(failure))); +}); + +test("composite action without push sinks passes", () => { + const yaml = "name: helper\nruns:\n using: composite\n steps:\n - run: docker build -t local/img .\n shell: bash\n"; + assert.deepEqual(checkCompositeActionText(yaml, ".github/actions/helper/action.yml"), []); +}); + function validManifest(overrides = {}) { return { name: "@atomicmemory/sdk", diff --git a/scripts/ci/release-policy.mjs b/scripts/ci/release-policy.mjs index 543521b..bd0fc43 100644 --- a/scripts/ci/release-policy.mjs +++ b/scripts/ci/release-policy.mjs @@ -12,11 +12,21 @@ * - Every npm-published package sets publishConfig.access=public and * declares no file: / link: dependencies. * - CODEOWNERS covers the publish workflow file. + * - Container-image publishers are explicitly enumerated: any workflow that + * pushes images (docker push / buildx --push / output exporters / + * imagetools create / docker/build-push-action) must be either a + * publish-*.yml release lane (covered by the invariants above) or the + * enumerated internal operator publisher. That publisher's workflow YAML + * is parsed structurally and must guard every job on the + * atomicmemory-internal repository, assign IMAGE_NAME exactly once at the + * workflow level (pinned to the private internal package), and push only + * via buildx --push with every --tag deriving from that pin. */ import { readFileSync, readdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; const here = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(here, "../.."); @@ -24,6 +34,29 @@ const PUBLISH_WORKFLOW = ".github/workflows/publish-packages.yml"; const CODEOWNERS_FILE = ".github/CODEOWNERS"; const GUARD_REL_PATH = "scripts/guards/guard-npm-publish.mjs"; const PUBLISH_WORKFLOW_FILENAME_PREFIX = "publish-"; +const INTERNAL_IMAGE_WORKFLOW_FILENAME = "internal-core-docker-image.yml"; +const INTERNAL_IMAGE_NAME = "ghcr.io/atomicstrata/atomicmemory-core-internal"; +// The job-level `if` must equal this exactly (whitespace-normalized): a +// compound condition (e.g. `A || B`) could satisfy a substring match while +// still running in the mirrored public repository. +const INTERNAL_REPO_GUARD_EXACT = "github.repository == 'atomicstrata/atomicmemory-internal'"; +// --output=type=registry and --output type=image,push=true are buildx's +// long-form equivalents of --push; treat them as the same sink. +const OUTPUT_EXPORTER_RE = /(^|\s)(-o|--output)[=\s][^\n]*\b(type=registry|push=true)\b/; +// Non-buildx registry-push tools available on hosted runners. +const OTHER_PUSH_SINKS_RE = /\b(docker\s+image\s+push|docker\s+compose\s+push|docker-compose\s+push|podman\s+(image\s+)?push|buildah\s+push|skopeo\s+(copy|sync)|crane\s+(push|cp|copy)|oras\s+push)\b/; +// Reusable-workflow refs that publish; a job-level `uses:` of one of these +// from outside the audited release lane would launder a publish. +const PUBLISHING_WORKFLOW_REF_RE = /(^|\/)(publish-[^/@\s]*\.ya?ml|internal-core-docker-image\.yml)(@|$)/i; +const EXPRESSION_MARKER = "$" + "{{"; +const COMPOSITE_SCAN_SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", ".turbo", ".worktrees"]); +// Actions the enumerated internal publisher may use; anything else (any +// case) is a policy failure so a new action is an explicit policy change. +const INTERNAL_ALLOWED_ACTIONS = [ + "actions/checkout@", + "docker/setup-qemu-action@", + "docker/setup-buildx-action@", +]; const PUBLISHED_PACKAGE_PATHS = [ "packages/core/package.json", "packages/sdk/package.json", @@ -55,11 +88,240 @@ export function runPolicy(options) { const root = options.repoRoot; const failures = []; failures.push(...checkPublishWorkflows(root)); + failures.push(...checkImagePublishers(root)); failures.push(...checkPublishedPackages(root)); failures.push(...checkCodeownersCovers(root)); return failures; } +function checkImagePublishers(root) { + const failures = []; + for (const workflowPath of listWorkflowPaths(root)) { + const basename = workflowPath.split("/").pop(); + // Skip only what checkPublishWorkflows actually covers (publish-*.yml); + // a publish-*.yaml would otherwise escape both checks. + if (basename.startsWith(PUBLISH_WORKFLOW_FILENAME_PREFIX) && basename.endsWith(".yml")) continue; + const text = safeRead(root, workflowPath); + if (text === null) continue; + failures.push(...checkImagePublisherText(text, workflowPath)); + } + for (const actionPath of listCompositeActionPaths(root)) { + const text = safeRead(root, actionPath); + if (text === null) continue; + failures.push(...checkCompositeActionText(text, actionPath)); + } + return failures; +} + +/** + * Composite actions (action.yml) can run shell of their own; none may + * contain a push sink -- a workflow step's `uses: ./path` would otherwise + * publish without anything visible in the workflow file. + */ +export function checkCompositeActionText(rawText, filename) { + let doc; + try { + doc = parseYaml(rawText, { merge: true }) ?? {}; + } catch (error) { + return [`${filename}: action YAML failed to parse, so image-publisher policy cannot validate it (${error.message}).`]; + } + const steps = Array.isArray(doc?.runs?.steps) ? doc.runs.steps : []; + const shellText = steps + .map((step) => (typeof step?.run === "string" ? step.run.replace(/\\\r?\n\s*/g, " ") : "")) + .join("\n") + .replace(/(^|[\s"'])-([to])(?=[^\s=])/gm, "$1-$2 "); + const usesPushAction = steps.some( + (step) => typeof step?.uses === "string" && step.uses.toLowerCase().startsWith("docker/build-push-action"), + ); + if (pushesContainerImages(shellText) || usesPushAction) { + return [`${filename}: composite actions must not push container images; only publish-*.yml release lanes and ${INTERNAL_IMAGE_WORKFLOW_FILENAME} may publish.`]; + } + return []; +} + +function listCompositeActionPaths(root) { + const results = []; + const walk = (relDir) => { + let entries; + try { + entries = readdirSync(resolve(root, relDir), { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + const enter = entry.name === ".github" || (!COMPOSITE_SCAN_SKIP_DIRS.has(entry.name) && !entry.name.startsWith(".")); + if (enter) walk(relPath); + } else if (entry.name === "action.yml" || entry.name === "action.yaml") { + results.push(relPath); + } + } + }; + walk(""); + return results; +} + +/** + * Structured image-publisher validation. The workflow YAML is parsed (so + * quoted keys, flow mappings, anchors, and aliases cannot hide an + * assignment), run scripts are extracted per step with backslash-newline + * continuations joined (so no sink can be split across lines), and the + * destination contract is a positive assertion: one workflow-level + * IMAGE_NAME pin, every --tag derived from it, buildx --push as the only + * push path. A workflow that fails to parse fails the policy. + */ +export function checkImagePublisherText(rawText, filename) { + let doc; + try { + // merge: true resolves YAML merge keys (<<: *anchor) the way a runner + // would, so inherited steps cannot hide from the scan below. + doc = parseYaml(rawText, { merge: true }) ?? {}; + } catch (error) { + return [`${filename}: workflow YAML failed to parse, so image-publisher policy cannot validate it (${error.message}).`]; + } + if (typeof doc !== "object") { + return [`${filename}: workflow YAML is not a mapping, so image-publisher policy cannot validate it.`]; + } + + const jobs = Object.entries(doc.jobs ?? {}); + const runScripts = []; + const usedActions = []; + const jobLevelUses = []; + for (const [jobName, job] of jobs) { + if (typeof job?.uses === "string") jobLevelUses.push({ jobName, ref: job.uses }); + for (const step of Array.isArray(job?.steps) ? job.steps : []) { + if (typeof step?.run === "string") runScripts.push(step.run.replace(/\\\r?\n\s*/g, " ")); + if (typeof step?.uses === "string") usedActions.push(step.uses); + } + } + + const failuresEarly = []; + // A job-level `uses:` of a publishing reusable workflow is itself a + // publish path: only publish-*.yml release lanes may call one, and the + // enumerated internal publisher may not delegate to reusable workflows + // at all. + for (const { jobName, ref } of jobLevelUses) { + if (filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { + failuresEarly.push(`${filename}: job '${jobName}' must not call a reusable workflow (uses: ${ref}); the internal image publisher defines its own steps only.`); + } else if (PUBLISHING_WORKFLOW_REF_RE.test(ref)) { + failuresEarly.push(`${filename}: job '${jobName}' calls publishing reusable workflow ${ref}; only publish-*.yml release lanes may do that.`); + } + } + if (failuresEarly.length > 0 && !filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) return failuresEarly; + // Docker accepts compact short-option forms (-tVALUE, -oVALUE); split them + // so the sink and destination scans below see the canonical spaced form. + const shellText = runScripts.join("\n").replace(/(^|[\s"'])-([to])(?=[^\s=])/gm, "$1-$2 "); + // GitHub resolves action owner/repo case-insensitively. + const usesPushAction = usedActions.some((action) => action.toLowerCase().startsWith("docker/build-push-action")); + + if (!pushesContainerImages(shellText) && !usesPushAction) return failuresEarly; + if (!filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { + return [ + `${filename}: pushes container images but is neither a ${PUBLISH_WORKFLOW_FILENAME_PREFIX}*.yml release lane nor the enumerated internal image publisher (${INTERNAL_IMAGE_WORKFLOW_FILENAME}).`, + ]; + } + + const failures = failuresEarly; + // (1) Every job must carry the repository guard as its EXACT `if` + // condition; substring matching would accept compound conditions that + // also run in the mirrored public repository. + for (const [jobName, job] of jobs) { + const condition = typeof job?.if === "string" ? job.if.replace(/\s+/g, " ").trim() : ""; + if (condition !== INTERNAL_REPO_GUARD_EXACT) { + failures.push(`${filename}: job '${jobName}' must be guarded with exactly if: ${INTERNAL_REPO_GUARD_EXACT} (found: ${condition || "none"}).`); + } + } + // (2) Exactly one IMAGE_NAME env assignment may exist, at the workflow + // level, pinned to the internal package. The parser resolves quoted + // keys, flow mappings, and anchor/alias tricks before we count. + const assignments = collectImageNameAssignments(doc, jobs); + const pin = assignments.length === 1 ? assignments[0] : undefined; + if (!pin || pin.where !== "workflow env" || pin.value !== INTERNAL_IMAGE_NAME) { + const found = assignments.map((a) => `${a.where}=${a.value}`).join(", ") || "none"; + failures.push(`${filename}: must assign env IMAGE_NAME exactly once, at the workflow level, pinned to ${INTERNAL_IMAGE_NAME} (found: ${found}).`); + } + // (3) No shell-side reassignment (IMAGE_NAME=... in run blocks or + // GITHUB_ENV writes). + if (/\bIMAGE_NAME\s*=/.test(shellText)) { + failures.push(`${filename}: IMAGE_NAME must not be reassigned in shell or GITHUB_ENV; the single env pin is the only allowed assignment.`); + } + // (4) Every --tag / -t argument must reference ${IMAGE_NAME} directly, so + // pushes cannot be routed through another variable or a literal path. + for (const match of shellText.matchAll(/(?:^|[\s"'])(?:--tag|-t)["'=\s]+([^\s"']+)/g)) { + if (!/^\$\{IMAGE_NAME\}[:@]/.test(match[1])) { + failures.push(`${filename}: every --tag must derive from \${IMAGE_NAME} directly (found '--tag ${match[1]}').`); + } + } + // (5) The only allowed push path is a buildx --push run step: raw output + // exporters, docker push, imagetools create, and docker/build-push-action + // would all publish without a --tag this policy can pin. + if (OUTPUT_EXPORTER_RE.test(shellText)) { + failures.push(`${filename}: raw buildx output exporters (--output/-o type=registry or push=true) are forbidden; push only via --push with --tag \${IMAGE_NAME}.`); + } + if (/\bdocker\s+push\b/.test(shellText) || /\bimagetools\s+create\b/.test(shellText) || OTHER_PUSH_SINKS_RE.test(shellText)) { + failures.push(`${filename}: docker push, imagetools create, and non-buildx push tools (podman/buildah/skopeo/crane/oras/compose push) are forbidden here; they publish without a --tag the policy can pin. Push only via buildx --push.`); + } + if (usesPushAction) { + failures.push(`${filename}: docker/build-push-action is forbidden here; push only via a buildx --push run step so the --tag contract applies.`); + } + // (6) Run scripts must not interpolate GitHub expressions: text from + // dispatch inputs or the checked-out branch (e.g. a package.json + // version) would be substituted into shell before it runs. Dynamic + // values must reach shell as step env instead. + if (shellText.includes(EXPRESSION_MARKER)) { + failures.push(`${filename}: run scripts must not interpolate ${EXPRESSION_MARKER} ... }} GitHub expressions; pass dynamic values via step env so inputs and branch content cannot inject into shell.`); + } + // (7) Only enumerated actions may be used, so no third-party or local + // composite action can push on this workflow's behalf. + for (const action of usedActions) { + if (!INTERNAL_ALLOWED_ACTIONS.some((allowed) => action.toLowerCase().startsWith(allowed))) { + failures.push(`${filename}: uses: ${action} is not in the internal image publisher's action allowlist (${INTERNAL_ALLOWED_ACTIONS.join(", ")}).`); + } + } + return failures; +} + +function collectImageNameAssignments(doc, jobs) { + const assignments = []; + const collect = (env, where) => { + if (!env || typeof env !== "object") return; + for (const [key, value] of Object.entries(env)) { + if (key === "IMAGE_NAME") assignments.push({ where, value: String(value) }); + } + }; + collect(doc.env, "workflow env"); + for (const [jobName, job] of jobs) { + collect(job?.env, `job '${jobName}' env`); + collect(job?.container?.env, `job '${jobName}' container env`); + (Array.isArray(job?.steps) ? job.steps : []).forEach((step, index) => { + collect(step?.env, `job '${jobName}' step ${index + 1} env`); + }); + } + return assignments; +} + +function pushesContainerImages(text) { + return ( + /(^|\s)--push\b/.test(text) || + /\bdocker\s+push\b/.test(text) || + /\bimagetools\s+create\b/.test(text) || + OUTPUT_EXPORTER_RE.test(text) || + OTHER_PUSH_SINKS_RE.test(text) + ); +} + +function listWorkflowPaths(root) { + const dir = resolve(root, ".github/workflows"); + try { + return readdirSync(dir) + .filter((name) => name.endsWith(".yml") || name.endsWith(".yaml")) + .map((name) => `.github/workflows/${name}`); + } catch { + return []; + } +} + function checkPublishWorkflows(root) { const failures = []; for (const workflowPath of listPublishWorkflowPaths(root)) { @@ -148,7 +410,10 @@ function findUnpublishableDependencies(packageJson, filename) { function checkCodeownersCovers(root) { const text = safeRead(root, CODEOWNERS_FILE); if (text === null) return [`${CODEOWNERS_FILE}: missing; required to own ${PUBLISH_WORKFLOW}.`]; - return checkCodeownersText(text, PUBLISH_WORKFLOW); + return [ + ...checkCodeownersText(text, PUBLISH_WORKFLOW), + ...checkCodeownersText(text, `.github/workflows/${INTERNAL_IMAGE_WORKFLOW_FILENAME}`), + ]; } export function checkCodeownersText(text, requiredPath) {