diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 022beea..6fa8ea1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,15 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} + # release-policy.mjs parses workflow YAML with the root `yaml` + # devDependency, so this job needs an install (it previously ran on a + # bare checkout). + - name: Setup pnpm + run: | + corepack enable + corepack prepare pnpm@${PNPM_VERSION} --activate + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts - name: Run release-policy guardrails run: node scripts/ci/release-policy.mjs - name: Run release-policy unit tests diff --git a/.github/workflows/internal-core-docker-image.yml b/.github/workflows/internal-core-docker-image.yml new file mode 100644 index 0000000..70bac7b --- /dev/null +++ b/.github/workflows/internal-core-docker-image.yml @@ -0,0 +1,275 @@ +name: Internal Core Docker Image + +# Internal-only convenience lane. Builds packages/core/Dockerfile from any ref +# of atomicmemory-internal and pushes it to the PRIVATE package +# ghcr.io/atomicstrata/atomicmemory-core-internal for internal testing. +# +# This is NOT the release pipeline. The official image +# ghcr.io/atomicstrata/atomicmemory-core is published exclusively by +# publish-core-docker.yml running on the public repo, anchored to the +# published @atomicmemory/core npm package. Nothing here may ever push to the +# official package name, and the tag guard below refuses ref-derived tags that +# are release-shaped (semver with or without prerelease/build suffix) or that +# collide with a reserved alias, so an internal build can never masquerade as +# a release. Builds of main additionally publish the curated floating aliases +# `internal-latest` and `latest`; both live only on the private internal +# package and always point at the same digest. +# +# Naming contract with release-policy: scripts/ci/release-policy.mjs forbids +# workflow_dispatch on every `.github/workflows/publish-*.yml` because those +# are release lanes that must only be triggerable by the audited +# repository_dispatch release flow. This workflow is deliberately named +# OUTSIDE that prefix: it is operator-dispatchable by design and never touches +# release artifacts or release tag namespaces. If this file is ever renamed to +# publish-*.yml, release-policy will (correctly) fail it. +# +# Repo guard: like sync-to-private.yml, this file is mirrored into the public +# repo by the public export, so the job runs ONLY on +# atomicstrata/atomicmemory-internal. Keep this file free of anything +# confidential; it is public. + +on: + workflow_dispatch: + inputs: + ref: + description: "Branch, tag, or SHA of atomicmemory-internal to build" + required: false + default: main + type: string + platforms: + description: "Comma-separated buildx platforms" + required: false + default: linux/amd64,linux/arm64 + type: string + push: + branches: + - main + # Mirrors the core-image-affecting classification used by ci.yml's + # core-docker-smoke job (packages/core + root build-context inputs), + # plus this workflow itself. + paths: + - "packages/core/**" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "package.json" + - "turbo.json" + - ".github/workflows/internal-core-docker-image.yml" + +permissions: + contents: read + packages: write + +concurrency: + group: internal-core-docker-image-${{ inputs.ref || github.ref_name }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core-internal + DEFAULT_PLATFORMS: linux/amd64,linux/arm64 + +jobs: + publish: + name: build and push internal core image + if: github.repository == 'atomicstrata/atomicmemory-internal' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Checkout requested ref + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + # The core image is several GB per platform and the verify step loads + # each platform's image into dockerd on top of the buildx cache; the + # standard runner's free disk is not enough (the first run died on + # ENOSPC six minutes in). Reclaim the runner's preinstalled bloat + # before building. + - name: Free runner disk space + run: | + set -euo pipefail + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/share/boost + sudo docker image prune --all --force >/dev/null + df -h / + + - name: Compute image tags + id: tags + env: + RAW_REF: ${{ inputs.ref || github.ref_name }} + run: | + set -euo pipefail + + sha="$(git rev-parse HEAD)" + short_sha="${sha:0:7}" + core_version="$(node -p "require('./packages/core/package.json').version")" + + # Slugify the ref into a valid docker tag: lowercase, restricted + # charset, no leading separator, bounded length. + slug="$(tr '[:upper:]' '[:lower:]' <<<"${RAW_REF}" | sed -e 's![^a-z0-9._-]!-!g' -e 's!^[.-]*!!' | cut -c1-100)" + + tags=("sha-${short_sha}") + + # Both guards below apply to the REF-DERIVED slug only. The curated + # aliases added afterwards are hardcoded and intentional; the slug is + # whatever a branch happens to be named, so it is the only untrusted + # tag source and must fail closed. + # + # Skip the slug tag when the ref was itself a commit sha; sha- + # already covers it. + if [[ -n "${slug}" && ! "${slug}" =~ ^[0-9a-f]{7,40}$ ]]; then + # Reserved namespaces: a branch named latest or internal-latest + # could hijack a floating main alias, and sha-*/verify-* could + # overwrite another commit's tag or a verify scratch tag. + if [[ "${slug}" == "latest" || "${slug}" == "internal-latest" || "${slug}" =~ ^(sha|verify)- ]]; then + echo "::error::Ref slug '${slug}' collides with a reserved tag namespace (latest, internal-latest, sha-*, verify-*). Rename the branch or dispatch with the commit sha instead." + exit 1 + fi + # Release shapes: bare majors (1 / v2, the official floating-alias + # shape), bare semver (1.2 / 1.2.3 / v1.2.3), and semver with any + # prerelease or build suffix (1.2.3-rc.1, v1.2.3-beta.2, 1.2.3+abc). + if [[ "${slug}" =~ ^v?[0-9]+$ || "${slug}" =~ ^v?[0-9]+(\.[0-9]+){1,2}([-+.].*)?$ ]]; then + echo "::error::Ref slug '${slug}' looks like a release tag; internal builds must not publish semver-shaped tags. Rename the ref or dispatch with a different ref." + exit 1 + fi + tags+=("${slug}") + fi + + # Curated floating aliases for main. `latest` exists so a bare + # `docker pull ${IMAGE_NAME}` (which resolves to :latest) works; it + # is published only on this PRIVATE internal package and always + # points at the same digest as internal-latest. The official + # ghcr.io/atomicstrata/atomicmemory-core:latest is published solely + # by publish-core-docker.yml on the public repo and is untouched. + if [[ "${slug}" == "main" ]]; then + tags+=("internal-latest" "latest") + fi + + tag_args=() + refs=() + for tag in "${tags[@]}"; do + tag_args+=("--tag" "${IMAGE_NAME}:${tag}") + refs+=("${IMAGE_NAME}:${tag}") + done + + { + echo "sha=${sha}" + echo "short_sha=${short_sha}" + echo "core_version=${core_version}" + echo "tag_args=${tag_args[*]}" + echo "refs=${refs[*]}" + } >>"${GITHUB_OUTPUT}" + + echo "Building @atomicmemory/core ${core_version} at ${sha}" + printf 'Will push:\n' + printf ' - %s\n' "${refs[@]}" + + - name: Login to GHCR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + + # Dynamic values reach the run scripts below ONLY via step env, never + # by interpolating ${{ ... }} into shell text: a dispatch input or a + # hostile branch's package.json version could otherwise inject shell. + # release-policy enforces this (no ${{ in run scripts of this file). + - name: Build and verify every requested platform + # Every platform that will be pushed is first built locally and + # verified (deployed package version readable via node). No platform + # selection can bypass verification; non-native platforms run under + # QEMU, and their build layers are reused by the push step via the + # shared buildx cache. + env: + PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }} + SHORT_SHA: ${{ steps.tags.outputs.short_sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + run: | + set -euo pipefail + + IFS=',' read -r -a platforms <<<"${PLATFORMS}" + for platform in "${platforms[@]}"; do + platform="$(tr -d '[:space:]' <<<"${platform}")" + verify_suffix="verify-${SHORT_SHA}-${platform//\//-}" + + docker buildx build \ + --platform "${platform}" \ + --file packages/core/Dockerfile \ + --load \ + --tag "${IMAGE_NAME}:${verify_suffix}" \ + . + + image_version="$(docker run --rm --platform "${platform}" --entrypoint node "${IMAGE_NAME}:${verify_suffix}" -p "require('./package.json').version")" + if [[ "${image_version}" != "${CORE_VERSION}" ]]; then + echo "::error::${platform} image package version is ${image_version}, expected ${CORE_VERSION}" + exit 1 + fi + echo "${platform}: verified @atomicmemory/core ${image_version}" + + # Free dockerd storage before the next platform loads; the push + # step rebuilds from the (kept) buildx cache, not from dockerd. + docker image rm --force "${IMAGE_NAME}:${verify_suffix}" >/dev/null + done + + - name: Build and push internal image + env: + PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }} + TAG_ARGS: ${{ steps.tags.outputs.tag_args }} + GIT_SHA: ${{ steps.tags.outputs.sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + run: | + set -euo pipefail + + read -r -a tag_args <<<"${TAG_ARGS}" + + docker buildx build \ + --platform "${PLATFORMS}" \ + --file packages/core/Dockerfile \ + --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GIT_SHA}" \ + --label "org.opencontainers.image.version=${CORE_VERSION}" \ + --label "org.opencontainers.image.title=@atomicmemory/core (internal test build)" \ + "${tag_args[@]}" \ + --push \ + . + + - name: Report published image + env: + GIT_SHA: ${{ steps.tags.outputs.sha }} + SHORT_SHA: ${{ steps.tags.outputs.short_sha }} + CORE_VERSION: ${{ steps.tags.outputs.core_version }} + REFS: ${{ steps.tags.outputs.refs }} + run: | + set -euo pipefail + + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:sha-${SHORT_SHA}" --format '{{ json .Manifest }}' | jq -r '.digest')" + + { + echo "## Internal core image published" + echo "" + echo "- revision: \`${GIT_SHA}\`" + echo "- package version: \`${CORE_VERSION}\`" + echo "- digest-pinned ref (immutable, prefer this for sharing): \`${IMAGE_NAME}@${digest}\`" + echo "- tags (mutable, a rebuild of the same ref moves them):" + for image_ref in ${REFS}; do + echo " - \`${image_ref}\`" + done + echo "" + echo "Ops integration-smoke usage (from the smoke harness directory, after docker login ghcr.io):" + echo "" + echo '```' + echo "CORE_SOURCE_MODE=image \\" + echo "CORE_IMAGE_REF=${IMAGE_NAME}@${digest} \\" + echo "CORE_PACKAGE_VERSION=${CORE_VERSION} \\" + echo " bash ci/run-required.sh" + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" diff --git a/package.json b/package.json index 8280400..d275e9c 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,8 @@ "@typescript-eslint/parser": "^8.59.3", "eslint": "9.39.1", "fallow": "2.75.0", - "turbo": "2.9.14" + "turbo": "2.9.14", + "yaml": "^2.8.3" }, "pnpm": { "overrides": { diff --git a/packages/core/.env.example b/packages/core/.env.example index 236e7aa..7c1310c 100644 --- a/packages/core/.env.example +++ b/packages/core/.env.example @@ -22,8 +22,42 @@ OPENAI_API_KEY= # Shared API key clients must send as `Authorization: Bearer `. # Rotate by restarting the server with a new value. # Generate with: openssl rand -hex 32 -# Docker image local mode defaults this to `local-dev-key` when omitted. -CORE_API_KEY=replace-with-a-strong-random-secret +# Docker image local mode: auto-generated and persisted to +# $CORE_STATE_DIR/core-api-key (default /var/lib/atomicmemory/state/core-api-key) +# when omitted. Required in production/staging (RAW_STORAGE_DEPLOYMENT_ENV). +# CORE_API_KEY=replace-with-a-strong-random-secret + +# Optional: verify Cloud-issued RS256 JWTs for connected-local console access. +# JWKS_URL + ISSUER + AUDIENCE must be set together; when unset, only +# CORE_API_KEY is accepted. +# CLOUD_JWKS_URL=https://api.dev.atomicstrata.ai/.well-known/atomic-core/jwks.json +# CLOUD_JWT_ISSUER=https://api.dev.atomicstrata.ai +# CLOUD_JWT_AUDIENCE=atomicmemory-core +# Optional project binding. When set, tokens for other projects are rejected. +# When omitted (single-key local), Core trusts the token's own project_id. +# CLOUD_PROJECT_ID=proj_your_connected_local_project +# Temporary until Cloud mint includes memory_user_id (dev PoC uses user "default"): +# CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default +# Optional — when false (default), CORE_API_KEY is rejected once JWT verify is enabled. +# CLOUD_JWT_STATIC_KEY_FALLBACK=false + +# --- Connected-local (Docker entrypoint defaults) --- +# Minimal self-hosted + Cloud dev console: set only these in docker run. +# OPENAI_API_KEY=sk-... +# ATOMICMEMORY_API_KEY=amc_live_... +# Mount atomic-memory-state (or equivalent) so the entrypoint can persist +# an auto-generated CORE_API_KEY at /var/lib/atomicmemory/state/core-api-key. +# Retrieve after start: docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key +# Optional override: CORE_API_KEY=your-local-secret (skips auto-generation) +# Setting ATOMICMEMORY_API_KEY turns connected-local on; the Docker entrypoint +# then applies tier defaults when unset (CLOUD_ENV=dev): +# ATOMICMEMORY_API_URL, CLOUD_TRACE_SYNC_ENABLED=true, +# CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, CLOUD_JWT_AUDIENCE=atomicmemory-core, +# ALLOWED_ORIGINS, CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default, +# CLOUD_JWT_STATIC_KEY_FALLBACK=true +# CLOUD_PROJECT_ID is optional: set it to pin one project; when omitted Core +# trusts the Cloud-minted token's own project_id. Override tier with +# CLOUD_ENV=staging|production # Trusted-proxy identity guard (Radar C4). The shared CORE_API_KEY above # authenticates the *caller process*, not the end user; `user_id` is @@ -206,3 +240,21 @@ EMBEDDING_DIMENSIONS=1536 # On Railway, DATABASE_URL is injected by the Postgres plugin. # Set OPENAI_API_KEY in Railway service variables. # PORT is injected automatically by Railway. + +# --- Optional Cloud trace sync (OSS Core → Cloud observability) --- +# Disabled when all vars are absent, or when CLOUD_TRACE_SYNC_ENABLED=false. +# Partial configuration (some vars set without ENABLED=true) fails startup. +# CLOUD_TRACE_SYNC_ENABLED=true +# ATOMICMEMORY_API_URL=https://api.atomicstrata.ai +# ATOMICMEMORY_API_KEY=amc_replace_me +# CORE_INSTANCE_ID=optional-stable-installation-id +# CLOUD_TRACE_SYNC_BATCH_SIZE=100 +# CLOUD_TRACE_SYNC_MAX_ATTEMPTS=8 +# CLOUD_TRACE_SYNC_BASE_RETRY_MS=1000 +# CLOUD_TRACE_SYNC_MAX_RETRY_MS=300000 +# CLOUD_TRACE_SYNC_POLL_INTERVAL_MS=5000 +# CLOUD_TRACE_SYNC_SENT_RETENTION_MS=86400000 +# CLOUD_TRACE_SYNC_SHUTDOWN_DRAIN_MS=10000 +# CLOUD_TRACE_SYNC_CLAIM_STALE_MS=300000 +# CLOUD_TRACE_SYNC_DEAD_LETTER_RETENTION_MS=604800000 +# CLOUD_TRACE_SYNC_MAX_PENDING=10000 diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 7ce4a1d..febdff3 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -6,6 +6,34 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +## [1.2.0] - 2026-07-28 + +### Added +- Connected-local Cloud support with optional Cloud-issued RS256 JWT + verification, user/project claim binding, and static API-key fallback. +- Durable, content-free memory trace synchronization through a PostgreSQL + outbox with bounded backlog, retry, dead-letter, and auth-recovery handling. +- Automatic generation and persistence of local `CORE_API_KEY` values and a + stable Core instance ID, including connected-local Docker defaults. +- `EMBEDDING_DTYPE` configuration for larger quantized local embedding models. +- A background scheduler for deferred AUDN reconciliation. + +### Changed +- Deferred AUDN reconciliation now runs with bounded parallelism. + +### Fixed +- Anthropic extraction requests use an assistant prefill to enforce JSON + responses. +- Idempotent re-indexing advances extraction status only through valid state + transitions. +- Local API-key overrides persist across container restarts. + +### Security +- Cloud JWT verification fails closed on issuer, audience, algorithm, user, and + project mismatches. +- Outbound Cloud trace envelopes use allowlisted summary fields and exclude + verbatim memory and query content. + ## [1.1.1] - 2026-06-15 ### Added diff --git a/packages/core/README.md b/packages/core/README.md index f3f1a71..3918dd6 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -112,6 +112,46 @@ EOF docker compose -f docker-compose.image.yml up ``` +### Connected to Atomic Memory Cloud + +Link a local Core instance to a Cloud **Local** project with a single `amc_…` API +key (traces + heartbeat). Apps on localhost use **`CORE_API_KEY`** (Local access +key) — never paste the Cloud key into SDK/CLI calls. + +1. In Cloud: create a Local project → create an API key (`amc_…`). +2. Copy the console **Connect** `docker run` snippet (or use the template below). +3. After Core starts, retrieve the auto-generated local key (or use your own override). +4. SDK/CLI: `Authorization: Bearer $CORE_API_KEY` → `http://127.0.0.1:17350`. +5. Console **Memories** use an automatic Cloud-minted JWT (no key paste). + +```bash +export OPENAI_API_KEY=sk-... +export ATOMICMEMORY_API_KEY=amc_... # Cloud connect only +export ATOMICMEMORY_API_URL=https://api.atomicstrata.ai + +docker run -d --name atomic-memory \ + --restart unless-stopped \ + -p 127.0.0.1:17350:17350 \ + -v atomic-memory-data:/var/lib/atomicmemory/postgres \ + -v atomic-memory-state:/var/lib/atomicmemory/state \ + -e OPENAI_API_KEY \ + -e ATOMICMEMORY_API_KEY \ + -e ATOMICMEMORY_API_URL \ + ghcr.io/atomicstrata/atomicmemory-core:stable + +# Retrieve the generated local access key for SDK/CLI (after /health is ok): +docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key +``` + +**Optional:** pass `-e CORE_API_KEY=your-secret` to pin a user-defined local key +instead of auto-generation. The state volume still persists whichever key is in use. + +When `ATOMICMEMORY_API_KEY` is set, Core enables outbound trace sync and +periodic heartbeat to `POST /v1/runtimes/heartbeat`. Invalid or revoked Cloud +credentials degrade observability only — local memory ingest/search keep working. + +Local-only (no Cloud): run with only `OPENAI_API_KEY` — same image, no Cloud env. + ### Docker from source ```bash diff --git a/packages/core/SECURITY.md b/packages/core/SECURITY.md index 9aa68be..93091e2 100644 --- a/packages/core/SECURITY.md +++ b/packages/core/SECURITY.md @@ -106,9 +106,52 @@ Non-boolean values for `TRUSTED_PROXY_MODE` are rejected at startup in all envs. To run a hosted deployment without the guard you must change the deployment env, not silently disable the guard. +### Cloud JWT verification (optional, connected-local) + +When `CLOUD_JWKS_URL`, `CLOUD_JWT_ISSUER`, and `CLOUD_JWT_AUDIENCE` are all set, +core accepts short-lived RS256 JWTs minted by Atomic Strata Cloud (instead of +`CORE_API_KEY` unless fallback is explicitly enabled). Verified claims inject +`X-AtomicMemory-Asserted-User` from `memory_user_id` (not `sub`) and workspace +from `project_id`. When a request carries wire `user_id` (body, query, or +`X-AtomicMemory-User-Id`), it must match the token's `memory_user_id` or the +request fails closed with 403. Routers that parse JSON after auth (for example +`/v1/documents`) re-check this binding once the body is available. JWT-scoped +mutating routes without an explicit `user_id` default to the asserted identity +(for example `POST /v1/memories/reconcile` never runs a global reconcile for a +JWT caller). When `CLOUD_PROJECT_ID` is set, token `project_id` must match it; +when omitted, Core trusts the token's own `project_id` claim (single-tenant +local posture). The JWKS trio must be set together; partial configuration fails +startup. + +`CLOUD_JWT_STATIC_KEY_FALLBACK=true` preserves the legacy `CORE_API_KEY` +acceptance path for non-JWT bearer tokens on connected-local deployments. When +false or unset, non-JWT bearer tokens are rejected once Cloud JWT verification +is enabled. + +Core prefetches JWKS at startup; JWT auth remains degraded until keys load. + +### Connected-local local access key (Docker) + +For self-hosted Docker (`RAW_STORAGE_DEPLOYMENT_ENV=local`), the entrypoint +generates a random `CORE_API_KEY` on first boot when the env var is unset, +persists it at `/var/lib/atomicmemory/state/core-api-key` (mode `600`), and +reuses it on restart when the state volume is mounted. Operators may override +with `-e CORE_API_KEY=…` for pinned secrets. Treat the state file like a +credential: restrict volume access, never log or commit it, and rotate by +deleting the state file (or volume) and restarting Core. + ### Deferred (not implemented) Per-user tokens / a multi-tenant auth layer in core (so a leaked credential compromises one user instead of all) are a larger future change and are **out of scope** here. The current contract assumes a trusted proxy in front of core for multi-user deployments. + +## Outbound Cloud Trace Sync + +When `CLOUD_TRACE_SYNC_ENABLED=true`, Core enqueues **redacted operation +metadata** (counts, ids, durations) to a local Postgres outbox and uploads +it to the configured Cloud observability endpoint. Envelopes **must not** +carry verbatim memory text, search queries, or fact content — only +allowlisted summary fields pass the outbound sanitizer. Treat the Cloud API +key (`amc_*`) like any other deployment secret. diff --git a/packages/core/docker-compose.yml b/packages/core/docker-compose.yml index ec50207..63eec80 100644 --- a/packages/core/docker-compose.yml +++ b/packages/core/docker-compose.yml @@ -33,6 +33,8 @@ services: - ${ENV_FILE:-.env} extra_hosts: - "host.docker.internal:host-gateway" + volumes: + - corestate:/var/lib/atomicmemory/state depends_on: postgres: condition: service_healthy @@ -50,3 +52,4 @@ services: volumes: pgdata: + corestate: diff --git a/packages/core/package.json b/packages/core/package.json index cedc928..11d30d7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@atomicmemory/core", - "version": "1.1.1", + "version": "1.2.0", "description": "Open-source memory engine for AI applications — semantic retrieval, AUDN mutation, and contradiction-safe claim versioning.", "type": "module", "license": "Apache-2.0", @@ -92,6 +92,7 @@ "@filoz/synapse-sdk": "^0.41.0", "@huggingface/transformers": "^3.8.1", "express": "^5.1.0", + "jose": "^6.2.3", "multiformats": "^13.4.1", "node-pg-migrate": "8.0.4", "openai": "^4.80.0", diff --git a/packages/core/scripts/docker-entrypoint.sh b/packages/core/scripts/docker-entrypoint.sh index 5cf9dd1..1c4a566 100644 --- a/packages/core/scripts/docker-entrypoint.sh +++ b/packages/core/scripts/docker-entrypoint.sh @@ -9,13 +9,122 @@ set -euo pipefail APP_PID="" POSTGRES_STARTED=false -LOCAL_DOCKER_CORE_API_KEY="local-dev-key" LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +CORE_STATE_DIR="${CORE_STATE_DIR:-/var/lib/atomicmemory/state}" +CORE_API_KEY_FILE="$CORE_STATE_DIR/core-api-key" log() { printf '[entrypoint] %s\n' "$*" } +is_hosted_deployment_env() { + case "${1:-local}" in + production|staging) return 0 ;; + *) return 1 ;; + esac +} + +generate_core_api_key() { + od -vN32 -An -tx1 /dev/urandom | tr -d ' \n' +} + +persist_core_api_key() { + local key="$1" + mkdir -p "$CORE_STATE_DIR" + printf '%s\n' "$key" > "$CORE_API_KEY_FILE" + chmod 600 "$CORE_API_KEY_FILE" +} + +resolve_core_api_key() { + if is_hosted_deployment_env "${RAW_STORAGE_DEPLOYMENT_ENV:-local}"; then + if [ -z "${CORE_API_KEY:-}" ]; then + log "CORE_API_KEY is required when RAW_STORAGE_DEPLOYMENT_ENV=${RAW_STORAGE_DEPLOYMENT_ENV:-local}" + exit 1 + fi + export CORE_API_KEY + log "CORE_API_KEY from environment (hosted — not persisted locally)" + return + fi + + if [ -n "${CORE_API_KEY:-}" ]; then + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY from environment and persisted to $CORE_API_KEY_FILE" + return + fi + + if [ -s "$CORE_API_KEY_FILE" ]; then + CORE_API_KEY="$(tr -d '[:space:]' < "$CORE_API_KEY_FILE")" + if [ -n "$CORE_API_KEY" ]; then + export CORE_API_KEY + log "CORE_API_KEY loaded from $CORE_API_KEY_FILE" + return + fi + fi + + CORE_API_KEY="$(generate_core_api_key)" + persist_core_api_key "$CORE_API_KEY" + export CORE_API_KEY + log "CORE_API_KEY generated and persisted to $CORE_API_KEY_FILE" +} + +cloud_tier_api_url() { + case "${1:-dev}" in + dev) printf '%s' 'https://api.dev.atomicstrata.ai' ;; + staging) printf '%s' 'https://api.staging.atomicstrata.ai' ;; + production|prod) printf '%s' 'https://api.atomicstrata.ai' ;; + *) + log "Unknown CLOUD_ENV: ${1}" + exit 1 + ;; + esac +} + +cloud_tier_memory_origin() { + case "${1:-dev}" in + dev) printf '%s' 'https://memory.dev.atomicstrata.ai' ;; + staging) printf '%s' 'https://memory.staging.atomicstrata.ai' ;; + production|prod) printf '%s' 'https://memory.atomicstrata.ai' ;; + *) + log "Unknown CLOUD_ENV: ${1}" + exit 1 + ;; + esac +} + +# When running self-hosted Core for connected-local, apply tier defaults so +# operators only pass OPENAI_API_KEY + ATOMICMEMORY_API_KEY. The presence of +# ATOMICMEMORY_API_KEY is the single switch that turns connected-local on; +# CLOUD_PROJECT_ID is optional (Core trusts the token's project_id when unset). +apply_connected_local_defaults() { + if is_hosted_deployment_env "${RAW_STORAGE_DEPLOYMENT_ENV:-local}"; then + return + fi + + if [ -z "${ATOMICMEMORY_API_KEY:-}" ]; then + return + fi + + local tier="${CLOUD_ENV:-dev}" + local api_url memory_origin + + api_url="$(cloud_tier_api_url "$tier")" + memory_origin="$(cloud_tier_memory_origin "$tier")" + + export CLOUD_TRACE_SYNC_ENABLED="${CLOUD_TRACE_SYNC_ENABLED:-true}" + export ATOMICMEMORY_API_URL="${ATOMICMEMORY_API_URL:-$api_url}" + + export CLOUD_JWKS_URL="${CLOUD_JWKS_URL:-${api_url}/.well-known/atomic-core/jwks.json}" + export CLOUD_JWT_ISSUER="${CLOUD_JWT_ISSUER:-$api_url}" + export CLOUD_JWT_AUDIENCE="${CLOUD_JWT_AUDIENCE:-atomicmemory-core}" + export ALLOWED_ORIGINS="${ALLOWED_ORIGINS:-$memory_origin}" + export CLOUD_JWT_STATIC_KEY_FALLBACK="${CLOUD_JWT_STATIC_KEY_FALLBACK:-true}" + if [ "$tier" = "dev" ]; then + export CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID="${CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID:-default}" + fi + log "Connected-local defaults applied (CLOUD_ENV=$tier, api=$ATOMICMEMORY_API_URL)" +} + stop_postgres() { if [ "$POSTGRES_STARTED" = "true" ]; then log "Stopping embedded Postgres..." @@ -43,18 +152,12 @@ configure_local_defaults() { local deployment_env="${RAW_STORAGE_DEPLOYMENT_ENV:-local}" export RAW_STORAGE_DEPLOYMENT_ENV="$deployment_env" - if [ -z "${CORE_API_KEY:-}" ]; then - if [ "$deployment_env" = "production" ]; then - log "CORE_API_KEY is required when RAW_STORAGE_DEPLOYMENT_ENV=production" - exit 1 - fi - export CORE_API_KEY="$LOCAL_DOCKER_CORE_API_KEY" - log "CORE_API_KEY not set; using local Docker default '$CORE_API_KEY'" - fi + apply_connected_local_defaults + resolve_core_api_key if [ -z "${STORAGE_KEY_HMAC_SECRET:-}" ]; then - if [ "$deployment_env" = "production" ]; then - log "STORAGE_KEY_HMAC_SECRET is required when RAW_STORAGE_DEPLOYMENT_ENV=production" + if is_hosted_deployment_env "$deployment_env"; then + log "STORAGE_KEY_HMAC_SECRET is required when RAW_STORAGE_DEPLOYMENT_ENV=$deployment_env" exit 1 fi export STORAGE_KEY_HMAC_SECRET="$LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET" diff --git a/packages/core/scripts/docker-smoke-test.sh b/packages/core/scripts/docker-smoke-test.sh index 694935a..58728a4 100755 --- a/packages/core/scripts/docker-smoke-test.sh +++ b/packages/core/scripts/docker-smoke-test.sh @@ -26,6 +26,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +USER_SUPPLIED_CORE_API_KEY="${CORE_API_KEY:-}" COMPOSE_PROJECT="${COMPOSE_PROJECT:-}" COMPOSE_BASE_FILE="${COMPOSE_BASE_FILE:-docker-compose.yml}" APP_PORT="${APP_PORT:-}" @@ -34,6 +35,7 @@ SMOKE_ENV_FILE="$PROJECT_DIR/.env.docker-smoke-test" ENV_FILE_CREATED_BY_SCRIPT=0 SMOKE_CORE_API_KEY="test-core-api-key-do-not-leak" SMOKE_STORAGE_KEY_HMAC_SECRET="000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" +CORE_STATE_KEY_PATH="/var/lib/atomicmemory/state/core-api-key" HEALTH_TIMEOUT=90 HEALTH_INTERVAL=2 @@ -123,13 +125,21 @@ fi if [[ -z "${ENV_FILE:-}" && ! -f "$PROJECT_DIR/.env" ]]; then log "Creating stub smoke env file (compose env_file requires one)" - cat > "$SMOKE_ENV_FILE" < "$SMOKE_ENV_FILE" < "$SMOKE_ENV_FILE" </dev/null | tail -1 | cut -d= -f2- || true)" fi fi +if [[ -z "$CORE_API_KEY" ]]; then + log "No CORE_API_KEY in env — reading entrypoint-generated key from container" + CORE_API_KEY="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" +fi if [[ -z "$CORE_API_KEY" ]]; then fail "CORE_API_KEY is required for authenticated /v1 smoke checks" exit 1 fi AUTH_HEADER=("Authorization: Bearer ${CORE_API_KEY}") +saved_core_api_key="$CORE_API_KEY" +if [[ -n "${USER_SUPPLIED_CORE_API_KEY}" ]]; then + log "Test: explicit CORE_API_KEY override persisted to state volume" + state_key="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" + assert_ok "Explicit CORE_API_KEY override written to state volume" \ + '[ "$state_key" = "$saved_core_api_key" ]' + + log "Test: explicit CORE_API_KEY override survives recreate without env" + if (( ENV_FILE_CREATED_BY_SCRIPT == 1 )); then + cat > "$SMOKE_ENV_FILE" </dev/null +recreate_timeout=180 +elapsed=0 +while true; do + if curl -sf "http://localhost:${APP_PORT}/health" >/dev/null 2>&1; then + break + fi + if [[ $elapsed -ge $recreate_timeout ]]; then + fail "App did not become healthy after recreate within ${recreate_timeout}s" + warn "--- app container logs (recreate) ---" + docker compose -p "$COMPOSE_PROJECT" logs app 2>&1 | tail -40 + exit 1 + fi + sleep "$HEALTH_INTERVAL" + elapsed=$((elapsed + HEALTH_INTERVAL)) +done +recreated_key="$(docker compose -p "$COMPOSE_PROJECT" exec -T app \ + cat "$CORE_STATE_KEY_PATH" 2>/dev/null | tr -d '[:space:]')" +if [[ -n "${USER_SUPPLIED_CORE_API_KEY}" ]]; then + assert_ok "Explicit CORE_API_KEY override unchanged after recreate without env" \ + '[ "$recreated_key" = "$saved_core_api_key" ]' +else + assert_ok "CORE_API_KEY unchanged after container recreate (state volume)" \ + '[ "$recreated_key" = "$saved_core_api_key" ]' +fi +CORE_API_KEY="$recreated_key" +AUTH_HEADER=("Authorization: Bearer ${CORE_API_KEY}") + # --- Test 1: Root health --- log "Test: root /health endpoint" health_body=$(curl -sf "$BASE/health") diff --git a/packages/core/scripts/show-local-core-key.sh b/packages/core/scripts/show-local-core-key.sh new file mode 100644 index 0000000..ead1613 --- /dev/null +++ b/packages/core/scripts/show-local-core-key.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Print the persisted local CORE_API_KEY from a running Core container. +# Reads the state volume file written by docker-entrypoint.sh (generated or +# explicit override). Usage: ./scripts/show-local-core-key.sh [container_name] +set -euo pipefail +CONTAINER="${1:-atomic-memory}" +docker exec "$CONTAINER" cat /var/lib/atomicmemory/state/core-api-key diff --git a/packages/core/src/__tests__/cloud-jwt-config.test.ts b/packages/core/src/__tests__/cloud-jwt-config.test.ts new file mode 100644 index 0000000..4cbce1e --- /dev/null +++ b/packages/core/src/__tests__/cloud-jwt-config.test.ts @@ -0,0 +1,109 @@ +/** + * Cloud JWT env parsing — all-or-nothing validation. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const trackedEnvNames = [ + 'CLOUD_JWKS_URL', + 'CLOUD_JWT_ISSUER', + 'CLOUD_JWT_AUDIENCE', + 'CLOUD_PROJECT_ID', + 'CLOUD_JWT_STATIC_KEY_FALLBACK', + 'RAW_STORAGE_DEPLOYMENT_ENV', + 'DATABASE_URL', + 'CORE_API_KEY', + 'STORAGE_KEY_HMAC_SECRET', + 'EMBEDDING_DIMENSIONS', + 'EMBEDDING_PROVIDER', + 'OPENAI_API_KEY', +] as const; + +const originalEnv = Object.fromEntries( + trackedEnvNames.map((name) => [name, process.env[name]]), +) as Record<(typeof trackedEnvNames)[number], string | undefined>; + +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +beforeEach(() => { + process.env.DATABASE_URL = + 'postgresql://atomicmemory:atomicmemory@localhost:5433/atomicmemory_test'; + process.env.CORE_API_KEY = 'test-core-api-key'; + process.env.STORAGE_KEY_HMAC_SECRET = + '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f'; + process.env.EMBEDDING_DIMENSIONS = '384'; + process.env.EMBEDDING_PROVIDER = 'openai'; + process.env.OPENAI_API_KEY = 'test-openai-key'; + process.env.RAW_STORAGE_DEPLOYMENT_ENV = 'local'; + delete process.env.CLOUD_JWKS_URL; + delete process.env.CLOUD_JWT_ISSUER; + delete process.env.CLOUD_JWT_AUDIENCE; + delete process.env.CLOUD_PROJECT_ID; + delete process.env.CLOUD_JWT_STATIC_KEY_FALLBACK; +}); + +afterEach(() => { + for (const name of trackedEnvNames) restoreEnv(name, originalEnv[name]); + vi.resetModules(); +}); + +async function loadConfig() { + vi.resetModules(); + return (await import('../config.js')).config; +} + +describe('parseCloudJwtConfig via config module', () => { + it('returns null when all CLOUD JWT vars are unset', async () => { + expect((await loadConfig()).cloudJwt).toBeNull(); + }); + + it('loads profile when all required Cloud JWT vars are set', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + process.env.CLOUD_PROJECT_ID = 'proj_test'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt).toEqual({ + jwksUrl: 'https://api.test/.well-known/atomic-core/jwks.json', + issuer: 'https://api.test', + audience: 'atomicmemory-core', + projectId: 'proj_test', + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }); + }); + + it('leaves projectId null when CLOUD_PROJECT_ID is unset (single-key local)', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt).toEqual({ + jwksUrl: 'https://api.test/.well-known/atomic-core/jwks.json', + issuer: 'https://api.test', + audience: 'atomicmemory-core', + projectId: null, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }); + }); + + it('parses explicit static-key fallback boolean', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/.well-known/atomic-core/jwks.json'; + process.env.CLOUD_JWT_ISSUER = 'https://api.test'; + process.env.CLOUD_JWT_AUDIENCE = 'atomicmemory-core'; + process.env.CLOUD_PROJECT_ID = 'proj_test'; + process.env.CLOUD_JWT_STATIC_KEY_FALLBACK = 'true'; + const cfg = await loadConfig(); + expect(cfg.cloudJwt?.staticKeyFallbackEnabled).toBe(true); + }); + + it('fails startup when only some Cloud JWT vars are set', async () => { + process.env.CLOUD_JWKS_URL = 'https://api.test/jwks.json'; + vi.resetModules(); + await expect(import('../config.js')).rejects.toThrow(/must all be set together/); + }); +}); diff --git a/packages/core/src/__tests__/deployment-config.test.ts b/packages/core/src/__tests__/deployment-config.test.ts index a767a83..26ece68 100644 --- a/packages/core/src/__tests__/deployment-config.test.ts +++ b/packages/core/src/__tests__/deployment-config.test.ts @@ -186,9 +186,29 @@ describe('deployment configuration', () => { it('entrypoint provides local Docker auth and storage defaults', () => { const entrypoint = readFileSync(resolve(ROOT, 'scripts/docker-entrypoint.sh'), 'utf-8'); - expect(entrypoint).toContain('LOCAL_DOCKER_CORE_API_KEY="local-dev-key"'); + expect(entrypoint).toContain('CORE_API_KEY_FILE'); + expect(entrypoint).toContain('generate_core_api_key'); + expect(entrypoint).toContain('persist_core_api_key'); + expect(entrypoint).toContain('is_hosted_deployment_env'); + expect(entrypoint).toContain('apply_connected_local_defaults'); + expect(entrypoint).toContain('CLOUD_ENV'); expect(entrypoint).toContain('LOCAL_DOCKER_STORAGE_KEY_HMAC_SECRET='); - expect(entrypoint).toContain('RAW_STORAGE_DEPLOYMENT_ENV=production'); + expect(entrypoint).toContain('production|staging'); + expect(entrypoint).toContain('CORE_API_KEY is required'); + expect(entrypoint).toContain('CORE_API_KEY from environment and persisted to $CORE_API_KEY_FILE'); + expect(entrypoint).toContain('CORE_API_KEY from environment (hosted — not persisted locally)'); + expect(entrypoint).toContain('CORE_API_KEY loaded from $CORE_API_KEY_FILE'); + expect(entrypoint).toContain('CORE_API_KEY generated and persisted'); + }); + + it('show-local-core-key helper reads persisted state volume path', () => { + const helper = readFileSync(resolve(ROOT, 'scripts/show-local-core-key.sh'), 'utf-8'); + expect(helper).toContain('/var/lib/atomicmemory/state/core-api-key'); + }); + + it('compose mounts persistent Core state for generated local API keys', () => { + const compose = readFileSync(resolve(ROOT, 'docker-compose.yml'), 'utf-8'); + expect(compose).toContain('corestate:/var/lib/atomicmemory/state'); }); it('creates non-root user', () => { diff --git a/packages/core/src/__tests__/helpers/jwt-auth-headers.ts b/packages/core/src/__tests__/helpers/jwt-auth-headers.ts new file mode 100644 index 0000000..2beed09 --- /dev/null +++ b/packages/core/src/__tests__/helpers/jwt-auth-headers.ts @@ -0,0 +1,10 @@ +/** + * Bearer header helper for Cloud-issued JWT auth in integration tests. + */ + +export function jwtAuthHeader(accessToken: string): Record { + if (accessToken.length === 0) { + throw new Error('jwtAuthHeader: accessToken must be non-empty'); + } + return { Authorization: `Bearer ${accessToken}` }; +} diff --git a/packages/core/src/__tests__/setup.ts b/packages/core/src/__tests__/setup.ts index f897ef1..ebef260 100644 --- a/packages/core/src/__tests__/setup.ts +++ b/packages/core/src/__tests__/setup.ts @@ -15,6 +15,11 @@ process.env.CORE_API_KEY ??= 'test-core-api-key'; process.env.STORAGE_KEY_HMAC_SECRET ??= '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f'; process.env.DATABASE_URL ??= 'postgresql://atomicmem:atomicmem@localhost:5433/atomicmem_test'; process.env.EMBEDDING_DIMENSIONS ??= '1536'; +// Contributor shells may export EMBEDDING_PROVIDER=stub for other packages; +// core tests require a real provider name at config load time. +if (!process.env.EMBEDDING_PROVIDER || process.env.EMBEDDING_PROVIDER === 'stub') { + process.env.EMBEDDING_PROVIDER = 'openai'; +} process.env.RAW_STORAGE_DEPLOYMENT_ENV ??= 'local'; // Mirror .env.test.example for route seam tests when no local env file exists; // production config still defaults this flag to false in src/config.ts. diff --git a/packages/core/src/__tests__/trace-sync-config.test.ts b/packages/core/src/__tests__/trace-sync-config.test.ts new file mode 100644 index 0000000..01859b9 --- /dev/null +++ b/packages/core/src/__tests__/trace-sync-config.test.ts @@ -0,0 +1,60 @@ +/** + * Cloud trace sync env parsing. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { parseCloudTraceSyncConfig } from '../cloud/trace-sync-config.js'; + +const tracked = [ + 'CLOUD_TRACE_SYNC_ENABLED', + 'ATOMICMEMORY_API_URL', + 'ATOMICMEMORY_API_KEY', + 'CORE_INSTANCE_ID', +] as const; + +const original = Object.fromEntries(tracked.map((name) => [name, process.env[name]])) as Record< + (typeof tracked)[number], + string | undefined +>; + +function restore(name: string, value: string | undefined): void { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; +} + +beforeEach(() => { + for (const name of tracked) delete process.env[name]; +}); + +afterEach(() => { + for (const name of tracked) restore(name, original[name]); +}); + +describe('parseCloudTraceSyncConfig', () => { + it('returns null when all vars are absent', () => { + expect(parseCloudTraceSyncConfig()).toBeNull(); + }); + + it('returns null when explicitly disabled even with sibling vars present', () => { + process.env.CLOUD_TRACE_SYNC_ENABLED = 'false'; + process.env.ATOMICMEMORY_API_URL = 'https://api.test'; + process.env.ATOMICMEMORY_API_KEY = 'amc_test_key'; + expect(parseCloudTraceSyncConfig()).toBeNull(); + }); + + it('loads config when enabled with credentials', () => { + process.env.CLOUD_TRACE_SYNC_ENABLED = 'true'; + process.env.ATOMICMEMORY_API_URL = 'https://api.test/'; + process.env.ATOMICMEMORY_API_KEY = 'amc_test_key'; + const cfg = parseCloudTraceSyncConfig(); + expect(cfg?.enabled).toBe(true); + expect(cfg?.apiUrl).toBe('https://api.test'); + expect(cfg?.claimStaleMs).toBeGreaterThan(0); + expect(cfg?.maxPending).toBeGreaterThan(0); + }); + + it('throws on partial config without explicit disable', () => { + process.env.ATOMICMEMORY_API_URL = 'https://api.test'; + expect(() => parseCloudTraceSyncConfig()).toThrow(/Partial Cloud trace sync/); + }); +}); diff --git a/packages/core/src/app/__tests__/versioned-mount.test.ts b/packages/core/src/app/__tests__/versioned-mount.test.ts index 1bc2868..758223f 100644 --- a/packages/core/src/app/__tests__/versioned-mount.test.ts +++ b/packages/core/src/app/__tests__/versioned-mount.test.ts @@ -14,7 +14,8 @@ * re-test route logic, which is covered by the router-level test files. */ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { exportJWK, generateKeyPair, SignJWT } from 'jose'; import { pool } from '../../db/pool.js'; import { setupTestSchema } from '../../db/__tests__/test-fixtures.js'; @@ -22,9 +23,16 @@ import { createCoreRuntime } from '../runtime-container.js'; import { createApp } from '../create-app.js'; import { bindEphemeral, type BootedApp } from '../bind-ephemeral.js'; import { authHeader } from '../../__tests__/helpers/auth-headers.js'; +import { jwtAuthHeader } from '../../__tests__/helpers/jwt-auth-headers.js'; +import { config } from '../../config.js'; const TEST_USER = 'versioned-mount-user'; const TEST_AGENT = 'versioned-mount-agent'; +const JWKS_URL = 'https://cloud.test/.well-known/atomic-core/jwks.json'; +const JWT_ISSUER = 'https://api.test'; +const JWT_AUDIENCE = 'atomicmemory-core'; +const JWT_PROJECT = 'proj_mount'; +const JWT_USER = 'cloud-jwt-mount-user'; /** * Assert the response is HTTP 401 with `error_code: 'unauthenticated'`. @@ -39,14 +47,68 @@ async function expectUnauthenticatedJson(res: Response): Promise { describe('createApp /v1 mount coverage', () => { let booted: BootedApp; + let jwtBooted: BootedApp; + let jwtKeyPairPromise: ReturnType; + + async function mintMountTestJwt(memoryUserId: string = JWT_USER): Promise { + const { privateKey } = await jwtKeyPairPromise; + return new SignJWT({ + project_id: JWT_PROJECT, + memory_user_id: memoryUserId, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'mount-test-kid' }) + .setIssuer(JWT_ISSUER) + .setAudience(JWT_AUDIENCE) + .setSubject(memoryUserId) + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey); + } beforeAll(async () => { await setupTestSchema(pool); booted = await bindEphemeral(createApp(await createCoreRuntime({ pool }))); + + jwtKeyPairPromise = generateKeyPair('RS256'); + const { publicKey } = await jwtKeyPairPromise; + const jwk = await exportJWK(publicKey); + jwk.kid = 'mount-test-kid'; + jwk.alg = 'RS256'; + jwk.use = 'sig'; + + const realFetch = globalThis.fetch.bind(globalThis); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url === JWKS_URL) { + return new Response(JSON.stringify({ keys: [jwk] }), { status: 200 }); + } + return realFetch(input, init); + }), + ); + + const jwtRuntime = await createCoreRuntime({ + pool, + config: { + ...config, + cloudJwt: { + jwksUrl: JWKS_URL, + issuer: JWT_ISSUER, + audience: JWT_AUDIENCE, + projectId: JWT_PROJECT, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + }, + }, + }); + jwtBooted = await bindEphemeral(createApp(jwtRuntime)); }); afterAll(async () => { await booted.close(); + await jwtBooted.close(); + vi.unstubAllGlobals(); await pool.end(); }); @@ -217,6 +279,64 @@ describe('createApp /v1 mount coverage', () => { }); expect(res.status).toBe(204); }); + + it('POST /v1/memories/ingest rejects Cloud JWT body user_id mismatch with 403', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/memories/ingest`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: 'other-user', + content: 'hello', + source_site: 'test', + }), + }); + expect(res.status).toBe(403); + const body = (await res.json()) as { error_code: string }; + expect(body.error_code).toBe('forbidden'); + }); + + it('POST /v1/documents rejects Cloud JWT body user_id mismatch with 403', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/documents`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({ + user_id: 'other-user', + source_site: 'drive', + provider: 'test', + external_id: 'doc-cross-user', + }), + }); + expect(res.status).toBe(403); + const body = (await res.json()) as { error_code: string }; + expect(body.error_code).toBe('forbidden'); + }); + + it('POST /v1/memories/reconcile scopes Cloud JWT to asserted user when body omits user_id', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch(`${jwtBooted.baseUrl}/v1/memories/reconcile`, { + method: 'POST', + headers: { ...jwtAuthHeader(token), 'Content-Type': 'application/json' }, + body: JSON.stringify({}), + }); + expect(res.status).toBe(200); + }); + + it('GET /v1/memories/list accepts a Cloud JWT via jwtAuthHeader', async () => { + const token = await mintMountTestJwt(); + + const res = await fetch( + `${jwtBooted.baseUrl}/v1/memories/list?user_id=${JWT_USER}`, + { headers: jwtAuthHeader(token) }, + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(Array.isArray(body.memories)).toBe(true); + }); }); async function expectUnauthenticatedRawUpload( diff --git a/packages/core/src/app/create-app.ts b/packages/core/src/app/create-app.ts index ea62510..38c0c5c 100644 --- a/packages/core/src/app/create-app.ts +++ b/packages/core/src/app/create-app.ts @@ -18,6 +18,7 @@ import { embedText } from '../services/embedding.js'; import { createStorageRouter } from '../routes/storage.js'; import { createEntityRouter } from '../routes/entities.js'; import { MAX_INDEX_TEXT_BYTES } from '../schemas/documents.js'; +import { createAuthMiddleware } from '../middleware/dual-auth.js'; import { requireBearer } from '../middleware/require-bearer.js'; import { assertedUserGuard } from '../middleware/asserted-user.js'; import { rejectNulInRequestTarget, rejectNulInBody } from '../middleware/reject-nul-bytes.js'; @@ -27,6 +28,7 @@ import { CORE_CAPABILITIES } from './capabilities-descriptor.js'; import { verifyCapabilitiesDescriptor } from './verify-capabilities.js'; import { SearchBodySchema } from '../schemas/memories.js'; import type { CoreRuntime } from './runtime-container.js'; +import { getCloudTraceHealthSnapshotAsync, toCloudTracePublicHealth } from '../services/cloud-trace-sync.js'; /** Default JSON-body cap for non-document routers. */ const DEFAULT_JSON_BODY_LIMIT = '1mb'; @@ -80,7 +82,14 @@ export function createApp(runtime: CoreRuntime): ReturnType { // expected-key buffer is captured a single time (timingSafeEqual // wants matching-length buffers; cheap but allocation-stable). // `/health` and `/openapi.json` stay outside this scope. - const auth = requireBearer(runtime.config.coreApiKey); + const authBundle = createAuthMiddleware({ + coreApiKey: runtime.config.coreApiKey, + cloudJwt: runtime.config.cloudJwt, + }); + const auth = authBundle.middleware; + if (authBundle.prefetchJwks) { + app.set('cloudJwtPrefetch', authBundle.prefetchJwks); + } // defense-in-depth: when TRUSTED_PROXY_MODE is on, the trusted // caller must restate the user it authenticated in the @@ -100,17 +109,17 @@ export function createApp(runtime: CoreRuntime): ReturnType { verifyCapabilitiesDescriptor(CORE_CAPABILITIES, memoryRouter, SearchBodySchema); app.use( '/v1/memories', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, assertUser, memoryRouter, ); app.use( '/v1/agents', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, assertUser, createAgentRouter(runtime.repos.trust), ); @@ -181,9 +190,9 @@ export function createApp(runtime: CoreRuntime): ReturnType { app.use( '/v1/entities', - auth, express.json({ limit: DEFAULT_JSON_BODY_LIMIT }), rejectNulInBody, + auth, createEntityRouter({ pool: runtime.pool, memory: runtime.repos.memory, @@ -260,8 +269,19 @@ export function createApp(runtime: CoreRuntime): ReturnType { // `/health` is intentionally unversioned — it is an infrastructure // liveness probe (load balancers, Docker, Railway), not part of the // versioned application API. Versioned endpoints live under `/v1/*`. - app.get('/health', (_req, res) => { - res.json({ status: 'ok' }); + app.get('/health', async (_req, res) => { + const cloudTraceSync = runtime.config.cloudTraceSync?.enabled + ? toCloudTracePublicHealth( + await getCloudTraceHealthSnapshotAsync(runtime.pool, runtime.config.cloudTraceSync), + ) + : undefined; + const status = cloudTraceSync?.status === 'degraded' || cloudTraceSync?.status === 'paused' + ? 'degraded' + : 'ok'; + res.json({ + status, + ...(cloudTraceSync ? { cloud_trace_sync: cloudTraceSync } : {}), + }); }); // `GET /openapi.json` serves the committed OpenAPI spec. Like diff --git a/packages/core/src/app/runtime-container.ts b/packages/core/src/app/runtime-container.ts index 5adba5f..41b7791 100644 --- a/packages/core/src/app/runtime-container.ts +++ b/packages/core/src/app/runtime-container.ts @@ -167,6 +167,8 @@ export interface CoreRuntimeConfig { eventChainPackagingEnabled: boolean; /** Reflect channel gate (BEAM-0.85 Phase 1, Task 1.9). */ reflectEnabled: boolean; + /** Opt-in OSS Core → Cloud trace upload profile. Null when disabled. */ + cloudTraceSync: import('../config.js').CloudTraceSyncConfig | null; /** Top-K reflections to fetch when reflect retrieval is enabled. */ reflectRetrievalTopK: number; /** Anthropic model used by the reflect worker. */ diff --git a/packages/core/src/cloud/__tests__/heartbeat-client.test.ts b/packages/core/src/cloud/__tests__/heartbeat-client.test.ts new file mode 100644 index 0000000..9a1ae16 --- /dev/null +++ b/packages/core/src/cloud/__tests__/heartbeat-client.test.ts @@ -0,0 +1,50 @@ +/** + * Heartbeat client unit tests. + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { sendRuntimeHeartbeat } from '../heartbeat-client.js'; + +describe('sendRuntimeHeartbeat', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns ok on 200', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ ok: true, status: 200 }), + ); + const result = await sendRuntimeHeartbeat({ + apiUrl: 'https://api.test', + apiKey: 'amc_test_key', + payload: { + core_instance_id: 'inst-1', + core_version: '1.0.0', + connector_version: '1.0.0', + capabilities: ['memory.read'], + }, + }); + expect(result.ok).toBe(true); + expect(result.errorCode).toBeNull(); + }); + + it('maps auth failures', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ ok: false, status: 401 }), + ); + const result = await sendRuntimeHeartbeat({ + apiUrl: 'https://api.test', + apiKey: 'amc_test_key', + payload: { + core_instance_id: 'inst-1', + core_version: '1.0.0', + connector_version: '1.0.0', + capabilities: [], + }, + }); + expect(result.ok).toBe(false); + expect(result.errorCode).toBe('auth_401'); + }); +}); diff --git a/packages/core/src/cloud/cloud-api-client.ts b/packages/core/src/cloud/cloud-api-client.ts new file mode 100644 index 0000000..5ea5425 --- /dev/null +++ b/packages/core/src/cloud/cloud-api-client.ts @@ -0,0 +1,25 @@ +/** + * Shared outbound POST helper for Cloud API calls (traces, heartbeat). + */ + +export interface CloudApiPostOptions { + apiUrl: string; + apiKey: string; + path: string; + body: unknown; + timeoutMs?: number; +} + +export async function cloudApiPost(options: CloudApiPostOptions): Promise { + const base = options.apiUrl.replace(/\/+$/, ''); + const path = options.path.startsWith('/') ? options.path : `/${options.path}`; + return fetch(`${base}${path}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${options.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(options.body), + signal: AbortSignal.timeout(options.timeoutMs ?? 30_000), + }); +} diff --git a/packages/core/src/cloud/env.ts b/packages/core/src/cloud/env.ts new file mode 100644 index 0000000..8d074ae --- /dev/null +++ b/packages/core/src/cloud/env.ts @@ -0,0 +1,25 @@ +/** + * Small env helpers for Cloud config parsers (isolated from monolithic config.ts). + */ + +export function optionalEnv(name: string): string | undefined { + return process.env[name] || undefined; +} + +export function parsePositiveIntEnv(name: string, fallback: number): number { + const raw = optionalEnv(name); + if (!raw) return fallback; + const parsed = Number.parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + +export function parseStrictBoolEnv(name: string, fallback: boolean): boolean { + const raw = optionalEnv(name); + if (raw === undefined) return fallback; + if (raw === 'true') return true; + if (raw === 'false') return false; + throw new Error(`${name} must be 'true' or 'false' (got '${raw}')`); +} diff --git a/packages/core/src/cloud/heartbeat-client.ts b/packages/core/src/cloud/heartbeat-client.ts new file mode 100644 index 0000000..15a6ab8 --- /dev/null +++ b/packages/core/src/cloud/heartbeat-client.ts @@ -0,0 +1,41 @@ +/** + * Outbound runtime heartbeat to Cloud (amc_ auth + core_instance_id). + */ + +import { cloudApiPost } from './cloud-api-client.js'; +import type { CloudRuntimeHeartbeatPayload } from './types.js'; + +export interface SendRuntimeHeartbeatInput { + apiUrl: string; + apiKey: string; + payload: CloudRuntimeHeartbeatPayload; +} + +export interface SendRuntimeHeartbeatResult { + ok: boolean; + status: number; + errorCode: string | null; +} + +export async function sendRuntimeHeartbeat( + input: SendRuntimeHeartbeatInput, +): Promise { + try { + const response = await cloudApiPost({ + apiUrl: input.apiUrl, + apiKey: input.apiKey, + path: '/v1/runtimes/heartbeat', + body: input.payload, + }); + if (response.ok) { + return { ok: true, status: response.status, errorCode: null }; + } + return { + ok: false, + status: response.status, + errorCode: response.status === 401 || response.status === 403 ? `auth_${response.status}` : `http_${response.status}`, + }; + } catch { + return { ok: false, status: 0, errorCode: 'network_error' }; + } +} diff --git a/packages/core/src/cloud/instance-id.ts b/packages/core/src/cloud/instance-id.ts new file mode 100644 index 0000000..ac048c6 --- /dev/null +++ b/packages/core/src/cloud/instance-id.ts @@ -0,0 +1,31 @@ +/** + * Stable Core installation id for Cloud runtime identity and trace envelopes. + */ + +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const DEFAULT_STATE_DIR = join(homedir(), '.atomicmemory', 'state'); + +function resolveInstanceIdPath(): string { + const stateDir = process.env.CORE_STATE_DIR?.trim() || DEFAULT_STATE_DIR; + return join(stateDir, 'core-instance-id'); +} + +export async function resolveCoreInstanceId(configuredId: string): Promise { + if (configuredId.trim()) return configuredId.trim(); + const instanceIdPath = resolveInstanceIdPath(); + try { + const existing = await readFile(instanceIdPath, 'utf8'); + const trimmed = existing.trim(); + if (trimmed) return trimmed; + } catch { + // first boot — generate below + } + const generated = randomUUID(); + await mkdir(dirname(instanceIdPath), { recursive: true }); + await writeFile(instanceIdPath, `${generated}\n`, 'utf8'); + return generated; +} diff --git a/packages/core/src/cloud/jwt-config.ts b/packages/core/src/cloud/jwt-config.ts new file mode 100644 index 0000000..694b4fb --- /dev/null +++ b/packages/core/src/cloud/jwt-config.ts @@ -0,0 +1,66 @@ +/** + * Parse Cloud JWT verification env (inbound console/SDK tokens). + */ + +import type { CloudJwtConfig } from './types.js'; +import { optionalEnv, parseStrictBoolEnv } from './env.js'; + +function requireCloudJwtFields( + jwksUrl: string | undefined, + issuer: string | undefined, + audience: string | undefined, +): asserts jwksUrl is string { + if (jwksUrl && issuer && audience) return; + throw new Error( + 'CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, and CLOUD_JWT_AUDIENCE must all be set together when enabling Cloud JWT verification', + ); +} + +function buildCloudJwtConfig( + jwksUrl: string, + issuer: string, + audience: string, + projectId: string | undefined, + staticKeyFallbackRaw: string | undefined, +): CloudJwtConfig { + const staticKeyFallbackEnabled = + staticKeyFallbackRaw === undefined + ? false + : parseStrictBoolEnv('CLOUD_JWT_STATIC_KEY_FALLBACK', false); + const legacyDefaultMemoryUserId = + optionalEnv('CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID')?.trim() || null; + return { + jwksUrl, + issuer, + audience, + projectId: projectId?.trim() || null, + staticKeyFallbackEnabled, + legacyDefaultMemoryUserId, + }; +} + +export function parseCloudJwtConfig(): CloudJwtConfig | null { + const jwksUrl = optionalEnv('CLOUD_JWKS_URL'); + const issuer = optionalEnv('CLOUD_JWT_ISSUER'); + const audience = optionalEnv('CLOUD_JWT_AUDIENCE'); + const projectId = optionalEnv('CLOUD_PROJECT_ID'); + const staticKeyFallbackRaw = optionalEnv('CLOUD_JWT_STATIC_KEY_FALLBACK'); + if ( + jwksUrl === undefined && + issuer === undefined && + audience === undefined && + projectId === undefined && + staticKeyFallbackRaw === undefined + ) { + return null; + } + + requireCloudJwtFields(jwksUrl, issuer, audience); + try { + void new URL(jwksUrl); + } catch { + throw new Error(`CLOUD_JWKS_URL must be a valid URL, got '${jwksUrl}'`); + } + + return buildCloudJwtConfig(jwksUrl, issuer!, audience!, projectId, staticKeyFallbackRaw); +} diff --git a/packages/core/src/cloud/trace-sync-config.ts b/packages/core/src/cloud/trace-sync-config.ts new file mode 100644 index 0000000..40f0375 --- /dev/null +++ b/packages/core/src/cloud/trace-sync-config.ts @@ -0,0 +1,64 @@ +/** + * Parse Cloud trace sync env (outbound amc_ uploads). + */ + +import type { CloudTraceSyncConfig } from './types.js'; +import { optionalEnv, parsePositiveIntEnv } from './env.js'; + +function requireCloudTraceSyncCredentials( + apiUrl: string | undefined, + apiKey: string | undefined, +): asserts apiUrl is string { + if (apiUrl && apiKey) return; + throw new Error( + 'Cloud trace sync requires ATOMICMEMORY_API_URL and ATOMICMEMORY_API_KEY when CLOUD_TRACE_SYNC_ENABLED=true', + ); +} + +function buildCloudTraceSyncConfig( + apiUrl: string, + apiKey: string, + instanceId: string | undefined, +): CloudTraceSyncConfig { + if (!apiKey.startsWith('amc_')) { + throw new Error('ATOMICMEMORY_API_KEY must be a project-scoped amc_ Cloud API key'); + } + + return { + enabled: true, + apiUrl: apiUrl.replace(/\/+$/, ''), + apiKey, + instanceId: instanceId?.trim() || '', + batchSize: parsePositiveIntEnv('CLOUD_TRACE_SYNC_BATCH_SIZE', 100), + maxAttempts: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_ATTEMPTS', 8), + baseRetryMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_BASE_RETRY_MS', 1_000), + maxRetryMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_RETRY_MS', 300_000), + pollIntervalMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_POLL_INTERVAL_MS', 5_000), + sentRetentionMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_SENT_RETENTION_MS', 86_400_000), + shutdownDrainMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_SHUTDOWN_DRAIN_MS', 10_000), + claimStaleMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_CLAIM_STALE_MS', 300_000), + deadLetterRetentionMs: parsePositiveIntEnv('CLOUD_TRACE_SYNC_DEAD_LETTER_RETENTION_MS', 604_800_000), + maxPending: parsePositiveIntEnv('CLOUD_TRACE_SYNC_MAX_PENDING', 10_000), + }; +} + +export function parseCloudTraceSyncConfig(): CloudTraceSyncConfig | null { + const enabledRaw = optionalEnv('CLOUD_TRACE_SYNC_ENABLED'); + const apiUrl = optionalEnv('ATOMICMEMORY_API_URL'); + const apiKey = optionalEnv('ATOMICMEMORY_API_KEY'); + const instanceId = optionalEnv('CORE_INSTANCE_ID'); + const partial = Boolean(enabledRaw || apiUrl || apiKey || instanceId); + + if (!partial) return null; + if (enabledRaw === 'false') return null; + + const enabled = enabledRaw === 'true'; + if (!enabled) { + throw new Error( + 'Partial Cloud trace sync configuration detected. Set CLOUD_TRACE_SYNC_ENABLED=true with ATOMICMEMORY_API_URL and ATOMICMEMORY_API_KEY, or remove all Cloud trace sync env vars.', + ); + } + + requireCloudTraceSyncCredentials(apiUrl, apiKey); + return buildCloudTraceSyncConfig(apiUrl, apiKey!, instanceId); +} diff --git a/packages/core/src/cloud/types.ts b/packages/core/src/cloud/types.ts new file mode 100644 index 0000000..7d537bf --- /dev/null +++ b/packages/core/src/cloud/types.ts @@ -0,0 +1,41 @@ +/** + * Cloud-connected local configuration types (trace sync + JWT verify). + */ + +export interface CloudTraceSyncConfig { + enabled: boolean; + apiUrl: string; + apiKey: string; + instanceId: string; + batchSize: number; + maxAttempts: number; + baseRetryMs: number; + maxRetryMs: number; + pollIntervalMs: number; + sentRetentionMs: number; + shutdownDrainMs: number; + claimStaleMs: number; + deadLetterRetentionMs: number; + maxPending: number; +} + +export interface CloudJwtConfig { + jwksUrl: string; + issuer: string; + audience: string; + /** + * Optional bound Cloud project for connected-local JWT verification. + * When null, Core trusts the token's own `project_id` claim. + */ + projectId: string | null; + staticKeyFallbackEnabled: boolean; + legacyDefaultMemoryUserId: string | null; +} + +export interface CloudRuntimeHeartbeatPayload { + core_instance_id: string; + core_version: string; + connector_version: string; + capabilities: string[]; + local_url?: string; +} diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index b36e109..b176428 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -22,6 +22,11 @@ import { parseFilecoinProviderConfig, type FilecoinProviderConfig, } from './storage/providers/filecoin/config.js'; +import { parseCloudJwtConfig } from './cloud/jwt-config.js'; +import { parseCloudTraceSyncConfig } from './cloud/trace-sync-config.js'; +import { optionalEnv, parsePositiveIntEnv, parseStrictBoolEnv } from './cloud/env.js'; +import type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; +export type { CloudJwtConfig, CloudTraceSyncConfig } from './cloud/types.js'; export type EmbeddingProviderName = 'openai' | 'ollama' | 'openai-compatible' | 'transformers' | 'voyage'; export type LLMProviderName = EmbeddingProviderName | 'groq' | 'anthropic' | 'google-genai' | 'claude-code' | 'codex'; @@ -75,6 +80,12 @@ export interface RuntimeConfig { * restarting the server with a new value. */ coreApiKey: string; + /** + * Optional Cloud-issued JWT verification (JWKS). When set, `/v1/*` accepts + * either `CORE_API_KEY` or a valid RS256 JWT with `aud` matching + * `CLOUD_JWT_AUDIENCE`. All three env vars must be set together. + */ + cloudJwt: CloudJwtConfig | null; /** * Trusted-proxy identity guard. When `true`, every * user-scoped request carrying a `user_id` MUST also carry the same @@ -168,6 +179,8 @@ export interface RuntimeConfig { retrievalTraceEnabled: boolean; ingestTraceDir: string; ingestTraceEnabled: boolean; + /** Opt-in OSS Core → Cloud trace upload profile. Null when disabled. */ + cloudTraceSync: CloudTraceSyncConfig | null; extractionCacheEnabled: boolean; extractionCacheDir: string; embeddingCacheEnabled: boolean; @@ -225,6 +238,9 @@ export interface RuntimeConfig { temporalQueryConstraintBoost: number; deferredAudnEnabled: boolean; deferredAudnBatchSize: number; + deferredAudnConcurrency: number; + deferredAudnAutoReconcile: boolean; + deferredAudnReconcileIntervalMs: number; compositeGroupingEnabled: boolean; compositeMinClusterSize: number; compositeMaxClusterSize: number; @@ -720,10 +736,6 @@ function requireEnv(name: string): string { return value; } -function optionalEnv(name: string): string | undefined { - return process.env[name] || undefined; -} - function parseEmbeddingProvider( value: string | undefined, fallback: EmbeddingProviderName, @@ -802,31 +814,6 @@ function parseRegexEnv(name: string): string | undefined { } } -function parsePositiveIntEnv(name: string, fallback: number): number { - const raw = optionalEnv(name); - if (!raw) return fallback; - const parsed = parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed < 1) { - throw new Error(`${name} must be a positive integer`); - } - return parsed; -} - -/** - * Strict boolean env parser. Unlike the widespread `(env ?? 'false') === - * 'true'` idiom, this rejects any value other than the literal `'true'` / - * `'false'` rather than silently treating a typo (e.g. `TRUSTED_PROXY_MODE= - * ture`) as `false`. Used for security-relevant gates where a silent - * mis-parse would disable a guard. Fails closed at startup. - */ -function parseStrictBoolEnv(name: string, fallback: boolean): boolean { - const raw = optionalEnv(name); - if (raw === undefined) return fallback; - if (raw === 'true') return true; - if (raw === 'false') return false; - throw new Error(`${name} must be 'true' or 'false' (got '${raw}')`); -} - function parseVectorBackend(value: string | undefined): VectorBackendName { if (!value) return 'pgvector'; if (value === 'pgvector' || value === 'ruvector-mock' || value === 'zvec-mock') return value; @@ -1251,6 +1238,7 @@ const embeddingProvider = parseEmbeddingProvider(optionalEnv('EMBEDDING_PROVIDER const llmProvider = parseLlmProvider(optionalEnv('LLM_PROVIDER'), 'openai'); const rawStorageDeploymentEnv = parseRawStorageDeploymentEnv(optionalEnv('RAW_STORAGE_DEPLOYMENT_ENV')); const trustedProxyMode = resolveTrustedProxyMode(rawStorageDeploymentEnv); +const cloudJwt = parseCloudJwtConfig(); const retrievalProfile = parseRetrievalProfile(optionalEnv('RETRIEVAL_PROFILE')); const retrievalProfileSettings = getRetrievalProfile(retrievalProfile); const DEFAULT_SIMILARITY_THRESHOLD = 0.3; @@ -1300,6 +1288,7 @@ export const config: RuntimeConfig = { databaseUrl: requireEnv('DATABASE_URL'), openaiApiKey, coreApiKey: requireEnv('CORE_API_KEY'), + cloudJwt, trustedProxyMode, coreAdminApiKey: optionalEnv('CORE_ADMIN_API_KEY'), coreTestScopeAllowPattern: parseRegexEnv('CORE_TEST_SCOPE_ALLOW_PATTERN'), @@ -1368,6 +1357,7 @@ export const config: RuntimeConfig = { retrievalTraceEnabled: (optionalEnv('RETRIEVAL_TRACE_ENABLED') ?? 'false') === 'true', ingestTraceDir: optionalEnv('INGEST_TRACE_DIR') ?? './.traces/ingest', ingestTraceEnabled: (optionalEnv('INGEST_TRACE_ENABLED') ?? 'false') === 'true', + cloudTraceSync: parseCloudTraceSyncConfig(), extractionCacheEnabled: (optionalEnv('EXTRACTION_CACHE_ENABLED') ?? 'false') === 'true', extractionCacheDir: optionalEnv('EXTRACTION_CACHE_DIR') ?? './.eval-cache', embeddingCacheEnabled: (optionalEnv('EMBEDDING_CACHE_ENABLED') ?? 'false') === 'true', @@ -1425,6 +1415,9 @@ export const config: RuntimeConfig = { temporalQueryConstraintBoost: parseFloat(optionalEnv('TEMPORAL_QUERY_CONSTRAINT_BOOST') ?? '2'), deferredAudnEnabled: (optionalEnv('DEFERRED_AUDN_ENABLED') ?? 'false') === 'true', deferredAudnBatchSize: parseInt(optionalEnv('DEFERRED_AUDN_BATCH_SIZE') ?? '20', 10), + deferredAudnConcurrency: parseInt(optionalEnv('DEFERRED_AUDN_CONCURRENCY') ?? '8', 10), + deferredAudnAutoReconcile: (optionalEnv('DEFERRED_AUDN_AUTO_RECONCILE') ?? 'false') === 'true', + deferredAudnReconcileIntervalMs: parseInt(optionalEnv('DEFERRED_AUDN_RECONCILE_INTERVAL_MS') ?? '5000', 10), compositeGroupingEnabled: (optionalEnv('COMPOSITE_GROUPING_ENABLED') ?? 'true') === 'true', compositeMinClusterSize: parseInt(optionalEnv('COMPOSITE_MIN_CLUSTER_SIZE') ?? '2', 10), compositeMaxClusterSize: parseInt(optionalEnv('COMPOSITE_MAX_CLUSTER_SIZE') ?? '3', 10), @@ -1665,6 +1658,8 @@ export const INTERNAL_POLICY_CONFIG_FIELDS = [ 'fastAudnEnabled', 'fastAudnDuplicateThreshold', // Observation / deferred 'observationNetworkEnabled', 'deferredAudnEnabled', 'deferredAudnBatchSize', + 'deferredAudnConcurrency', + 'deferredAudnAutoReconcile', 'deferredAudnReconcileIntervalMs', // Composite grouping 'compositeGroupingEnabled', 'compositeMinClusterSize', 'compositeMaxClusterSize', 'compositeSimilarityThreshold', diff --git a/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts b/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts new file mode 100644 index 0000000..27368f2 --- /dev/null +++ b/packages/core/src/db/__tests__/cloud-trace-outbox-repository.test.ts @@ -0,0 +1,75 @@ +/** + * Integration tests for Cloud trace outbox repository. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import { pool } from '../pool.js'; +import { setupTestSchema } from './test-fixtures.js'; +import { + claimCloudTraceBatch, + countPendingCloudTraces, + enqueueCloudTraceOutbox, + markCloudTraceSent, +} from '../cloud-trace-outbox-repository.js'; +import { buildCloudTraceEnvelope } from '../../services/cloud-trace-envelope.js'; + +function sampleEnvelope(eventId: string = randomUUID()) { + return buildCloudTraceEnvelope({ + eventId, + coreInstanceId: 'core-test-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 1, + summary: { user_id: 'test-user', operation_detail: 'test ingest' }, + }); +} + +beforeAll(async () => { + await setupTestSchema(pool); +}); + +beforeEach(async () => { + await pool.query('DELETE FROM cloud_trace_outbox'); +}); + +afterAll(async () => { + await pool.end(); +}); + +describe('cloud trace outbox repository', () => { + it('enqueue and claim a pending row', async () => { + const envelope = sampleEnvelope(); + await enqueueCloudTraceOutbox(pool, envelope); + const claimed = await claimCloudTraceBatch(pool, 10, 300_000); + expect(claimed).toHaveLength(1); + expect(claimed[0]?.eventId).toBe(envelope.event_id); + const state = await pool.query( + `SELECT delivery_state FROM cloud_trace_outbox WHERE event_id = $1`, + [envelope.event_id], + ); + expect(state.rows[0]?.delivery_state).toBe('claimed'); + }); + + it('reclaims stale claimed rows after crash', async () => { + const envelope = sampleEnvelope(); + await enqueueCloudTraceOutbox(pool, envelope); + const first = await claimCloudTraceBatch(pool, 10, 300_000); + expect(first).toHaveLength(1); + + await pool.query( + `UPDATE cloud_trace_outbox + SET claimed_at = now() - interval '10 minutes' + WHERE event_id = $1`, + [envelope.event_id], + ); + + const reclaimed = await claimCloudTraceBatch(pool, 10, 60_000); + expect(reclaimed).toHaveLength(1); + expect(reclaimed[0]?.eventId).toBe(envelope.event_id); + + await markCloudTraceSent(pool, envelope.event_id); + expect(await countPendingCloudTraces(pool)).toBe(0); + }); +}); diff --git a/packages/core/src/db/__tests__/migration-backcompat.test.ts b/packages/core/src/db/__tests__/migration-backcompat.test.ts index 2be5450..397c922 100644 --- a/packages/core/src/db/__tests__/migration-backcompat.test.ts +++ b/packages/core/src/db/__tests__/migration-backcompat.test.ts @@ -32,7 +32,13 @@ const pool = useMigrationTestPool({ beforeEach, afterAll }); // no legacy table. Allowlisting it covers its PK, indexes, CHECK and the FK to // memories, since isAllowedNewIndex/isAllowedNewConstraint pass any object whose // table is a new allowed table. -const ALLOWED_NEW_TABLES = new Set(['pgmigrations', 'schema_version', 'entity_settings', 'entity_edges']); +const ALLOWED_NEW_TABLES = new Set([ + 'pgmigrations', + 'schema_version', + 'entity_settings', + 'entity_edges', + 'cloud_trace_outbox', +]); const ALLOWED_NEW_INDEXES = new Set([ // schema_version primary key is auto-generated by Postgres on `applied_at`. 'schema_version_pkey', diff --git a/packages/core/src/db/cloud-trace-outbox-repository.ts b/packages/core/src/db/cloud-trace-outbox-repository.ts new file mode 100644 index 0000000..1eb841e --- /dev/null +++ b/packages/core/src/db/cloud-trace-outbox-repository.ts @@ -0,0 +1,187 @@ +/** + * Durable Postgres outbox for Cloud trace upload. + */ + +import type pg from 'pg'; +import type { CloudTraceEnvelopeV2 } from '../services/cloud-trace-envelope.js'; + +export type CloudTraceDeliveryState = 'pending' | 'claimed' | 'sent' | 'dead_letter'; + +export interface CloudTraceOutboxRow { + eventId: string; + schemaVersion: number; + payload: CloudTraceEnvelopeV2; + deliveryState: CloudTraceDeliveryState; + attemptCount: number; + nextAttemptAt: Date; + lastErrorCode: string | null; + createdAt: Date; + claimedAt: Date | null; + sentAt: Date | null; + deadLetterAt: Date | null; +} + +export async function enqueueCloudTraceOutbox( + pool: pg.Pool, + envelope: CloudTraceEnvelopeV2, +): Promise { + await pool.query( + `INSERT INTO cloud_trace_outbox (event_id, schema_version, payload, delivery_state) + VALUES ($1, $2, $3::jsonb, 'pending') + ON CONFLICT (event_id) DO NOTHING`, + [envelope.event_id, envelope.schema_version, JSON.stringify(envelope)], + ); +} + +export async function claimCloudTraceBatch( + pool: pg.Pool, + limit: number, + staleAfterMs: number, +): Promise { + const client = await pool.connect(); + try { + await client.query('BEGIN'); + const result = await client.query( + `SELECT event_id, schema_version, payload, delivery_state, attempt_count, + next_attempt_at, last_error_code, created_at, claimed_at, sent_at, dead_letter_at + FROM cloud_trace_outbox + WHERE ( + delivery_state = 'pending' + AND next_attempt_at <= now() + ) + OR ( + delivery_state = 'claimed' + AND claimed_at < now() - ($2::bigint * interval '1 millisecond') + ) + ORDER BY created_at ASC + FOR UPDATE SKIP LOCKED + LIMIT $1`, + [limit, staleAfterMs], + ); + const ids = result.rows.map((row) => row.event_id as string); + if (ids.length > 0) { + await client.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'claimed', claimed_at = now() + WHERE event_id = ANY($1::uuid[])`, + [ids], + ); + } + await client.query('COMMIT'); + return result.rows.map(mapRow); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } finally { + client.release(); + } +} + +export async function markCloudTraceSent(pool: pg.Pool, eventId: string): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'sent', sent_at = now(), last_error_code = NULL + WHERE event_id = $1`, + [eventId], + ); +} + +export async function scheduleCloudTraceRetry( + pool: pg.Pool, + eventId: string, + errorCode: string, + nextAttemptAt: Date, +): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'pending', + attempt_count = attempt_count + 1, + next_attempt_at = $2, + last_error_code = $3, + claimed_at = NULL + WHERE event_id = $1`, + [eventId, nextAttemptAt, errorCode], + ); +} + +export async function countPendingCloudTraces(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT COUNT(*)::int AS count + FROM cloud_trace_outbox + WHERE delivery_state IN ('pending', 'claimed')`, + ); + return result.rows[0]?.count ?? 0; +} + +export async function countDeadLetterCloudTraces(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT COUNT(*)::int AS count + FROM cloud_trace_outbox + WHERE delivery_state = 'dead_letter'`, + ); + return result.rows[0]?.count ?? 0; +} + +export async function getOldestPendingCloudTraceAgeMs(pool: pg.Pool): Promise { + const result = await pool.query( + `SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) * 1000 AS age_ms + FROM cloud_trace_outbox + WHERE delivery_state IN ('pending', 'claimed')`, + ); + const age = result.rows[0]?.age_ms; + if (age == null) return null; + const parsed = Number(age); + return Number.isFinite(parsed) ? Math.round(parsed) : null; +} + +export async function markCloudTraceDeadLetter( + pool: pg.Pool, + eventId: string, + errorCode: string, +): Promise { + await pool.query( + `UPDATE cloud_trace_outbox + SET delivery_state = 'dead_letter', + dead_letter_at = now(), + last_error_code = $2, + claimed_at = NULL + WHERE event_id = $1`, + [eventId, errorCode], + ); +} + +export async function purgeSentCloudTraces(pool: pg.Pool, retentionMs: number): Promise { + await pool.query( + `DELETE FROM cloud_trace_outbox + WHERE delivery_state = 'sent' + AND sent_at IS NOT NULL + AND sent_at < now() - ($1::bigint * interval '1 millisecond')`, + [retentionMs], + ); +} + +export async function purgeDeadLetterCloudTraces(pool: pg.Pool, retentionMs: number): Promise { + await pool.query( + `DELETE FROM cloud_trace_outbox + WHERE delivery_state = 'dead_letter' + AND dead_letter_at IS NOT NULL + AND dead_letter_at < now() - ($1::bigint * interval '1 millisecond')`, + [retentionMs], + ); +} + +function mapRow(row: pg.QueryResultRow): CloudTraceOutboxRow { + return { + eventId: row.event_id, + schemaVersion: row.schema_version, + payload: row.payload as CloudTraceEnvelopeV2, + deliveryState: row.delivery_state, + attemptCount: row.attempt_count, + nextAttemptAt: row.next_attempt_at, + lastErrorCode: row.last_error_code, + createdAt: row.created_at, + claimedAt: row.claimed_at, + sentAt: row.sent_at, + deadLetterAt: row.dead_letter_at, + }; +} diff --git a/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql b/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql new file mode 100644 index 0000000..df2676f --- /dev/null +++ b/packages/core/src/db/migrations/0005_cloud_trace_outbox.sql @@ -0,0 +1,25 @@ +-- Durable outbox for connected-local Cloud trace upload. + +CREATE TABLE IF NOT EXISTS cloud_trace_outbox ( + event_id UUID PRIMARY KEY, + schema_version INTEGER NOT NULL, + payload JSONB NOT NULL, + delivery_state TEXT NOT NULL CHECK ( + delivery_state IN ('pending', 'claimed', 'sent', 'dead_letter') + ), + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + claimed_at TIMESTAMPTZ, + sent_at TIMESTAMPTZ, + dead_letter_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_pending_idx + ON cloud_trace_outbox (delivery_state, next_attempt_at) + WHERE delivery_state IN ('pending', 'claimed'); + +CREATE INDEX IF NOT EXISTS cloud_trace_outbox_sent_retention_idx + ON cloud_trace_outbox (sent_at) + WHERE delivery_state = 'sent'; diff --git a/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts b/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts new file mode 100644 index 0000000..65bc66a --- /dev/null +++ b/packages/core/src/middleware/__tests__/cloud-jwt-user-binding.test.ts @@ -0,0 +1,20 @@ +/** + * Unit tests for Cloud JWT user binding helpers. + */ + +import { describe, expect, it } from 'vitest'; +import { resolveReconcileUserId } from '../cloud-jwt-user-binding.js'; + +describe('resolveReconcileUserId', () => { + it('uses body user_id when present', () => { + expect(resolveReconcileUserId('bob', 'alice')).toBe('bob'); + }); + + it('defaults to asserted JWT user when body user_id is absent', () => { + expect(resolveReconcileUserId(undefined, 'alice')).toBe('alice'); + }); + + it('returns undefined for static-key callers with no body user_id', () => { + expect(resolveReconcileUserId(undefined, null)).toBeUndefined(); + }); +}); diff --git a/packages/core/src/middleware/__tests__/dual-auth.test.ts b/packages/core/src/middleware/__tests__/dual-auth.test.ts new file mode 100644 index 0000000..966a522 --- /dev/null +++ b/packages/core/src/middleware/__tests__/dual-auth.test.ts @@ -0,0 +1,337 @@ +/** + * Unit tests for dual-auth middleware (static CORE_API_KEY + optional Cloud JWT). + */ + +import { exportJWK, generateKeyPair, SignJWT } from 'jose'; +import type { Request, Response } from 'express'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CloudJwtConfig } from '../../config.js'; +import { ASSERTED_USER_HEADER } from '../asserted-user.js'; +import { createAuthMiddleware } from '../dual-auth.js'; + +const CORE_KEY = 'test-shared-secret-do-not-leak'; +const JWKS_URL = 'https://cloud.test/.well-known/atomic-core/jwks.json'; +const ISSUER = 'https://api.test'; +const AUDIENCE = 'atomicmemory-core'; +const PROJECT_ID = 'proj_abc'; +const MEMORY_USER_ID = 'tenant-user-1'; + +function cloudJwtConfig(overrides: Partial = {}): CloudJwtConfig { + return { + jwksUrl: JWKS_URL, + issuer: ISSUER, + audience: AUDIENCE, + projectId: PROJECT_ID, + staticKeyFallbackEnabled: false, + legacyDefaultMemoryUserId: null, + ...overrides, + }; +} + +function buildRes(): { + res: Response; + statusCode: number; + body: unknown; +} { + const stub: { + res: Response; + status: (code: number) => Response; + json: (body: unknown) => Response; + statusCode: number; + body: unknown; + } = { statusCode: 0, body: undefined } as never; + stub.status = vi.fn((code: number) => { + stub.statusCode = code; + return stub.res; + }); + stub.json = vi.fn((body: unknown) => { + stub.body = body; + return stub.res; + }); + stub.res = { status: stub.status, json: stub.json } as unknown as Response; + return stub; +} + +async function signTestJwt(claims: Record): Promise { + const { privateKey } = await keyPair(); + return new SignJWT(claims) + .setProtectedHeader({ alg: 'RS256', kid: 'test-kid' }) + .setIssuer(ISSUER) + .setAudience(AUDIENCE) + .setSubject('api_key:key_test') + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey); +} + +let keyPairPromise: ReturnType | undefined; + +async function keyPair() { + keyPairPromise ??= generateKeyPair('RS256'); + return keyPairPromise; +} + +async function stubJwksFetch(): Promise { + const { publicKey } = await keyPair(); + const jwk = await exportJWK(publicKey); + jwk.kid = 'test-kid'; + jwk.alg = 'RS256'; + jwk.use = 'sig'; + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url === JWKS_URL) { + return new Response(JSON.stringify({ keys: [jwk] }), { status: 200 }); + } + throw new Error(`unexpected fetch: ${url}`); + }), + ); +} + +async function runHandler( + handler: ReturnType['middleware'], + req: Request, +): Promise<{ stub: ReturnType; next: ReturnType }> { + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { stub, next }; +} + +afterEach(() => { + vi.unstubAllGlobals(); + keyPairPromise = undefined; +}); + +describe('createAuthMiddleware', () => { + it('delegates to static bearer when cloudJwt is disabled', () => { + const { middleware: handler } = createAuthMiddleware({ coreApiKey: CORE_KEY, cloudJwt: null }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('accepts a valid Cloud JWT and injects memory-user headers', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(req.headers[ASSERTED_USER_HEADER.toLowerCase()]).toBe(MEMORY_USER_ID); + expect(stub.statusCode).toBe(0); + }); + + it('rejects request user_id different from memory_user_id with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: 'other-user' }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect((stub.body as { error_code: string }).error_code).toBe('forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects token project_id different from configured project with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: 'proj_other', + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect(next).not.toHaveBeenCalled(); + }); + + it('trusts the token project_id when no project is bound (single-key local)', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ projectId: null }), + }); + const token = await signTestJwt({ + project_id: 'proj_inferred_from_token', + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('rejects body user_id different from memory_user_id with 403', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const req = { + headers: { authorization: `Bearer ${token}` }, + body: { user_id: 'other-user' }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(403); + expect((stub.body as { error_code: string }).error_code).toBe('forbidden'); + expect(next).not.toHaveBeenCalled(); + }); + + it('accepts legacy Cloud mint without memory_user_id when legacy default is configured', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ legacyDefaultMemoryUserId: MEMORY_USER_ID }), + }); + const token = await signTestJwt({ project_id: PROJECT_ID }); + const req = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('rejects JWT missing memory_user_id with 401', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const token = await signTestJwt({ project_id: PROJECT_ID }); + const req = { headers: { authorization: `Bearer ${token}` } } as unknown as Request; + const { stub, next } = await runHandler(handler, req); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects an expired Cloud JWT', async () => { + await stubJwksFetch(); + const { privateKey } = await keyPair(); + const token = await new SignJWT({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'test-kid' }) + .setIssuer(ISSUER) + .setAudience(AUDIENCE) + .setSubject('api_key:key_test') + .setIssuedAt(Math.floor(Date.now() / 1000) - 600) + .setExpirationTime(Math.floor(Date.now() / 1000) - 300) + .sign(privateKey); + + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + const req = { headers: { authorization: `Bearer ${token}` } } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('rejects non-JWT bearer when static fallback is disabled', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ staticKeyFallbackEnabled: false }), + }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(stub.statusCode).toBe(401); + expect(next).not.toHaveBeenCalled(); + }); + + it('falls back to CORE_API_KEY only when static fallback is explicitly enabled', async () => { + await stubJwksFetch(); + const { middleware: handler } = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig({ staticKeyFallbackEnabled: true }), + }); + const req = { + headers: { authorization: `Bearer ${CORE_KEY}` }, + } as unknown as Request; + const stub = buildRes(); + const next = vi.fn(); + handler(req, stub.res, next); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); + + it('prefetch marks JWKS ready and cached verify survives fetch outage', async () => { + await stubJwksFetch(); + const bundle = createAuthMiddleware({ + coreApiKey: CORE_KEY, + cloudJwt: cloudJwtConfig(), + }); + expect(await bundle.prefetchJwks!()).toBe(true); + expect(bundle.isJwksReady!()).toBe(true); + + const token = await signTestJwt({ + project_id: PROJECT_ID, + memory_user_id: MEMORY_USER_ID, + }); + const warmReq = { + headers: { authorization: `Bearer ${token}` }, + query: { user_id: MEMORY_USER_ID }, + } as unknown as Request; + await runHandler(bundle.middleware, warmReq); + + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new Error('jwks unavailable'); + }), + ); + + const { stub, next } = await runHandler(bundle.middleware, warmReq); + expect(next).toHaveBeenCalledTimes(1); + expect(stub.statusCode).toBe(0); + }); +}); diff --git a/packages/core/src/middleware/cloud-jwt-user-binding.ts b/packages/core/src/middleware/cloud-jwt-user-binding.ts new file mode 100644 index 0000000..82fa496 --- /dev/null +++ b/packages/core/src/middleware/cloud-jwt-user-binding.ts @@ -0,0 +1,89 @@ +/** + * Cloud JWT user binding helpers shared by dual-auth and routers that + * parse JSON after auth (e.g. `/v1/documents`). + */ + +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { ASSERTED_USER_HEADER } from './asserted-user.js'; + +const ASSERTED_USER_HEADER_LOWER = ASSERTED_USER_HEADER.toLowerCase(); + +/** Read wire `user_id` from parsed body, query, or direct-storage header. */ +function readRequestUserId(req: Request): string | null { + const fromBody = pickUserId(req.body); + if (fromBody !== null) return fromBody; + const fromQuery = pickUserId(req.query); + if (fromQuery !== null) return fromQuery; + const raw = req.headers['x-atomicmemory-user-id']; + const value = Array.isArray(raw) ? raw[0] : raw; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function pickUserId(source: unknown): string | null { + if (typeof source !== 'object' || source === null) return null; + const value = (source as Record)['user_id']; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +function readAssertedUserHeader(req: Request): string | null { + const raw = req.headers[ASSERTED_USER_HEADER_LOWER]; + const value = Array.isArray(raw) ? raw[0] : raw; + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function respondForbidden(res: Response, reason: string): void { + res.status(403).json({ error_code: 'forbidden', error: reason }); +} + +/** + * When a JWT authenticated the request, wire `user_id` must match the + * verified `memory_user_id` injected as {@link ASSERTED_USER_HEADER}. + */ +export function enforceMemoryUserBinding( + req: Request, + memoryUserId: string, + res: Response, +): boolean { + const requestUserId = readRequestUserId(req); + if (requestUserId !== null && requestUserId !== memoryUserId) { + respondForbidden(res, 'request user_id does not match token memory_user_id'); + return false; + } + return true; +} + +/** Re-check binding after a router-owned body parser runs post-auth. */ +function enforceCloudJwtUserBindingFromHeader(req: Request, res: Response): boolean { + const assertedUser = readAssertedUserHeader(req); + if (assertedUser === null) { + return true; + } + return enforceMemoryUserBinding(req, assertedUser, res); +} + +/** Express guard for routers that parse JSON after global auth. */ +export function cloudJwtUserBindingGuard(): RequestHandler { + return (req: Request, res: Response, next: NextFunction): void => { + if (enforceCloudJwtUserBindingFromHeader(req, res)) { + next(); + } + }; +} + +/** JWT-authenticated reconcile and similar routes: never run global scope. */ +export function readAssertedUserId(req: Request): string | null { + return readAssertedUserHeader(req); +} + +/** Prefer explicit body user_id; otherwise scope JWT callers to asserted identity. */ +export function resolveReconcileUserId( + bodyUserId: string | undefined, + assertedUserId: string | null, +): string | undefined { + if (bodyUserId) { + return bodyUserId; + } + return assertedUserId ?? undefined; +} diff --git a/packages/core/src/middleware/dual-auth.ts b/packages/core/src/middleware/dual-auth.ts new file mode 100644 index 0000000..83ee4c7 --- /dev/null +++ b/packages/core/src/middleware/dual-auth.ts @@ -0,0 +1,166 @@ +/** + * Dual authentication for SDK-facing `/v1/*` routes. + * + * Accepts either the deployment-wide `CORE_API_KEY` (when explicitly enabled + * as a fallback) or a Cloud-issued RS256 JWT verified against a remote JWKS. + * On JWT success the verified `memory_user_id` / `project_id` claims are + * injected as asserted-user headers so downstream scope enforcement sees a + * stable end-user identity. + */ + +import type { NextFunction, Request, RequestHandler, Response } from 'express'; +import { createRemoteJWKSet, jwtVerify, type JWTVerifyGetKey } from 'jose'; +import type { CloudJwtConfig } from '../config.js'; +import { ASSERTED_USER_HEADER } from './asserted-user.js'; +import { enforceMemoryUserBinding } from './cloud-jwt-user-binding.js'; +import { readBearerToken, requireBearer, respondUnauthenticated } from './require-bearer.js'; +const JWKS_PREFETCH_TIMEOUT_MS = 5_000; + +export interface DualAuthOptions { + coreApiKey: string; + cloudJwt: CloudJwtConfig | null; +} + +export interface AuthMiddlewareBundle { + middleware: RequestHandler; + prefetchJwks?: () => Promise; + isJwksReady?: () => boolean; +} + +export interface CloudJwtVerifier { + config: CloudJwtConfig; + jwks: JWTVerifyGetKey; + prefetch: () => Promise; + isReady: () => boolean; + markReady: () => void; +} + +function looksLikeJwt(token: string): boolean { + const parts = token.split('.'); + return parts.length === 3 && parts.every((part) => part.length > 0); +} + +function respondForbidden(res: Response, reason: string): void { + res.status(403).json({ error_code: 'forbidden', error: reason }); +} + +function injectAssertedIdentity(req: Request, memoryUserId: string): void { + req.headers[ASSERTED_USER_HEADER.toLowerCase()] = memoryUserId; +} + +/** + * True when a project binding is configured and the token's `project_id` + * claim does not match it. A null binding means single-tenant local mode: + * Core trusts whatever project the Cloud-minted token carries. + */ +function tokenProjectMismatch(boundProjectId: string | null, tokenProjectId: string): boolean { + return boundProjectId !== null && tokenProjectId !== boundProjectId; +} + +function readRequiredStringClaim(payload: Record, claim: string): string | null { + const value = payload[claim]; + return typeof value === 'string' && value.length > 0 ? value : null; +} + +/** + * Create a shared JWKS verifier for Cloud JWT auth. The remote JWK set is + * constructed once and reused across requests. + */ +function createCloudJwtVerifier(cloudJwt: CloudJwtConfig): CloudJwtVerifier { + const jwks = createRemoteJWKSet(new URL(cloudJwt.jwksUrl)); + let jwksReady = false; + + return { + config: cloudJwt, + jwks, + isReady: () => jwksReady, + markReady: () => { + jwksReady = true; + }, + async prefetch(): Promise { + try { + const response = await fetch(cloudJwt.jwksUrl, { + signal: AbortSignal.timeout(JWKS_PREFETCH_TIMEOUT_MS), + }); + if (response.ok) { + jwksReady = true; + } + } catch { + // Degraded until a verify path loads keys into the jose cache. + } + return jwksReady; + }, + }; +} + +/** + * Build auth middleware: static bearer and/or Cloud JWT (when configured). + */ +export function createAuthMiddleware(options: DualAuthOptions): AuthMiddlewareBundle { + const staticBearer = requireBearer(options.coreApiKey); + if (!options.cloudJwt) { + return { middleware: staticBearer }; + } + + const verifier = createCloudJwtVerifier(options.cloudJwt); + + const middleware: RequestHandler = (req: Request, res: Response, next: NextFunction): void => { + const token = readBearerToken(req); + if (token === null) { + respondUnauthenticated(res, 'missing or malformed Authorization header'); + return; + } + + if (!looksLikeJwt(token)) { + if (options.cloudJwt!.staticKeyFallbackEnabled) { + staticBearer(req, res, next); + return; + } + respondUnauthenticated(res, 'invalid api key'); + return; + } + + void (async () => { + try { + const { payload } = await jwtVerify(token, verifier.jwks, { + algorithms: ['RS256'], + issuer: options.cloudJwt!.issuer, + audience: options.cloudJwt!.audience, + }); + verifier.markReady(); + + const principalId = typeof payload.sub === 'string' ? payload.sub : null; + let memoryUserId = readRequiredStringClaim(payload as Record, 'memory_user_id'); + if (!memoryUserId) { + memoryUserId = options.cloudJwt!.legacyDefaultMemoryUserId; + } + const projectId = readRequiredStringClaim(payload as Record, 'project_id'); + + if (!principalId || !memoryUserId || !projectId) { + respondUnauthenticated(res, 'invalid api key'); + return; + } + + if (tokenProjectMismatch(options.cloudJwt!.projectId, projectId)) { + respondForbidden(res, 'token project_id does not match configured Cloud project'); + return; + } + + if (!enforceMemoryUserBinding(req, memoryUserId, res)) { + return; + } + + injectAssertedIdentity(req, memoryUserId); + next(); + } catch { + respondUnauthenticated(res, 'invalid api key'); + } + })(); + }; + + return { + middleware, + prefetchJwks: () => verifier.prefetch(), + isJwksReady: () => verifier.isReady(), + }; +} diff --git a/packages/core/src/middleware/require-bearer.ts b/packages/core/src/middleware/require-bearer.ts index dbff639..b431909 100644 --- a/packages/core/src/middleware/require-bearer.ts +++ b/packages/core/src/middleware/require-bearer.ts @@ -22,6 +22,19 @@ import type { Request, RequestHandler, Response, NextFunction } from 'express'; const BEARER_PREFIX = 'Bearer '; +/** Read the bearer token from `Authorization`, or null when missing/malformed. */ +export function readBearerToken(req: Request): string | null { + const raw = req.headers['authorization']; + if (typeof raw !== 'string' || !raw.startsWith(BEARER_PREFIX)) return null; + const token = raw.slice(BEARER_PREFIX.length).trim(); + return token.length > 0 ? token : null; +} + +/** Standard 401 envelope for bearer/JWT auth failures. */ +export function respondUnauthenticated(res: Response, reason: string): void { + res.status(401).json({ error_code: 'unauthenticated', error: reason }); +} + /** * Build a middleware that admits a request only when its * `Authorization` header carries the configured shared key. The @@ -35,7 +48,7 @@ export function requireBearer(expectedApiKey: string): RequestHandler { } const expectedBuffer = Buffer.from(expectedApiKey, 'utf8'); return (req: Request, res: Response, next: NextFunction): void => { - const headerValue = readAuthorizationHeader(req); + const headerValue = readBearerToken(req); if (headerValue === null) { respondUnauthenticated(res, 'missing or malformed Authorization header'); return; @@ -52,14 +65,3 @@ export function requireBearer(expectedApiKey: string): RequestHandler { next(); }; } - -function readAuthorizationHeader(req: Request): string | null { - const raw = req.headers['authorization']; - if (typeof raw !== 'string' || !raw.startsWith(BEARER_PREFIX)) return null; - const token = raw.slice(BEARER_PREFIX.length).trim(); - return token.length > 0 ? token : null; -} - -function respondUnauthenticated(res: Response, reason: string): void { - res.status(401).json({ error_code: 'unauthenticated', error: reason }); -} diff --git a/packages/core/src/routes/documents.ts b/packages/core/src/routes/documents.ts index f128354..9ee71bd 100644 --- a/packages/core/src/routes/documents.ts +++ b/packages/core/src/routes/documents.ts @@ -35,6 +35,7 @@ import { handleRouteError } from './route-errors.js'; import { validateBody, validateQuery, validateParams } from '../middleware/validate.js'; import { validateResponse } from '../middleware/validate-response.js'; import { rejectNulInBody } from '../middleware/reject-nul-bytes.js'; +import { cloudJwtUserBindingGuard } from '../middleware/cloud-jwt-user-binding.js'; import { DOCUMENT_RESPONSE_SCHEMAS } from './response-schema-map.js'; import { DocumentByIdQuerySchema, @@ -209,6 +210,7 @@ export function createDocumentRouter( // raw-upload route's Buffer body is skipped by the guard. router.use(express.json({ limit: ROUTER_JSON_BODY_LIMIT })); router.use(rejectNulInBody); + router.use(cloudJwtUserBindingGuard()); // Step 5 — JSON-body routes. The Phase C failure-marker routes // (`/:id/extraction-failure`, `/:id/index-failure`) live with the @@ -383,6 +385,7 @@ function registerIndexRoute(router: Router, service: DocumentService): void { '/:id/index', indexJsonParser, rejectNulInBody, + cloudJwtUserBindingGuard(), validateParams(DocumentIdParamSchema), validateBody(IndexDocumentBodySchema), async (req: Request, res: Response) => { diff --git a/packages/core/src/routes/memories.ts b/packages/core/src/routes/memories.ts index 119be5d..125dc45 100644 --- a/packages/core/src/routes/memories.ts +++ b/packages/core/src/routes/memories.ts @@ -48,6 +48,10 @@ import { import type { AgentScope, WorkspaceContext } from '../db/repository-types.js'; import { handleRouteError } from './route-errors.js'; import { validateBody, validateQuery, validateParams } from '../middleware/validate.js'; +import { + readAssertedUserId, + resolveReconcileUserId, +} from '../middleware/cloud-jwt-user-binding.js'; import { CORS_ALLOWED_HEADERS_VALUE } from '../app/cors-headers.js'; import { validateResponse } from '../middleware/validate-response.js'; import { MEMORY_RESPONSE_SCHEMAS } from './response-schema-map.js'; @@ -757,8 +761,9 @@ function registerReconcileRoute(router: Router, service: MemoryService): void { router.post('/reconcile', validateBody(ReconcileBodySchema), async (req: Request, res: Response) => { try { const { userId } = req.body as { userId: string | undefined }; - const result = userId - ? await service.reconcileDeferred(userId) + const effectiveUserId = resolveReconcileUserId(userId, readAssertedUserId(req)); + const result = effectiveUserId + ? await service.reconcileDeferred(effectiveUserId) : await service.reconcileDeferredAll(); res.json(formatReconciliationResponse(result)); } catch (err) { diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 7f8a1b2..1becfbc 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -15,17 +15,32 @@ import { pool } from './db/pool.js'; import { createCoreRuntime, type CoreRuntime } from './app/runtime-container.js'; import { createApp } from './app/create-app.js'; import { checkEmbeddingDimensions } from './app/startup-checks.js'; +import { startDeferredAudnScheduler, type DeferredAudnScheduler } from './services/deferred-audn-scheduler.js'; +import { startCloudTraceSync, stopCloudTraceSync } from './services/cloud-trace-sync.js'; // Process-lifecycle signal handlers reference `runtime` via a closure // captured AFTER `bootstrap()` resolves — wired below. Reconciler // startup stays disabled until `buildReconcilerDeps` returns a // non-null bundle backed by the active storage provider. let runtime: CoreRuntime | null = null; +let deferredAudnScheduler: DeferredAudnScheduler | null = null; async function bootstrap(): Promise { runtime = await createCoreRuntime({ pool }); const app = createApp(runtime); + const prefetchCloudJwt = app.get('cloudJwtPrefetch') as (() => Promise) | undefined; + if (prefetchCloudJwt) { + const ready = await prefetchCloudJwt(); + if (!ready) { + console.warn( + '[startup] Cloud JWKS prefetch did not complete; JWT auth remains degraded until keys load', + ); + } else { + console.log('[startup] Cloud JWKS prefetch ok'); + } + } + const check = await checkEmbeddingDimensions(runtime.pool, runtime.config); if (!check.ok) { console.error(`[startup] FATAL: ${check.message}`); @@ -33,9 +48,28 @@ async function bootstrap(): Promise { } console.log(`[startup] ${check.message}`); + if (runtime.config.cloudTraceSync?.enabled) { + await startCloudTraceSync(runtime.pool, runtime.config.cloudTraceSync); + console.log('[startup] Cloud trace sync uploader started'); + } + app.listen(runtime.config.port, () => { console.log(`AtomicMemory Core running on http://localhost:${runtime!.config.port}`); }); + + // Drain the deferred-AUDN queue in the background so ingest stays fast while + // AUDN is still applied. Opt-in: only when deferred AUDN is enabled. + if (runtime.config.deferredAudnEnabled && runtime.config.deferredAudnAutoReconcile) { + const memory = runtime.services.memory; + deferredAudnScheduler = startDeferredAudnScheduler({ + reconcile: () => memory.reconcileDeferredAll(), + intervalMs: runtime.config.deferredAudnReconcileIntervalMs, + onError: (err) => console.error('[deferred-audn-scheduler] reconcile tick failed:', err), + }); + console.log( + `[startup] deferred-AUDN auto-reconcile every ${runtime.config.deferredAudnReconcileIntervalMs}ms`, + ); + } } bootstrap().catch((err) => { @@ -54,6 +88,8 @@ process.on('unhandledRejection', (reason) => { async function shutdown(signal: string): Promise { console.log(`[shutdown] Received ${signal}, closing...`); + if (deferredAudnScheduler) await deferredAudnScheduler.stop(); + await stopCloudTraceSync(); const closing = runtime ? runtime.pool.end() : pool.end(); await closing; process.exit(0); diff --git a/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts b/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts new file mode 100644 index 0000000..1508d44 --- /dev/null +++ b/packages/core/src/services/__tests__/cloud-trace-envelope.test.ts @@ -0,0 +1,85 @@ +/** + * Unit tests for Cloud trace envelope redaction. + */ + +import { describe, expect, it } from 'vitest'; +import { buildCloudTraceEnvelope } from '../cloud-trace-envelope.js'; + +describe('buildCloudTraceEnvelope', () => { + it('builds a v2 envelope with allowlisted summary fields', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 42, + summary: { + user_id: 'tenant-user-1', + operation_detail: 'consensus ingest (2 fact(s))', + new_memory_id: 'mem_abc', + content_len: 42, + }, + evidence: { source: 'core' }, + }); + + expect(envelope.schema_version).toBe(2); + expect(envelope.summary.user_id).toBe('tenant-user-1'); + expect(envelope.summary.operation_detail).toBe('consensus ingest (2 fact(s))'); + expect(envelope.summary.content_len).toBe(42); + expect(envelope).not.toHaveProperty('actor'); + }); + + it('drops disallowed summary keys including raw content previews', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.search', + outcome: 'success', + durationMs: 10, + summary: { + user_id: 'tenant-user-1', + input_summary: 'secret query text', + query_len: 17, + }, + }); + + expect(envelope.summary).toEqual({ user_id: 'tenant-user-1', query_len: 17 }); + }); + + it('allowlists only bounded evidence string fields', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.search', + outcome: 'success', + durationMs: 10, + evidence: { + mode: 'verbatim', + source_site: 'chatgpt', + as_of: '2026-01-01T00:00:00Z', + api_key: 'secret', + nested: { route: 'search' }, + tokens: ['leak'], + }, + }); + + expect(envelope.evidence).toEqual({ mode: 'verbatim', source_site: 'chatgpt' }); + }); + + it('drops oversized source_site evidence strings', () => { + const envelope = buildCloudTraceEnvelope({ + eventId: '550e8400-e29b-41d4-a716-446655440000', + coreInstanceId: 'core-local-1', + occurredAt: '2026-07-10T16:00:00Z', + operation: 'memory.ingest', + outcome: 'success', + durationMs: 1, + evidence: { source_site: 'x'.repeat(121) }, + }); + + expect(envelope.evidence).toEqual({}); + }); +}); diff --git a/packages/core/src/services/__tests__/cloud-trace-sync.test.ts b/packages/core/src/services/__tests__/cloud-trace-sync.test.ts new file mode 100644 index 0000000..9f8028a --- /dev/null +++ b/packages/core/src/services/__tests__/cloud-trace-sync.test.ts @@ -0,0 +1,22 @@ +/** + * Unit tests for Cloud trace sync auth pause recovery. + */ + +import { describe, expect, it } from 'vitest'; +import { resetCloudTraceAuthPause } from '../cloud-trace-sync.js'; + +describe('resetCloudTraceAuthPause', () => { + it('clears paused and auth-related lastErrorCode', () => { + const state = { paused: true, lastErrorCode: 'auth_401' }; + resetCloudTraceAuthPause(state); + expect(state.paused).toBe(false); + expect(state.lastErrorCode).toBeNull(); + }); + + it('clears non-auth lastErrorCode after a successful upload', () => { + const state = { paused: true, lastErrorCode: 'network_error' }; + resetCloudTraceAuthPause(state); + expect(state.paused).toBe(false); + expect(state.lastErrorCode).toBeNull(); + }); +}); diff --git a/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts b/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts new file mode 100644 index 0000000..99d32c5 --- /dev/null +++ b/packages/core/src/services/__tests__/deferred-audn-scheduler.test.ts @@ -0,0 +1,68 @@ +/** + * Tests for the deferred-AUDN background scheduler: it polls reconcile() on the + * interval, never overlaps ticks, surfaces errors via onError, and stop() awaits + * the in-flight drain. Uses fake timers to stay deterministic (no real sleeps). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + startDeferredAudnScheduler, + type DeferredAudnSchedulerOptions, +} from '../deferred-audn-scheduler.js'; +import type { ReconciliationResult } from '../deferred-audn.js'; + +const EMPTY: ReconciliationResult = { + processed: 0, resolved: 0, noops: 0, updates: 0, + supersedes: 0, deletes: 0, adds: 0, errors: 0, durationMs: 0, +}; + +function makeOptions(over: Partial = {}): DeferredAudnSchedulerOptions { + return { + reconcile: vi.fn(async () => EMPTY), + intervalMs: 1000, + onError: vi.fn(), + ...over, + }; +} + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe('startDeferredAudnScheduler', () => { + it('is idle before the first tick', () => { + const s = startDeferredAudnScheduler(makeOptions()); + expect(s.isRunning).toBe(false); + }); + + it('calls reconcile on each interval tick', async () => { + const opts = makeOptions(); + startDeferredAudnScheduler(opts); + await vi.advanceTimersByTimeAsync(2500); + expect(opts.reconcile).toHaveBeenCalledTimes(2); + }); + + it('does not overlap ticks while a reconcile is in flight', async () => { + let release!: () => void; + const reconcile = vi.fn(() => new Promise((r) => { release = () => r(EMPTY); })); + startDeferredAudnScheduler(makeOptions({ reconcile })); + await vi.advanceTimersByTimeAsync(3000); + expect(reconcile).toHaveBeenCalledTimes(1); + release(); + }); + + it('routes a rejected reconcile to onError instead of throwing', async () => { + const onError = vi.fn(); + const reconcile = vi.fn(async () => { throw new Error('boom'); }); + startDeferredAudnScheduler(makeOptions({ reconcile, onError })); + await vi.advanceTimersByTimeAsync(1000); + expect(onError).toHaveBeenCalledOnce(); + }); + + it('stop() clears the interval and awaits the in-flight drain', async () => { + const opts = makeOptions(); + const s = startDeferredAudnScheduler(opts); + await vi.advanceTimersByTimeAsync(1000); + await s.stop(); + await vi.advanceTimersByTimeAsync(5000); + expect(opts.reconcile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/src/services/__tests__/memory-service-config.test.ts b/packages/core/src/services/__tests__/memory-service-config.test.ts index 80d08d3..1f6dd32 100644 --- a/packages/core/src/services/__tests__/memory-service-config.test.ts +++ b/packages/core/src/services/__tests__/memory-service-config.test.ts @@ -125,6 +125,7 @@ describe('MemoryService config seam', () => { '', undefined, undefined, + false, ); }); @@ -152,6 +153,7 @@ describe('MemoryService config seam', () => { 'https://example.test/thread', sessionTimestamp, 'thread-1', + false, ); }); @@ -193,6 +195,7 @@ describe('MemoryService config seam', () => { '', undefined, undefined, + false, ); }); @@ -220,6 +223,7 @@ describe('MemoryService config seam', () => { 'https://example.test/quick', sessionTimestamp, 'thread-quick', + false, ); }); @@ -300,6 +304,7 @@ describe('MemoryService config seam', () => { workspace, sessionTimestamp, 'thread-workspace', + false, ); }); diff --git a/packages/core/src/services/audn-decision-executor.ts b/packages/core/src/services/audn-decision-executor.ts index 3b94b73..4ccab08 100644 --- a/packages/core/src/services/audn-decision-executor.ts +++ b/packages/core/src/services/audn-decision-executor.ts @@ -10,6 +10,7 @@ import { type AUDNDecision } from './extraction.js'; import { emitAuditEvent } from './audit-events.js'; import { recordContradictionLesson } from './lesson-service.js'; import { emitLineageEvent } from './memory-lineage.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; import { applyOpinionSignal, audnActionToOpinionSignal } from './memory-network.js'; import { buildAtomicFactProjection, buildForesightProjections } from './memcell-projection.js'; import { @@ -241,6 +242,22 @@ async function updateCanonicalFact( throw new Error(`AUDN UPDATE failed: missing successor canonical object for "${target.memoryId}"`); } await deps.stores.memory.updateMemoryMetadata(userId, target.memoryId, { cmo_id: lineage.cmoId }); + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.update', + durationMs: 0, + userId, + summary: { + fact_len: fact.fact.length, + previous_memory_id: target.memoryId, + new_memory_id: target.memoryId, + }, + evidence: { source: 'audn-update' }, + }, + ); return { outcome: 'updated', memoryId: target.memoryId }; } diff --git a/packages/core/src/services/cloud-trace-envelope.ts b/packages/core/src/services/cloud-trace-envelope.ts new file mode 100644 index 0000000..16be1f5 --- /dev/null +++ b/packages/core/src/services/cloud-trace-envelope.ts @@ -0,0 +1,112 @@ +/** + * Redacted Cloud trace envelope builder for OSS Core outbound sync. + * + * Keeps secrets and verbatim memory/query text out of the durable outbox payload. + * Summary and evidence fields are allowlisted; unknown keys are dropped. + */ + +const CLOUD_TRACE_SCHEMA_VERSION = 2 as const; +const MAX_EVIDENCE_STRING_LEN = 120; + +export type CloudTraceOperation = + | 'memory.ingest' + | 'memory.update' + | 'memory.delete' + | 'memory.search'; + +export type CloudTraceOutcome = 'success' | 'error'; + +export interface BuildCloudTraceEnvelopeInput { + eventId: string; + coreInstanceId: string; + occurredAt: string; + operation: CloudTraceOperation; + outcome: CloudTraceOutcome; + durationMs: number; + summary?: Record; + evidence?: Record; +} + +export interface CloudTraceEnvelopeV2 { + schema_version: typeof CLOUD_TRACE_SCHEMA_VERSION; + event_id: string; + core_instance_id: string; + occurred_at: string; + operation: CloudTraceOperation; + outcome: CloudTraceOutcome; + duration_ms: number; + summary: Record; + evidence: Record; +} + +/** Summary keys permitted on outbound Cloud trace envelopes. */ +const ALLOWED_SUMMARY_KEYS = new Set([ + 'user_id', + 'result_count', + 'new_memory_id', + 'previous_memory_id', + 'episode_id', + 'memories_stored', + 'memories_updated', + 'memories_deleted', + 'content_len', + 'query_len', + 'fact_len', + 'operation_detail', +]); + +/** Evidence keys permitted on outbound Cloud trace envelopes. */ +const ALLOWED_EVIDENCE_KEYS = new Set(['mode', 'source', 'source_site']); + +/** + * Build a version-2 envelope suitable for Cloud trace ingest. + * Allowlists summary and evidence fields; drops unknown keys and arrays. + */ +export function buildCloudTraceEnvelope( + input: BuildCloudTraceEnvelopeInput, +): CloudTraceEnvelopeV2 { + if (!input.eventId || !input.coreInstanceId) { + throw new Error('cloud trace envelope requires eventId and coreInstanceId'); + } + if (input.durationMs < 0) { + throw new Error('durationMs must be non-negative'); + } + + return { + schema_version: CLOUD_TRACE_SCHEMA_VERSION, + event_id: input.eventId, + core_instance_id: input.coreInstanceId, + occurred_at: input.occurredAt, + operation: input.operation, + outcome: input.outcome, + duration_ms: input.durationMs, + summary: allowlistSummary(input.summary ?? {}), + evidence: allowlistEvidence(input.evidence ?? {}), + }; +} + +function allowlistSummary(value: Record): Record { + const out: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!ALLOWED_SUMMARY_KEYS.has(key)) continue; + if (typeof entry === 'number' && Number.isFinite(entry)) { + out[key] = entry; + continue; + } + if (typeof entry === 'string' && entry.length > 0 && entry.length <= MAX_EVIDENCE_STRING_LEN) { + out[key] = entry; + } + } + return out; +} + +function allowlistEvidence(value: Record): Record { + const out: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (!ALLOWED_EVIDENCE_KEYS.has(key)) continue; + if (typeof entry === 'string' && entry.length > 0 && entry.length <= MAX_EVIDENCE_STRING_LEN) { + out[key] = entry; + } + } + return out; +} diff --git a/packages/core/src/services/cloud-trace-sync.ts b/packages/core/src/services/cloud-trace-sync.ts new file mode 100644 index 0000000..98a69c7 --- /dev/null +++ b/packages/core/src/services/cloud-trace-sync.ts @@ -0,0 +1,414 @@ +/** + * OSS Core → Cloud trace sync runtime: enqueue hooks, background uploader, health. + */ + +import { randomUUID } from 'node:crypto'; +import type pg from 'pg'; +import { cloudApiPost } from '../cloud/cloud-api-client.js'; +import { resolveCoreInstanceId } from '../cloud/instance-id.js'; +import { sendRuntimeHeartbeat } from '../cloud/heartbeat-client.js'; +import type { CloudTraceSyncConfig } from '../config.js'; +import { readPackageVersion } from '../db/migration-schema.js'; +import { + claimCloudTraceBatch, + countDeadLetterCloudTraces, + countPendingCloudTraces, + enqueueCloudTraceOutbox, + getOldestPendingCloudTraceAgeMs, + markCloudTraceDeadLetter, + markCloudTraceSent, + purgeDeadLetterCloudTraces, + purgeSentCloudTraces, + scheduleCloudTraceRetry, +} from '../db/cloud-trace-outbox-repository.js'; +import { + buildCloudTraceEnvelope, + type CloudTraceOperation, + type CloudTraceOutcome, +} from './cloud-trace-envelope.js'; + +const RUNTIME_CAPABILITIES = ['memory.read', 'memory.write', 'trace.stream'] as const; +const HEALTH_CACHE_TTL_MS = 5_000; + +export interface CloudTraceHealthSnapshot { + status: 'healthy' | 'degraded' | 'paused' | 'disabled'; + pendingCount: number; + oldestPendingAgeMs: number | null; + lastSuccessAt: string | null; + deadLetterCount: number; + lastErrorCode: string | null; + lastHeartbeatAt: string | null; + heartbeatStatus: 'ok' | 'failed' | 'auth_paused' | 'skipped'; + enqueueFailures: number; +} + +/** Public liveness view — no backlog counts or error codes. */ +export interface CloudTracePublicHealth { + status: 'healthy' | 'degraded' | 'paused' | 'disabled'; +} + +export interface RecordCloudTraceOperationInput { + operation: CloudTraceOperation; + outcome?: CloudTraceOutcome; + durationMs: number; + userId?: string; + summary?: Record; + evidence?: Record; +} + +interface UploaderRuntime { + pool: pg.Pool; + config: CloudTraceSyncConfig; + instanceId: string; + timer: NodeJS.Timeout | null; + draining: boolean; + paused: boolean; + inFlight: Promise; + lastSuccessAt: Date | null; + lastErrorCode: string | null; + enqueueFailures: number; + lastHeartbeatAt: Date | null; + heartbeatStatus: CloudTraceHealthSnapshot['heartbeatStatus']; + healthCache: { expiresAt: number; snapshot: CloudTraceHealthSnapshot } | null; +} + +let uploaderRuntime: UploaderRuntime | null = null; + +export function recordCloudTraceOperation( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig | null | undefined, + instanceId: string | null | undefined, + input: RecordCloudTraceOperationInput, +): void { + if (!syncConfig?.enabled || !instanceId) return; + void enqueueCloudTraceFromInput(pool, syncConfig, instanceId, input).catch((error) => { + if (uploaderRuntime) uploaderRuntime.enqueueFailures += 1; + console.error('[cloud-trace-sync] enqueue failed:', error); + }); +} + +async function enqueueCloudTraceFromInput( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig, + instanceId: string, + input: RecordCloudTraceOperationInput, +): Promise { + const pending = await countPendingCloudTraces(pool); + if (pending >= syncConfig.maxPending) { + if (uploaderRuntime) uploaderRuntime.enqueueFailures += 1; + throw new Error(`cloud trace outbox backlog at cap (${syncConfig.maxPending})`); + } + const summary = { + user_id: input.userId ?? 'default', + ...(input.summary ?? {}), + }; + const envelope = buildCloudTraceEnvelope({ + eventId: randomUUID(), + coreInstanceId: instanceId, + occurredAt: new Date().toISOString(), + operation: input.operation, + outcome: input.outcome ?? 'success', + durationMs: Math.max(0, Math.round(input.durationMs)), + summary, + evidence: input.evidence, + }); + await enqueueCloudTraceOutbox(pool, envelope); +} + +export async function startCloudTraceSync( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig, +): Promise { + if (uploaderRuntime) return; + const instanceId = await resolveCoreInstanceId(syncConfig.instanceId); + syncConfig.instanceId = instanceId; + uploaderRuntime = { + pool, + config: syncConfig, + instanceId, + timer: null, + draining: false, + paused: false, + inFlight: Promise.resolve(), + lastSuccessAt: null, + lastErrorCode: null, + enqueueFailures: 0, + lastHeartbeatAt: null, + heartbeatStatus: 'skipped', + healthCache: null, + }; + scheduleUploaderTick(); +} + +export async function stopCloudTraceSync(): Promise { + const runtime = uploaderRuntime; + if (!runtime) return; + runtime.draining = true; + if (runtime.timer) clearTimeout(runtime.timer); + runtime.timer = null; + const deadline = Date.now() + runtime.config.shutdownDrainMs; + while (Date.now() < deadline) { + await runtime.inFlight; + const pending = await countPendingCloudTraces(runtime.pool); + if (pending === 0) break; + if (runtime.paused) break; + const claimed = await claimCloudTraceBatch( + runtime.pool, + runtime.config.batchSize, + runtime.config.claimStaleMs, + ); + if (claimed.length === 0) break; + for (const row of claimed) { + await uploadRow(runtime, row.eventId, row.payload, row.attemptCount); + } + await sleep(Math.min(runtime.config.pollIntervalMs, deadline - Date.now())); + } + uploaderRuntime = null; +} + +function sleep(ms: number): Promise { + if (ms <= 0) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function getCloudTraceHealthSnapshot(): CloudTraceHealthSnapshot { + const runtime = uploaderRuntime; + if (!runtime) { + return { + status: 'disabled', + pendingCount: 0, + oldestPendingAgeMs: null, + lastSuccessAt: null, + deadLetterCount: 0, + lastErrorCode: null, + lastHeartbeatAt: null, + heartbeatStatus: 'skipped', + enqueueFailures: 0, + }; + } + return { + status: runtime.paused ? 'paused' : runtime.lastErrorCode ? 'degraded' : 'healthy', + pendingCount: 0, + oldestPendingAgeMs: null, + lastSuccessAt: runtime.lastSuccessAt?.toISOString() ?? null, + deadLetterCount: 0, + lastErrorCode: runtime.lastErrorCode, + lastHeartbeatAt: runtime.lastHeartbeatAt?.toISOString() ?? null, + heartbeatStatus: runtime.heartbeatStatus, + enqueueFailures: runtime.enqueueFailures, + }; +} + +export async function getCloudTraceHealthSnapshotAsync( + pool: pg.Pool, + syncConfig: CloudTraceSyncConfig | null | undefined, +): Promise { + if (!syncConfig?.enabled) { + return getCloudTraceHealthSnapshot(); + } + const runtime = uploaderRuntime; + const now = Date.now(); + if (runtime?.healthCache && runtime.healthCache.expiresAt > now) { + return runtime.healthCache.snapshot; + } + const [pendingCount, oldestPendingAgeMs, deadLetterCount] = await Promise.all([ + countPendingCloudTraces(pool), + getOldestPendingCloudTraceAgeMs(pool), + countDeadLetterCloudTraces(pool), + ]); + let status: CloudTraceHealthSnapshot['status'] = 'healthy'; + if (runtime?.paused) status = 'paused'; + else if (deadLetterCount > 0 || (oldestPendingAgeMs ?? 0) > 300_000) status = 'degraded'; + const snapshot: CloudTraceHealthSnapshot = { + status, + pendingCount, + oldestPendingAgeMs, + lastSuccessAt: runtime?.lastSuccessAt?.toISOString() ?? null, + deadLetterCount, + lastErrorCode: runtime?.lastErrorCode ?? null, + lastHeartbeatAt: runtime?.lastHeartbeatAt?.toISOString() ?? null, + heartbeatStatus: runtime?.heartbeatStatus ?? 'skipped', + enqueueFailures: runtime?.enqueueFailures ?? 0, + }; + if (runtime) { + runtime.healthCache = { expiresAt: now + HEALTH_CACHE_TTL_MS, snapshot }; + } + return snapshot; +} + +export function toCloudTracePublicHealth( + snapshot: CloudTraceHealthSnapshot, +): CloudTracePublicHealth { + return { status: snapshot.status }; +} + +function scheduleUploaderTick(): void { + const runtime = uploaderRuntime; + if (!runtime || runtime.draining) return; + runtime.timer = setTimeout(() => { + runtime.inFlight = uploadOnce(runtime) + .catch((error) => { + runtime.lastErrorCode = 'upload_loop_error'; + console.error('[cloud-trace-sync] upload tick failed:', error); + }) + .finally(() => { + purgeSentCloudTraces(runtime.pool, runtime.config.sentRetentionMs).catch(() => {}); + purgeDeadLetterCloudTraces(runtime.pool, runtime.config.deadLetterRetentionMs).catch( + () => {}, + ); + scheduleUploaderTick(); + }); + }, runtime.config.pollIntervalMs); +} + +async function uploadOnce(runtime: UploaderRuntime): Promise { + if (!runtime.draining) { + await sendHeartbeatOnce(runtime); + } + if (runtime.paused) return; + const batch = await claimCloudTraceBatch( + runtime.pool, + runtime.config.batchSize, + runtime.config.claimStaleMs, + ); + for (const row of batch) { + await uploadRow(runtime, row.eventId, row.payload, row.attemptCount); + } +} + +function resolveLocalUrl(): string { + const port = process.env.PORT ?? '17350'; + return `http://127.0.0.1:${port}`; +} + +function clearAuthPause(runtime: UploaderRuntime): void { + resetCloudTraceAuthPause(runtime); +} + +/** Clear sticky auth pause after a successful heartbeat or upload. */ +export function resetCloudTraceAuthPause(state: { + paused: boolean; + lastErrorCode: string | null; +}): void { + state.paused = false; + state.lastErrorCode = null; +} + +async function sendHeartbeatOnce(runtime: UploaderRuntime): Promise { + const packageVersion = readPackageVersion(); + const result = await sendRuntimeHeartbeat({ + apiUrl: runtime.config.apiUrl, + apiKey: runtime.config.apiKey, + payload: { + core_instance_id: runtime.instanceId, + core_version: packageVersion, + connector_version: packageVersion, + capabilities: [...RUNTIME_CAPABILITIES], + local_url: resolveLocalUrl(), + }, + }); + if (result.ok) { + runtime.lastHeartbeatAt = new Date(); + runtime.heartbeatStatus = 'ok'; + clearAuthPause(runtime); + return; + } + runtime.heartbeatStatus = result.errorCode?.startsWith('auth_') ? 'auth_paused' : 'failed'; + if (result.errorCode?.startsWith('auth_')) { + runtime.paused = true; + runtime.lastErrorCode = result.errorCode; + } +} + +async function uploadRow( + runtime: UploaderRuntime, + eventId: string, + payload: unknown, + attemptCount: number, +): Promise { + let response: Response; + try { + response = await cloudApiPost({ + apiUrl: runtime.config.apiUrl, + apiKey: runtime.config.apiKey, + path: '/v1/observability/traces', + body: payload, + }); + } catch { + await handleRetry(runtime, eventId, attemptCount, 'network_error', null); + return; + } + + await handleUploadResponse(runtime, eventId, response, attemptCount); +} + +async function handleUploadResponse( + runtime: UploaderRuntime, + eventId: string, + response: Response, + attemptCount: number, +): Promise { + const status = response.status; + + if (status === 401 || status === 403) { + runtime.paused = true; + runtime.lastErrorCode = `auth_${status}`; + await scheduleCloudTraceRetry( + runtime.pool, + eventId, + runtime.lastErrorCode, + new Date(Date.now() + runtime.config.maxRetryMs), + ); + return; + } + + if (status === 200 || status === 201) { + await markCloudTraceSent(runtime.pool, eventId); + runtime.lastSuccessAt = new Date(); + clearAuthPause(runtime); + return; + } + + if (status === 408 || status === 429 || status >= 500) { + const retryAfter = parseRetryAfterMs(response.headers.get('retry-after')); + await handleRetry(runtime, eventId, attemptCount, `http_${status}`, retryAfter); + return; + } + + await markCloudTraceDeadLetter(runtime.pool, eventId, `http_${status}`); + runtime.lastErrorCode = `http_${status}`; +} + +async function handleRetry( + runtime: UploaderRuntime, + eventId: string, + attemptCount: number, + errorCode: string, + retryAfterMs: number | null, +): Promise { + runtime.lastErrorCode = errorCode; + if (attemptCount + 1 >= runtime.config.maxAttempts) { + await markCloudTraceDeadLetter(runtime.pool, eventId, errorCode); + return; + } + const backoff = Math.min( + runtime.config.maxRetryMs, + runtime.config.baseRetryMs * 2 ** attemptCount, + ); + const delayMs = retryAfterMs ?? backoff; + await scheduleCloudTraceRetry( + runtime.pool, + eventId, + errorCode, + new Date(Date.now() + delayMs), + ); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const seconds = Number.parseInt(value, 10); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1_000; + const dateMs = Date.parse(value); + if (Number.isFinite(dateMs)) return Math.max(0, dateMs - Date.now()); + return null; +} \ No newline at end of file diff --git a/packages/core/src/services/deferred-audn-scheduler.ts b/packages/core/src/services/deferred-audn-scheduler.ts new file mode 100644 index 0000000..496bbee --- /dev/null +++ b/packages/core/src/services/deferred-audn-scheduler.ts @@ -0,0 +1,66 @@ +/** + * Background scheduler that drains the deferred-AUDN queue off the ingest path. + * + * With `DEFERRED_AUDN_ENABLED=true`, ingest stores facts immediately and queues + * the per-fact AUDN decision for later, so a write returns fast. Without a + * drainer, that queue only empties when something calls `/v1/memories/reconcile`. + * This scheduler polls `reconcile()` on an interval so AUDN is applied + * automatically in the background — making ingest latency low while keeping the + * supersede/dedup/contradiction decisions that AUDN provides. + * + * It mirrors `raw-storage-reconciler-scheduler.ts`: one in-flight tick at a time, + * `stop()` clears the interval and awaits the in-flight drain. `createCoreRuntime` + * does NOT start it — the process lifecycle (server.ts) opts in via config so + * tests can drive `reconcile()` directly and stay deterministic. + */ +import type { ReconciliationResult } from './deferred-audn.js'; + +export interface DeferredAudnSchedulerOptions { + /** Drains one batch of the deferred-AUDN queue (e.g. MemoryService.reconcileDeferredAll). */ + reconcile: () => Promise; + /** Polling interval — `setInterval(tick, intervalMs)`. */ + intervalMs: number; + /** Called when a tick's reconcile rejects, so the error is surfaced, not swallowed. */ + onError: (err: unknown) => void; +} + +export interface DeferredAudnScheduler { + /** Clears the interval and awaits the in-flight `reconcile()`. */ + stop(): Promise; + /** True while a `reconcile()` invocation is in flight. */ + readonly isRunning: boolean; +} + +/** Start polling `reconcile()` every `intervalMs`; returns a stop handle. */ +export function startDeferredAudnScheduler( + options: DeferredAudnSchedulerOptions, +): DeferredAudnScheduler { + let isRunning = false; + let stopped = false; + let inFlight: Promise = Promise.resolve(); + + const tick = (): void => { + if (stopped || isRunning) return; + isRunning = true; + inFlight = options + .reconcile() + .then(() => undefined) + .catch((err) => options.onError(err)) + .finally(() => { + isRunning = false; + }); + }; + + const handle = setInterval(tick, options.intervalMs); + + return { + get isRunning() { + return isRunning; + }, + async stop() { + stopped = true; + clearInterval(handle); + await inFlight; + }, + }; +} diff --git a/packages/core/src/services/deferred-audn.ts b/packages/core/src/services/deferred-audn.ts index 76163d4..2231ee9 100644 --- a/packages/core/src/services/deferred-audn.ts +++ b/packages/core/src/services/deferred-audn.ts @@ -102,6 +102,17 @@ export async function getReconciliationStatus( return { pending, enabled: config.deferredAudnEnabled }; } +/** Tally a single resolved AUDN action into the running result. */ +function tallyReconcileAction(result: ReconciliationResult, action: string): void { + switch (action) { + case 'NOOP': result.noops++; break; + case 'UPDATE': result.updates++; break; + case 'SUPERSEDE': result.supersedes++; break; + case 'DELETE': result.deletes++; break; + case 'ADD': result.adds++; break; + } +} + async function processReconciliationBatch( pool: pg.Pool, repo: MemoryStore, @@ -113,21 +124,24 @@ async function processReconciliationBatch( supersedes: 0, deletes: 0, adds: 0, errors: 0, durationMs: 0, }; - for (const memory of deferred) { - result.processed++; - try { - const action = await reconcileSingleMemory(pool, repo, memory); - result.resolved++; - switch (action) { - case 'NOOP': result.noops++; break; - case 'UPDATE': result.updates++; break; - case 'SUPERSEDE': result.supersedes++; break; - case 'DELETE': result.deletes++; break; - case 'ADD': result.adds++; break; + // Each deferred memory re-fetches its own candidates and is independent, so the + // per-memory LLM AUDN decisions run concurrently (capped) instead of serially — + // the reconcile pass is LLM-bound, so this is a ~Nx wall-clock win. + const concurrency = Math.max(1, config.deferredAudnConcurrency); + for (let i = 0; i < deferred.length; i += concurrency) { + const chunk = deferred.slice(i, i + concurrency); + const settled = await Promise.allSettled( + chunk.map((memory) => reconcileSingleMemory(pool, repo, memory)), + ); + for (const outcome of settled) { + result.processed++; + if (outcome.status === 'fulfilled') { + result.resolved++; + tallyReconcileAction(result, outcome.value); + } else { + result.errors++; + console.error('[deferred-audn] Error reconciling memory:', outcome.reason); } - } catch (err) { - result.errors++; - console.error(`[deferred-audn] Error reconciling memory ${memory.id}:`, err); } } diff --git a/packages/core/src/services/embedding.ts b/packages/core/src/services/embedding.ts index 21d6630..106e7c5 100644 --- a/packages/core/src/services/embedding.ts +++ b/packages/core/src/services/embedding.ts @@ -257,9 +257,13 @@ type TransformersPipeline = (texts: string | string[], options: Record { const { pipeline } = await import('@huggingface/transformers'); - console.log(`[embedding] Loading local WASM model: ${model}`); + // EMBEDDING_DTYPE lets larger local models (e.g. bge-base) load as quantized + // (q8) instead of fp32 — fp32 of a big model is a ~400MB download + slow WASM + // init that can hang. Small models (MiniLM) stay fine on the fp32 default. + const dtype = (process.env.EMBEDDING_DTYPE ?? 'fp32') as 'fp32' | 'fp16' | 'q8' | 'q4'; + console.log(`[embedding] Loading local model: ${model} (dtype=${dtype})`); const start = performance.now(); - const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' }); + const extractor = await pipeline('feature-extraction', model, { dtype }); console.log(`[embedding] Model loaded in ${(performance.now() - start).toFixed(0)}ms`); return extractor as unknown as TransformersPipeline; } diff --git a/packages/core/src/services/llm.ts b/packages/core/src/services/llm.ts index a2f117c..489f685 100644 --- a/packages/core/src/services/llm.ts +++ b/packages/core/src/services/llm.ts @@ -348,12 +348,19 @@ class AnthropicLLM implements LLMProvider { async chat(messages: ChatMessage[], options: ChatOptions = {}): Promise { const split = splitAnthropicMessages(messages); + // Anthropic has no response_format; enforce JSON via an assistant prefill ('{'). + // Without this, jsonMode callers (extraction/AUDN) receive conversational prose + // and extract zero facts. The prefill character is prepended back to the result. + const prefillJson = options.jsonMode === true; + const reqMessages = prefillJson + ? [...split.messages, { role: 'assistant' as const, content: '{' }] + : split.messages; const request = () => this.client.messages.create({ model: this.model, max_tokens: options.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS, temperature: options.temperature ?? 0, ...(split.system !== undefined ? { system: split.system } : {}), - messages: split.messages, + messages: reqMessages, }); const started = performance.now(); const response = await retryOnRateLimit(request); @@ -363,7 +370,8 @@ class AnthropicLLM implements LLMProvider { null, ); recordChatCost(this.model, usage, started); - return extractAnthropicText(response); + const text = extractAnthropicText(response); + return prefillJson ? `{${text}` : text; } } diff --git a/packages/core/src/services/memory-crud.ts b/packages/core/src/services/memory-crud.ts index df5f50c..53284ef 100644 --- a/packages/core/src/services/memory-crud.ts +++ b/packages/core/src/services/memory-crud.ts @@ -21,6 +21,7 @@ import { markCleanupFailedAndSyncArtifact, markCleanupSuccessAndSyncArtifact, } from '../db/raw-doc-artifact-sync.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; export interface ClaimSlotBackfillResult { scanned: number; @@ -77,12 +78,28 @@ export async function expandMemoriesInWorkspace( } export async function deleteMemory(deps: MemoryServiceDeps, id: string, userId: string) { + const existing = await deps.stores.memory.getMemory(id, userId); + if (!existing) return; const version = await deps.stores.claim.getClaimVersionByMemoryId(userId, id); const target = version ? { claimId: version.claim_id, versionId: version.id } : null; await deps.stores.memory.softDeleteMemory(userId, id); if (config.auditLoggingEnabled) { emitAuditEvent('memory:delete', userId, {}, { memoryId: id }); } + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.delete', + durationMs: 0, + userId, + summary: { + operation_detail: 'memory.delete', + previous_memory_id: id, + }, + }, + ); if (!target) return; await deps.stores.claim.supersedeClaimVersion(userId, target.versionId, null); await deps.stores.claim.invalidateClaim(userId, target.claimId); diff --git a/packages/core/src/services/memory-ingest.ts b/packages/core/src/services/memory-ingest.ts index d3ea6f3..ffb2544 100644 --- a/packages/core/src/services/memory-ingest.ts +++ b/packages/core/src/services/memory-ingest.ts @@ -24,6 +24,7 @@ import type { MemoryServiceDeps, } from './memory-service-types.js'; import type { ReflectionJobsRepository } from '../db/reflection-jobs-repository.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; /** Enqueue a reflection job for (userId, conversationId) when the feature gate is on. */ async function maybeEnqueueReflectionJob( @@ -86,6 +87,8 @@ function buildIngestResult(episodeId: string, factsCount: number, acc: IngestAcc } function finalizeIngestResult( + deps: MemoryServiceDeps, + ingestStart: number, episodeId: string, factsCount: number, acc: IngestAccumulator, @@ -94,10 +97,30 @@ function finalizeIngestResult( traceCollector: IngestTraceCollector, traceMetadata: Parameters[0], ): IngestResult { - return { + const result = { ...buildIngestResult(episodeId, factsCount, acc, linksCreated, compositesCreated), ingestTraceId: traceCollector.finalize(traceMetadata), }; + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.ingest', + durationMs: performance.now() - ingestStart, + userId: traceMetadata.userId, + summary: { + operation_detail: `${traceMetadata.mode} ingest (${factsCount} fact(s))`, + result_count: acc.counters.stored + acc.counters.updated, + episode_id: episodeId, + memories_stored: acc.counters.stored, + memories_updated: acc.counters.updated, + memories_deleted: acc.counters.deleted, + }, + evidence: { mode: traceMetadata.mode, source_site: traceMetadata.sourceSite }, + }, + ); + return result; } /** Full consensus-based ingest pipeline. */ @@ -186,6 +209,8 @@ export async function performIngest( await maybeEnqueueReflectionJob(deps.reflectionJobs, deps.reflectEnabled, userId, episodeId); console.log(`[timing] ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${facts.length} facts, ${postWrite.compositesCreated} composites, topic=${postWrite.topicAbstractionApplied})`); return finalizeIngestResult( + deps, + ingestStart, episodeId, facts.length, acc, @@ -236,6 +261,8 @@ export async function performQuickIngest( console.log(`[timing] quick-ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${extractedFacts.length} facts, ${acc.counters.stored} stored, ${acc.counters.skipped} skipped)`); return finalizeIngestResult( + deps, + ingestStart, episodeId, extractedFacts.length, acc, @@ -286,6 +313,7 @@ export async function performStoreVerbatim( metadata?: MemoryMetadata, sessionId?: string, ): Promise { + const ingestStart = performance.now(); const episodeId = await deps.stores.episode.storeEpisode({ userId, content, sourceSite, sourceUrl, sessionId }); const embedding = await embedText(content); const writeSecurity = assessWriteSecurity(content, sourceSite, deps.config); @@ -329,7 +357,7 @@ export async function performStoreVerbatim( memoryId, }); - return { + const result: IngestResult = { episodeId, factsExtracted: 1, memoriesStored: isUpdate ? 0 : 1, @@ -343,6 +371,23 @@ export async function performStoreVerbatim( compositesCreated: 0, ingestTraceId: traceCollector.finalize({ mode: 'verbatim', userId, sourceSite, sourceUrl, episodeId, factsExtracted: 1 }), }; + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: isUpdate ? 'memory.update' : 'memory.ingest', + durationMs: performance.now() - ingestStart, + userId, + summary: { + content_len: content.length, + new_memory_id: memoryId, + previous_memory_id: isUpdate ? memoryId : undefined, + }, + evidence: { mode: 'verbatim', source_site: sourceSite }, + }, + ); + return result; } /** @@ -462,6 +507,8 @@ export async function performWorkspaceIngest( console.log(`[timing] ws-ingest.total: ${(performance.now() - ingestStart).toFixed(1)}ms (${facts.length} facts, workspace=${workspace.workspaceId})`); return finalizeIngestResult( + deps, + ingestStart, episodeId, facts.length, acc, diff --git a/packages/core/src/services/retrieval-side-effects.ts b/packages/core/src/services/retrieval-side-effects.ts index 34671f5..f7470ac 100644 --- a/packages/core/src/services/retrieval-side-effects.ts +++ b/packages/core/src/services/retrieval-side-effects.ts @@ -12,6 +12,7 @@ import type { SearchResult } from '../db/repository-types.js'; import type { MemoryServiceDeps } from './memory-service-types.js'; import { emitAuditEvent } from './audit-events.js'; +import { recordCloudTraceOperation } from './cloud-trace-sync.js'; /** Run post-search side effects. Swallows per-memory touch failures. */ export function recordSearchSideEffects( @@ -34,4 +35,19 @@ export function recordSearchSideEffects( topScore: outputMemories[0]?.score ?? 0, }, { sourceSite }); } + recordCloudTraceOperation( + deps.stores.pool, + deps.config.cloudTraceSync, + deps.config.cloudTraceSync?.instanceId, + { + operation: 'memory.search', + durationMs: 0, + userId, + summary: { + query_len: query.length, + result_count: outputMemories.length, + }, + evidence: { source_site: sourceSite }, + }, + ); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a710ef5..c632df6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: turbo: specifier: 2.9.14 version: 2.9.14 + yaml: + specifier: ^2.8.3 + version: 2.9.0 adapters/langchain: dependencies: @@ -189,6 +192,9 @@ importers: express: specifier: ^5.1.0 version: 5.2.1 + jose: + specifier: ^6.2.3 + version: 6.2.3 multiformats: specifier: ^13.4.1 version: 13.4.2 diff --git a/scripts/ci/__tests__/release-policy.test.mjs b/scripts/ci/__tests__/release-policy.test.mjs index 8ec1944..46475f1 100644 --- a/scripts/ci/__tests__/release-policy.test.mjs +++ b/scripts/ci/__tests__/release-policy.test.mjs @@ -10,6 +10,8 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { checkCodeownersText, + checkCompositeActionText, + checkImagePublisherText, checkPackageManifest, checkWorkflowText, } from "../release-policy.mjs"; @@ -73,6 +75,234 @@ test("CODEOWNERS with only an unrelated rule does not cover the workflow", () => assert.ok(failures.some((failure) => /no rule covers/.test(failure))); }); +const VALID_INTERNAL_IMAGE_YAML = [ + "env:", + " IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core-internal", + "jobs:", + " publish:", + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " steps:", + ' - run: docker buildx build --tag "${IMAGE_NAME}:sha-abc1234" --push .', + "", +].join("\n"); + +test("workflow without image pushes is not held to image-publisher policy", () => { + assert.deepEqual(checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker build .\n", ".github/workflows/ci.yml"), []); +}); + +test("non-enumerated workflow pushing images is rejected", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --push .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane nor the enumerated internal image publisher/.test(failure))); +}); + +test("internal image publisher without the repository guard is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace(/^.*github\.repository.*\n/m, ""); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /repository guard|github\.repository/.test(failure))); +}); + +test("internal image publisher pinned to the wrong image name is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace("atomicmemory-core-internal", "atomicmemory-core"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /assign env IMAGE_NAME exactly once/.test(failure))); +}); + +test("internal image publisher tagging a literal registry path is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', '--tag "ghcr.io/atomicstrata/atomicmemory-core:latest"'); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("internal image publisher pushing through another variable is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', '--tag "${ALT_IMAGE}:latest"'); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("internal image publisher overriding IMAGE_NAME via quoted key or flow mapping is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + ' steps:\n - env: { "IMAGE_NAME": "ghcr.io/atomicstrata/atomicmemory-core" }', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("workflow splitting --output type=registry across continuation lines is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - run: |\n docker buildx build --output \\\n type=registry .\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher using docker push or imagetools create is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + ' - run: docker push "${IMAGE_NAME}:sha-abc1234"\n'; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /forbidden here; they publish without a --tag/.test(failure))); +}); + +test("valid internal image publisher fixture passes", () => { + assert.deepEqual(checkImagePublisherText(VALID_INTERNAL_IMAGE_YAML, ".github/workflows/internal-core-docker-image.yml"), []); +}); + +test("workflow pushing via --output=type=registry is treated as a publisher", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --output=type=registry,name=ghcr.io/x/y .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("workflow pushing via --output type=image,push=true is treated as a publisher", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --output type=image,name=ghcr.io/x/y,push=true .\n", ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher with a step-level IMAGE_NAME override is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + " steps:\n - env:\n IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core", + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("internal image publisher reassigning IMAGE_NAME in shell is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " steps:", + ' steps:\n - run: echo "IMAGE_NAME=ghcr.io/atomicstrata/atomicmemory-core" >> "$GITHUB_ENV"', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not be reassigned/.test(failure))); +}); + +test("internal image publisher using raw output exporters is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace("--push .", "--output=type=registry ."); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /raw buildx output exporters/.test(failure))); +}); + +test("internal image publisher overriding IMAGE_NAME via YAML anchor/alias is rejected", () => { + const yaml = [ + "env:", + " IMAGE_NAME: &pin ghcr.io/atomicstrata/atomicmemory-core-internal", + "jobs:", + " publish:", + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " steps:", + " - env:", + " IMAGE_NAME: *pin", + ' - run: docker buildx build --tag "${IMAGE_NAME}:sha-abc1234" --push .', + "", + ].join("\n"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /exactly once/.test(failure))); +}); + +test("workflow pushing via docker/build-push-action is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - uses: docker/build-push-action@v6\n with:\n push: true\n tags: ghcr.io/x/y:latest\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher using docker/build-push-action is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " - uses: docker/build-push-action@v6\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /docker\/build-push-action is forbidden/.test(failure))); +}); + +test("unparseable workflow YAML fails the image-publisher policy closed", () => { + const failures = checkImagePublisherText("jobs: [unclosed\n {bad", ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /failed to parse/.test(failure))); +}); + +test("internal image publisher using compact -tVALUE tagging is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace('--tag "${IMAGE_NAME}:sha-abc1234"', "-tghcr.io/atomicstrata/atomicmemory-core:latest"); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /derive from \$\{IMAGE_NAME\} directly/.test(failure))); +}); + +test("workflow pushing via compact -otype=registry is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - run: docker buildx build -otype=registry,name=ghcr.io/x/y .\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("workflow using a case-variant build-push-action ref is treated as a publisher", () => { + const yaml = "jobs:\n x:\n steps:\n - uses: Docker/build-push-action@v6\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("internal image publisher with a compound OR repository guard is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + " if: github.repository == 'atomicstrata/atomicmemory-internal'", + " if: github.repository == 'atomicstrata/atomicmemory-internal' || github.repository == 'atomicstrata/atomicmemory'", + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /guarded with exactly/.test(failure))); +}); + +test("internal image publisher interpolating GitHub expressions into run is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML.replace( + '--tag "${IMAGE_NAME}:sha-abc1234"', + '--tag "${IMAGE_NAME}:sha-abc1234" --platform "${{ inputs.platforms }}"', + ); + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not interpolate/.test(failure))); +}); + +test("internal image publisher using a non-allowlisted action is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " - uses: actions/setup-node@v4\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /action allowlist/.test(failure))); +}); + +test("workflow pushing via docker image push or skopeo copy is treated as a publisher", () => { + for (const cmd of ["docker image push ghcr.io/x/y", "skopeo copy oci:img docker://ghcr.io/x/y", "podman push ghcr.io/x/y", "docker compose push"]) { + const failures = checkImagePublisherText(`jobs:\n x:\n steps:\n - run: ${cmd}\n`, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure)), `expected publisher classification for: ${cmd}`); + } +}); + +test("a publish-*.yaml (yaml extension) pushing images is still held to image-publisher policy", () => { + const failures = checkImagePublisherText("jobs:\n x:\n steps:\n - run: docker buildx build --push .\n", ".github/workflows/publish-evil.yaml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("steps inherited through a YAML merge key are still scanned", () => { + const yaml = [ + "x: &tpl", + " steps:", + " - run: docker buildx build --push .", + "jobs:", + " evil:", + " <<: *tpl", + "", + ].join("\n"); + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /neither a publish-\*\.yml release lane/.test(failure))); +}); + +test("non-publish workflow calling a publishing reusable workflow is rejected", () => { + const yaml = "jobs:\n x:\n uses: ./.github/workflows/publish-core-docker.yml\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/nightly.yml"); + assert.ok(failures.some((failure) => /calls publishing reusable workflow/.test(failure))); +}); + +test("internal image publisher delegating to any reusable workflow is rejected", () => { + const yaml = VALID_INTERNAL_IMAGE_YAML + " delegate:\n if: github.repository == 'atomicstrata/atomicmemory-internal'\n uses: ./.github/workflows/other.yml\n"; + const failures = checkImagePublisherText(yaml, ".github/workflows/internal-core-docker-image.yml"); + assert.ok(failures.some((failure) => /must not call a reusable workflow/.test(failure))); +}); + +test("composite action containing a push sink is rejected", () => { + const yaml = "name: helper\nruns:\n using: composite\n steps:\n - run: crane cp img ghcr.io/x/y\n shell: bash\n"; + const failures = checkCompositeActionText(yaml, ".github/actions/helper/action.yml"); + assert.ok(failures.some((failure) => /composite actions must not push/.test(failure))); +}); + +test("composite action without push sinks passes", () => { + const yaml = "name: helper\nruns:\n using: composite\n steps:\n - run: docker build -t local/img .\n shell: bash\n"; + assert.deepEqual(checkCompositeActionText(yaml, ".github/actions/helper/action.yml"), []); +}); + function validManifest(overrides = {}) { return { name: "@atomicmemory/sdk", diff --git a/scripts/ci/release-policy.mjs b/scripts/ci/release-policy.mjs index 543521b..bd0fc43 100644 --- a/scripts/ci/release-policy.mjs +++ b/scripts/ci/release-policy.mjs @@ -12,11 +12,21 @@ * - Every npm-published package sets publishConfig.access=public and * declares no file: / link: dependencies. * - CODEOWNERS covers the publish workflow file. + * - Container-image publishers are explicitly enumerated: any workflow that + * pushes images (docker push / buildx --push / output exporters / + * imagetools create / docker/build-push-action) must be either a + * publish-*.yml release lane (covered by the invariants above) or the + * enumerated internal operator publisher. That publisher's workflow YAML + * is parsed structurally and must guard every job on the + * atomicmemory-internal repository, assign IMAGE_NAME exactly once at the + * workflow level (pinned to the private internal package), and push only + * via buildx --push with every --tag deriving from that pin. */ import { readFileSync, readdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; const here = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(here, "../.."); @@ -24,6 +34,29 @@ const PUBLISH_WORKFLOW = ".github/workflows/publish-packages.yml"; const CODEOWNERS_FILE = ".github/CODEOWNERS"; const GUARD_REL_PATH = "scripts/guards/guard-npm-publish.mjs"; const PUBLISH_WORKFLOW_FILENAME_PREFIX = "publish-"; +const INTERNAL_IMAGE_WORKFLOW_FILENAME = "internal-core-docker-image.yml"; +const INTERNAL_IMAGE_NAME = "ghcr.io/atomicstrata/atomicmemory-core-internal"; +// The job-level `if` must equal this exactly (whitespace-normalized): a +// compound condition (e.g. `A || B`) could satisfy a substring match while +// still running in the mirrored public repository. +const INTERNAL_REPO_GUARD_EXACT = "github.repository == 'atomicstrata/atomicmemory-internal'"; +// --output=type=registry and --output type=image,push=true are buildx's +// long-form equivalents of --push; treat them as the same sink. +const OUTPUT_EXPORTER_RE = /(^|\s)(-o|--output)[=\s][^\n]*\b(type=registry|push=true)\b/; +// Non-buildx registry-push tools available on hosted runners. +const OTHER_PUSH_SINKS_RE = /\b(docker\s+image\s+push|docker\s+compose\s+push|docker-compose\s+push|podman\s+(image\s+)?push|buildah\s+push|skopeo\s+(copy|sync)|crane\s+(push|cp|copy)|oras\s+push)\b/; +// Reusable-workflow refs that publish; a job-level `uses:` of one of these +// from outside the audited release lane would launder a publish. +const PUBLISHING_WORKFLOW_REF_RE = /(^|\/)(publish-[^/@\s]*\.ya?ml|internal-core-docker-image\.yml)(@|$)/i; +const EXPRESSION_MARKER = "$" + "{{"; +const COMPOSITE_SCAN_SKIP_DIRS = new Set(["node_modules", ".git", "dist", "build", ".turbo", ".worktrees"]); +// Actions the enumerated internal publisher may use; anything else (any +// case) is a policy failure so a new action is an explicit policy change. +const INTERNAL_ALLOWED_ACTIONS = [ + "actions/checkout@", + "docker/setup-qemu-action@", + "docker/setup-buildx-action@", +]; const PUBLISHED_PACKAGE_PATHS = [ "packages/core/package.json", "packages/sdk/package.json", @@ -55,11 +88,240 @@ export function runPolicy(options) { const root = options.repoRoot; const failures = []; failures.push(...checkPublishWorkflows(root)); + failures.push(...checkImagePublishers(root)); failures.push(...checkPublishedPackages(root)); failures.push(...checkCodeownersCovers(root)); return failures; } +function checkImagePublishers(root) { + const failures = []; + for (const workflowPath of listWorkflowPaths(root)) { + const basename = workflowPath.split("/").pop(); + // Skip only what checkPublishWorkflows actually covers (publish-*.yml); + // a publish-*.yaml would otherwise escape both checks. + if (basename.startsWith(PUBLISH_WORKFLOW_FILENAME_PREFIX) && basename.endsWith(".yml")) continue; + const text = safeRead(root, workflowPath); + if (text === null) continue; + failures.push(...checkImagePublisherText(text, workflowPath)); + } + for (const actionPath of listCompositeActionPaths(root)) { + const text = safeRead(root, actionPath); + if (text === null) continue; + failures.push(...checkCompositeActionText(text, actionPath)); + } + return failures; +} + +/** + * Composite actions (action.yml) can run shell of their own; none may + * contain a push sink -- a workflow step's `uses: ./path` would otherwise + * publish without anything visible in the workflow file. + */ +export function checkCompositeActionText(rawText, filename) { + let doc; + try { + doc = parseYaml(rawText, { merge: true }) ?? {}; + } catch (error) { + return [`${filename}: action YAML failed to parse, so image-publisher policy cannot validate it (${error.message}).`]; + } + const steps = Array.isArray(doc?.runs?.steps) ? doc.runs.steps : []; + const shellText = steps + .map((step) => (typeof step?.run === "string" ? step.run.replace(/\\\r?\n\s*/g, " ") : "")) + .join("\n") + .replace(/(^|[\s"'])-([to])(?=[^\s=])/gm, "$1-$2 "); + const usesPushAction = steps.some( + (step) => typeof step?.uses === "string" && step.uses.toLowerCase().startsWith("docker/build-push-action"), + ); + if (pushesContainerImages(shellText) || usesPushAction) { + return [`${filename}: composite actions must not push container images; only publish-*.yml release lanes and ${INTERNAL_IMAGE_WORKFLOW_FILENAME} may publish.`]; + } + return []; +} + +function listCompositeActionPaths(root) { + const results = []; + const walk = (relDir) => { + let entries; + try { + entries = readdirSync(resolve(root, relDir), { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const relPath = relDir ? `${relDir}/${entry.name}` : entry.name; + if (entry.isDirectory()) { + const enter = entry.name === ".github" || (!COMPOSITE_SCAN_SKIP_DIRS.has(entry.name) && !entry.name.startsWith(".")); + if (enter) walk(relPath); + } else if (entry.name === "action.yml" || entry.name === "action.yaml") { + results.push(relPath); + } + } + }; + walk(""); + return results; +} + +/** + * Structured image-publisher validation. The workflow YAML is parsed (so + * quoted keys, flow mappings, anchors, and aliases cannot hide an + * assignment), run scripts are extracted per step with backslash-newline + * continuations joined (so no sink can be split across lines), and the + * destination contract is a positive assertion: one workflow-level + * IMAGE_NAME pin, every --tag derived from it, buildx --push as the only + * push path. A workflow that fails to parse fails the policy. + */ +export function checkImagePublisherText(rawText, filename) { + let doc; + try { + // merge: true resolves YAML merge keys (<<: *anchor) the way a runner + // would, so inherited steps cannot hide from the scan below. + doc = parseYaml(rawText, { merge: true }) ?? {}; + } catch (error) { + return [`${filename}: workflow YAML failed to parse, so image-publisher policy cannot validate it (${error.message}).`]; + } + if (typeof doc !== "object") { + return [`${filename}: workflow YAML is not a mapping, so image-publisher policy cannot validate it.`]; + } + + const jobs = Object.entries(doc.jobs ?? {}); + const runScripts = []; + const usedActions = []; + const jobLevelUses = []; + for (const [jobName, job] of jobs) { + if (typeof job?.uses === "string") jobLevelUses.push({ jobName, ref: job.uses }); + for (const step of Array.isArray(job?.steps) ? job.steps : []) { + if (typeof step?.run === "string") runScripts.push(step.run.replace(/\\\r?\n\s*/g, " ")); + if (typeof step?.uses === "string") usedActions.push(step.uses); + } + } + + const failuresEarly = []; + // A job-level `uses:` of a publishing reusable workflow is itself a + // publish path: only publish-*.yml release lanes may call one, and the + // enumerated internal publisher may not delegate to reusable workflows + // at all. + for (const { jobName, ref } of jobLevelUses) { + if (filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { + failuresEarly.push(`${filename}: job '${jobName}' must not call a reusable workflow (uses: ${ref}); the internal image publisher defines its own steps only.`); + } else if (PUBLISHING_WORKFLOW_REF_RE.test(ref)) { + failuresEarly.push(`${filename}: job '${jobName}' calls publishing reusable workflow ${ref}; only publish-*.yml release lanes may do that.`); + } + } + if (failuresEarly.length > 0 && !filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) return failuresEarly; + // Docker accepts compact short-option forms (-tVALUE, -oVALUE); split them + // so the sink and destination scans below see the canonical spaced form. + const shellText = runScripts.join("\n").replace(/(^|[\s"'])-([to])(?=[^\s=])/gm, "$1-$2 "); + // GitHub resolves action owner/repo case-insensitively. + const usesPushAction = usedActions.some((action) => action.toLowerCase().startsWith("docker/build-push-action")); + + if (!pushesContainerImages(shellText) && !usesPushAction) return failuresEarly; + if (!filename.endsWith(INTERNAL_IMAGE_WORKFLOW_FILENAME)) { + return [ + `${filename}: pushes container images but is neither a ${PUBLISH_WORKFLOW_FILENAME_PREFIX}*.yml release lane nor the enumerated internal image publisher (${INTERNAL_IMAGE_WORKFLOW_FILENAME}).`, + ]; + } + + const failures = failuresEarly; + // (1) Every job must carry the repository guard as its EXACT `if` + // condition; substring matching would accept compound conditions that + // also run in the mirrored public repository. + for (const [jobName, job] of jobs) { + const condition = typeof job?.if === "string" ? job.if.replace(/\s+/g, " ").trim() : ""; + if (condition !== INTERNAL_REPO_GUARD_EXACT) { + failures.push(`${filename}: job '${jobName}' must be guarded with exactly if: ${INTERNAL_REPO_GUARD_EXACT} (found: ${condition || "none"}).`); + } + } + // (2) Exactly one IMAGE_NAME env assignment may exist, at the workflow + // level, pinned to the internal package. The parser resolves quoted + // keys, flow mappings, and anchor/alias tricks before we count. + const assignments = collectImageNameAssignments(doc, jobs); + const pin = assignments.length === 1 ? assignments[0] : undefined; + if (!pin || pin.where !== "workflow env" || pin.value !== INTERNAL_IMAGE_NAME) { + const found = assignments.map((a) => `${a.where}=${a.value}`).join(", ") || "none"; + failures.push(`${filename}: must assign env IMAGE_NAME exactly once, at the workflow level, pinned to ${INTERNAL_IMAGE_NAME} (found: ${found}).`); + } + // (3) No shell-side reassignment (IMAGE_NAME=... in run blocks or + // GITHUB_ENV writes). + if (/\bIMAGE_NAME\s*=/.test(shellText)) { + failures.push(`${filename}: IMAGE_NAME must not be reassigned in shell or GITHUB_ENV; the single env pin is the only allowed assignment.`); + } + // (4) Every --tag / -t argument must reference ${IMAGE_NAME} directly, so + // pushes cannot be routed through another variable or a literal path. + for (const match of shellText.matchAll(/(?:^|[\s"'])(?:--tag|-t)["'=\s]+([^\s"']+)/g)) { + if (!/^\$\{IMAGE_NAME\}[:@]/.test(match[1])) { + failures.push(`${filename}: every --tag must derive from \${IMAGE_NAME} directly (found '--tag ${match[1]}').`); + } + } + // (5) The only allowed push path is a buildx --push run step: raw output + // exporters, docker push, imagetools create, and docker/build-push-action + // would all publish without a --tag this policy can pin. + if (OUTPUT_EXPORTER_RE.test(shellText)) { + failures.push(`${filename}: raw buildx output exporters (--output/-o type=registry or push=true) are forbidden; push only via --push with --tag \${IMAGE_NAME}.`); + } + if (/\bdocker\s+push\b/.test(shellText) || /\bimagetools\s+create\b/.test(shellText) || OTHER_PUSH_SINKS_RE.test(shellText)) { + failures.push(`${filename}: docker push, imagetools create, and non-buildx push tools (podman/buildah/skopeo/crane/oras/compose push) are forbidden here; they publish without a --tag the policy can pin. Push only via buildx --push.`); + } + if (usesPushAction) { + failures.push(`${filename}: docker/build-push-action is forbidden here; push only via a buildx --push run step so the --tag contract applies.`); + } + // (6) Run scripts must not interpolate GitHub expressions: text from + // dispatch inputs or the checked-out branch (e.g. a package.json + // version) would be substituted into shell before it runs. Dynamic + // values must reach shell as step env instead. + if (shellText.includes(EXPRESSION_MARKER)) { + failures.push(`${filename}: run scripts must not interpolate ${EXPRESSION_MARKER} ... }} GitHub expressions; pass dynamic values via step env so inputs and branch content cannot inject into shell.`); + } + // (7) Only enumerated actions may be used, so no third-party or local + // composite action can push on this workflow's behalf. + for (const action of usedActions) { + if (!INTERNAL_ALLOWED_ACTIONS.some((allowed) => action.toLowerCase().startsWith(allowed))) { + failures.push(`${filename}: uses: ${action} is not in the internal image publisher's action allowlist (${INTERNAL_ALLOWED_ACTIONS.join(", ")}).`); + } + } + return failures; +} + +function collectImageNameAssignments(doc, jobs) { + const assignments = []; + const collect = (env, where) => { + if (!env || typeof env !== "object") return; + for (const [key, value] of Object.entries(env)) { + if (key === "IMAGE_NAME") assignments.push({ where, value: String(value) }); + } + }; + collect(doc.env, "workflow env"); + for (const [jobName, job] of jobs) { + collect(job?.env, `job '${jobName}' env`); + collect(job?.container?.env, `job '${jobName}' container env`); + (Array.isArray(job?.steps) ? job.steps : []).forEach((step, index) => { + collect(step?.env, `job '${jobName}' step ${index + 1} env`); + }); + } + return assignments; +} + +function pushesContainerImages(text) { + return ( + /(^|\s)--push\b/.test(text) || + /\bdocker\s+push\b/.test(text) || + /\bimagetools\s+create\b/.test(text) || + OUTPUT_EXPORTER_RE.test(text) || + OTHER_PUSH_SINKS_RE.test(text) + ); +} + +function listWorkflowPaths(root) { + const dir = resolve(root, ".github/workflows"); + try { + return readdirSync(dir) + .filter((name) => name.endsWith(".yml") || name.endsWith(".yaml")) + .map((name) => `.github/workflows/${name}`); + } catch { + return []; + } +} + function checkPublishWorkflows(root) { const failures = []; for (const workflowPath of listPublishWorkflowPaths(root)) { @@ -148,7 +410,10 @@ function findUnpublishableDependencies(packageJson, filename) { function checkCodeownersCovers(root) { const text = safeRead(root, CODEOWNERS_FILE); if (text === null) return [`${CODEOWNERS_FILE}: missing; required to own ${PUBLISH_WORKFLOW}.`]; - return checkCodeownersText(text, PUBLISH_WORKFLOW); + return [ + ...checkCodeownersText(text, PUBLISH_WORKFLOW), + ...checkCodeownersText(text, `.github/workflows/${INTERNAL_IMAGE_WORKFLOW_FILENAME}`), + ]; } export function checkCodeownersText(text, requiredPath) {