Add connected-local Cloud support and Core runtime improvements

.github/workflows/ci.yml

@@ -65,6 +65,15 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
+      # release-policy.mjs parses workflow YAML with the root `yaml`
+      # devDependency, so this job needs an install (it previously ran on a
+      # bare checkout).
+      - name: Setup pnpm
+        run: |
+          corepack enable
+          corepack prepare pnpm@${PNPM_VERSION} --activate
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile --ignore-scripts
       - name: Run release-policy guardrails
         run: node scripts/ci/release-policy.mjs
       - name: Run release-policy unit tests

.github/workflows/internal-core-docker-image.yml

@@ -0,0 +1,275 @@
+name: Internal Core Docker Image
+
+# Internal-only convenience lane. Builds packages/core/Dockerfile from any ref
+# of atomicmemory-internal and pushes it to the PRIVATE package
+# ghcr.io/atomicstrata/atomicmemory-core-internal for internal testing.
+#
+# This is NOT the release pipeline. The official image
+# ghcr.io/atomicstrata/atomicmemory-core is published exclusively by
+# publish-core-docker.yml running on the public repo, anchored to the
+# published @atomicmemory/core npm package. Nothing here may ever push to the
+# official package name, and the tag guard below refuses ref-derived tags that
+# are release-shaped (semver with or without prerelease/build suffix) or that
+# collide with a reserved alias, so an internal build can never masquerade as
+# a release. Builds of main additionally publish the curated floating aliases
+# `internal-latest` and `latest`; both live only on the private internal
+# package and always point at the same digest.
+#
+# Naming contract with release-policy: scripts/ci/release-policy.mjs forbids
+# workflow_dispatch on every `.github/workflows/publish-*.yml` because those
+# are release lanes that must only be triggerable by the audited
+# repository_dispatch release flow. This workflow is deliberately named
+# OUTSIDE that prefix: it is operator-dispatchable by design and never touches
+# release artifacts or release tag namespaces. If this file is ever renamed to
+# publish-*.yml, release-policy will (correctly) fail it.
+#
+# Repo guard: like sync-to-private.yml, this file is mirrored into the public
+# repo by the public export, so the job runs ONLY on
+# atomicstrata/atomicmemory-internal. Keep this file free of anything
+# confidential; it is public.
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Branch, tag, or SHA of atomicmemory-internal to build"
+        required: false
+        default: main
+        type: string
+      platforms:
+        description: "Comma-separated buildx platforms"
+        required: false
+        default: linux/amd64,linux/arm64
+        type: string
+  push:
+    branches:
+      - main
+    # Mirrors the core-image-affecting classification used by ci.yml's
+    # core-docker-smoke job (packages/core + root build-context inputs),
+    # plus this workflow itself.
+    paths:
+      - "packages/core/**"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - "package.json"
+      - "turbo.json"
+      - ".github/workflows/internal-core-docker-image.yml"
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: internal-core-docker-image-${{ inputs.ref || github.ref_name }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  IMAGE_NAME: ghcr.io/atomicstrata/atomicmemory-core-internal
+  DEFAULT_PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+  publish:
+    name: build and push internal core image
+    if: github.repository == 'atomicstrata/atomicmemory-internal'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout requested ref
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      # The core image is several GB per platform and the verify step loads
+      # each platform's image into dockerd on top of the buildx cache; the
+      # standard runner's free disk is not enough (the first run died on
+      # ENOSPC six minutes in). Reclaim the runner's preinstalled bloat
+      # before building.
+      - name: Free runner disk space
+        run: |
+          set -euo pipefail
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL /usr/local/share/boost
+          sudo docker image prune --all --force >/dev/null
+          df -h /
+
+      - name: Compute image tags
+        id: tags
+        env:
+          RAW_REF: ${{ inputs.ref || github.ref_name }}
+        run: |
+          set -euo pipefail
+
+          sha="$(git rev-parse HEAD)"
+          short_sha="${sha:0:7}"
+          core_version="$(node -p "require('./packages/core/package.json').version")"
+
+          # Slugify the ref into a valid docker tag: lowercase, restricted
+          # charset, no leading separator, bounded length.
+          slug="$(tr '[:upper:]' '[:lower:]' <<<"${RAW_REF}" | sed -e 's![^a-z0-9._-]!-!g' -e 's!^[.-]*!!' | cut -c1-100)"
+
+          tags=("sha-${short_sha}")
+
+          # Both guards below apply to the REF-DERIVED slug only. The curated
+          # aliases added afterwards are hardcoded and intentional; the slug is
+          # whatever a branch happens to be named, so it is the only untrusted
+          # tag source and must fail closed.
+          #
+          # Skip the slug tag when the ref was itself a commit sha; sha-<short>
+          # already covers it.
+          if [[ -n "${slug}" && ! "${slug}" =~ ^[0-9a-f]{7,40}$ ]]; then
+            # Reserved namespaces: a branch named latest or internal-latest
+            # could hijack a floating main alias, and sha-*/verify-* could
+            # overwrite another commit's tag or a verify scratch tag.
+            if [[ "${slug}" == "latest" || "${slug}" == "internal-latest" || "${slug}" =~ ^(sha|verify)- ]]; then
+              echo "::error::Ref slug '${slug}' collides with a reserved tag namespace (latest, internal-latest, sha-*, verify-*). Rename the branch or dispatch with the commit sha instead."
+              exit 1
+            fi
+            # Release shapes: bare majors (1 / v2, the official floating-alias
+            # shape), bare semver (1.2 / 1.2.3 / v1.2.3), and semver with any
+            # prerelease or build suffix (1.2.3-rc.1, v1.2.3-beta.2, 1.2.3+abc).
+            if [[ "${slug}" =~ ^v?[0-9]+$ || "${slug}" =~ ^v?[0-9]+(\.[0-9]+){1,2}([-+.].*)?$ ]]; then
+              echo "::error::Ref slug '${slug}' looks like a release tag; internal builds must not publish semver-shaped tags. Rename the ref or dispatch with a different ref."
+              exit 1
+            fi
+            tags+=("${slug}")
+          fi
+
+          # Curated floating aliases for main. `latest` exists so a bare
+          # `docker pull ${IMAGE_NAME}` (which resolves to :latest) works; it
+          # is published only on this PRIVATE internal package and always
+          # points at the same digest as internal-latest. The official
+          # ghcr.io/atomicstrata/atomicmemory-core:latest is published solely
+          # by publish-core-docker.yml on the public repo and is untouched.
+          if [[ "${slug}" == "main" ]]; then
+            tags+=("internal-latest" "latest")
+          fi
+
+          tag_args=()
+          refs=()
+          for tag in "${tags[@]}"; do
+            tag_args+=("--tag" "${IMAGE_NAME}:${tag}")
+            refs+=("${IMAGE_NAME}:${tag}")
+          done
+
+          {
+            echo "sha=${sha}"
+            echo "short_sha=${short_sha}"
+            echo "core_version=${core_version}"
+            echo "tag_args=${tag_args[*]}"
+            echo "refs=${refs[*]}"
+          } >>"${GITHUB_OUTPUT}"
+
+          echo "Building @atomicmemory/core ${core_version} at ${sha}"
+          printf 'Will push:\n'
+          printf '  - %s\n' "${refs[@]}"
+
+      - name: Login to GHCR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130
+        with:
+          platforms: arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+
+      # Dynamic values reach the run scripts below ONLY via step env, never
+      # by interpolating ${{ ... }} into shell text: a dispatch input or a
+      # hostile branch's package.json version could otherwise inject shell.
+      # release-policy enforces this (no ${{ in run scripts of this file).
+      - name: Build and verify every requested platform
+        # Every platform that will be pushed is first built locally and
+        # verified (deployed package version readable via node). No platform
+        # selection can bypass verification; non-native platforms run under
+        # QEMU, and their build layers are reused by the push step via the
+        # shared buildx cache.
+        env:
+          PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }}
+          SHORT_SHA: ${{ steps.tags.outputs.short_sha }}
+          CORE_VERSION: ${{ steps.tags.outputs.core_version }}
+        run: |
+          set -euo pipefail
+
+          IFS=',' read -r -a platforms <<<"${PLATFORMS}"
+          for platform in "${platforms[@]}"; do
+            platform="$(tr -d '[:space:]' <<<"${platform}")"
+            verify_suffix="verify-${SHORT_SHA}-${platform//\//-}"
+
+            docker buildx build \
+              --platform "${platform}" \
+              --file packages/core/Dockerfile \
+              --load \
+              --tag "${IMAGE_NAME}:${verify_suffix}" \
+              .
+
+            image_version="$(docker run --rm --platform "${platform}" --entrypoint node "${IMAGE_NAME}:${verify_suffix}" -p "require('./package.json').version")"
+            if [[ "${image_version}" != "${CORE_VERSION}" ]]; then
+              echo "::error::${platform} image package version is ${image_version}, expected ${CORE_VERSION}"
+              exit 1
+            fi
+            echo "${platform}: verified @atomicmemory/core ${image_version}"
+
+            # Free dockerd storage before the next platform loads; the push
+            # step rebuilds from the (kept) buildx cache, not from dockerd.
+            docker image rm --force "${IMAGE_NAME}:${verify_suffix}" >/dev/null
+          done
+
+      - name: Build and push internal image
+        env:
+          PLATFORMS: ${{ inputs.platforms || env.DEFAULT_PLATFORMS }}
+          TAG_ARGS: ${{ steps.tags.outputs.tag_args }}
+          GIT_SHA: ${{ steps.tags.outputs.sha }}
+          CORE_VERSION: ${{ steps.tags.outputs.core_version }}
+        run: |
+          set -euo pipefail
+
+          read -r -a tag_args <<<"${TAG_ARGS}"
+
+          docker buildx build \
+            --platform "${PLATFORMS}" \
+            --file packages/core/Dockerfile \
+            --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
+            --label "org.opencontainers.image.revision=${GIT_SHA}" \
+            --label "org.opencontainers.image.version=${CORE_VERSION}" \
+            --label "org.opencontainers.image.title=@atomicmemory/core (internal test build)" \
+            "${tag_args[@]}" \
+            --push \
+            .
+
+      - name: Report published image
+        env:
+          GIT_SHA: ${{ steps.tags.outputs.sha }}
+          SHORT_SHA: ${{ steps.tags.outputs.short_sha }}
+          CORE_VERSION: ${{ steps.tags.outputs.core_version }}
+          REFS: ${{ steps.tags.outputs.refs }}
+        run: |
+          set -euo pipefail
+
+          digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:sha-${SHORT_SHA}" --format '{{ json .Manifest }}' | jq -r '.digest')"
+
+          {
+            echo "## Internal core image published"
+            echo ""
+            echo "- revision: \`${GIT_SHA}\`"
+            echo "- package version: \`${CORE_VERSION}\`"
+            echo "- digest-pinned ref (immutable, prefer this for sharing): \`${IMAGE_NAME}@${digest}\`"
+            echo "- tags (mutable, a rebuild of the same ref moves them):"
+            for image_ref in ${REFS}; do
+              echo "  - \`${image_ref}\`"
+            done
+            echo ""
+            echo "Ops integration-smoke usage (from the smoke harness directory, after docker login ghcr.io):"
+            echo ""
+            echo '```'
+            echo "CORE_SOURCE_MODE=image \\"
+            echo "CORE_IMAGE_REF=${IMAGE_NAME}@${digest} \\"
+            echo "CORE_PACKAGE_VERSION=${CORE_VERSION} \\"
+            echo "  bash ci/run-required.sh"
+            echo '```'
+          } >>"${GITHUB_STEP_SUMMARY}"

package.json

@@ -48,7 +48,8 @@
     "@typescript-eslint/parser": "^8.59.3",
     "eslint": "9.39.1",
     "fallow": "2.75.0",
-    "turbo": "2.9.14"
+    "turbo": "2.9.14",
+    "yaml": "^2.8.3"
   },
   "pnpm": {
     "overrides": {

packages/core/.env.example

@@ -22,8 +22,42 @@ OPENAI_API_KEY=<openai-api-key>
 # Shared API key clients must send as `Authorization: Bearer <key>`.
 # Rotate by restarting the server with a new value.
 # Generate with: openssl rand -hex 32
-# Docker image local mode defaults this to `local-dev-key` when omitted.
-CORE_API_KEY=replace-with-a-strong-random-secret
+# Docker image local mode: auto-generated and persisted to
+# $CORE_STATE_DIR/core-api-key (default /var/lib/atomicmemory/state/core-api-key)
+# when omitted. Required in production/staging (RAW_STORAGE_DEPLOYMENT_ENV).
+# CORE_API_KEY=replace-with-a-strong-random-secret
+
+# Optional: verify Cloud-issued RS256 JWTs for connected-local console access.
+# JWKS_URL + ISSUER + AUDIENCE must be set together; when unset, only
+# CORE_API_KEY is accepted.
+# CLOUD_JWKS_URL=https://api.dev.atomicstrata.ai/.well-known/atomic-core/jwks.json
+# CLOUD_JWT_ISSUER=https://api.dev.atomicstrata.ai
+# CLOUD_JWT_AUDIENCE=atomicmemory-core
+# Optional project binding. When set, tokens for other projects are rejected.
+# When omitted (single-key local), Core trusts the token's own project_id.
+# CLOUD_PROJECT_ID=proj_your_connected_local_project
+# Temporary until Cloud mint includes memory_user_id (dev PoC uses user "default"):
+# CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default
+# Optional — when false (default), CORE_API_KEY is rejected once JWT verify is enabled.
+# CLOUD_JWT_STATIC_KEY_FALLBACK=false
+
+# --- Connected-local (Docker entrypoint defaults) ---
+# Minimal self-hosted + Cloud dev console: set only these in docker run.
+# OPENAI_API_KEY=sk-...
+# ATOMICMEMORY_API_KEY=amc_live_...
+# Mount atomic-memory-state (or equivalent) so the entrypoint can persist
+# an auto-generated CORE_API_KEY at /var/lib/atomicmemory/state/core-api-key.
+# Retrieve after start: docker exec atomic-memory cat /var/lib/atomicmemory/state/core-api-key
+# Optional override: CORE_API_KEY=your-local-secret (skips auto-generation)
+# Setting ATOMICMEMORY_API_KEY turns connected-local on; the Docker entrypoint
+# then applies tier defaults when unset (CLOUD_ENV=dev):
+#   ATOMICMEMORY_API_URL, CLOUD_TRACE_SYNC_ENABLED=true,
+#   CLOUD_JWKS_URL, CLOUD_JWT_ISSUER, CLOUD_JWT_AUDIENCE=atomicmemory-core,
+#   ALLOWED_ORIGINS, CLOUD_JWT_LEGACY_DEFAULT_MEMORY_USER_ID=default,
+#   CLOUD_JWT_STATIC_KEY_FALLBACK=true
+# CLOUD_PROJECT_ID is optional: set it to pin one project; when omitted Core
+# trusts the Cloud-minted token's own project_id. Override tier with
+# CLOUD_ENV=staging|production
 
 # Trusted-proxy identity guard (Radar C4). The shared CORE_API_KEY above
 # authenticates the *caller process*, not the end user; `user_id` is
@@ -206,3 +240,21 @@ EMBEDDING_DIMENSIONS=1536
 # On Railway, DATABASE_URL is injected by the Postgres plugin.
 # Set OPENAI_API_KEY in Railway service variables.
 # PORT is injected automatically by Railway.
+
+# --- Optional Cloud trace sync (OSS Core → Cloud observability) ---
+# Disabled when all vars are absent, or when CLOUD_TRACE_SYNC_ENABLED=false.
+# Partial configuration (some vars set without ENABLED=true) fails startup.
+# CLOUD_TRACE_SYNC_ENABLED=true
+# ATOMICMEMORY_API_URL=https://api.atomicstrata.ai
+# ATOMICMEMORY_API_KEY=amc_replace_me
+# CORE_INSTANCE_ID=optional-stable-installation-id
+# CLOUD_TRACE_SYNC_BATCH_SIZE=100
+# CLOUD_TRACE_SYNC_MAX_ATTEMPTS=8
+# CLOUD_TRACE_SYNC_BASE_RETRY_MS=1000
+# CLOUD_TRACE_SYNC_MAX_RETRY_MS=300000
+# CLOUD_TRACE_SYNC_POLL_INTERVAL_MS=5000
+# CLOUD_TRACE_SYNC_SENT_RETENTION_MS=86400000
+# CLOUD_TRACE_SYNC_SHUTDOWN_DRAIN_MS=10000
+# CLOUD_TRACE_SYNC_CLAIM_STALE_MS=300000
+# CLOUD_TRACE_SYNC_DEAD_LETTER_RETENTION_MS=604800000
+# CLOUD_TRACE_SYNC_MAX_PENDING=10000