ci.yml

name: CI

# PR-oriented fast gate (target wall time under ~10 minutes). Full multi-Python
# coverage stays on the Nightly workflow. Design/rationale:
# design/operations/pr-ci-fast-gate.md

on:
  push:
    branches: ["main"]
  pull_request:

concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

env:
  UV_PYTHON: "3.13"

jobs:
  changes:
    name: Path filters
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    outputs:
      workers: ${{ steps.filter.outputs.workers }}
      surfaces: ${{ steps.filter.outputs.surfaces }}
      adapters: ${{ steps.filter.outputs.adapters }}
      compose: ${{ steps.filter.outputs.compose }}
      shared_core: ${{ steps.filter.outputs.shared_core }}
      ci_control: ${{ steps.filter.outputs.ci_control }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: .github/ci/path-filters.yml

  quality:
    name: Quality
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Test inventory drift check
        run: python3 .github/ci/check_test_inventory.py
      - name: Architecture dependency check
        run: uv run lint-imports
      - name: Ruff lint
        run: uv run ruff check src/ benchmarks/
      - name: Ruff format check
        run: uv run ruff format --check src/ benchmarks/
      - name: Pyright type check
        run: uv run pyright src/ benchmarks/ --pythonversion 3.13

  unit:
    name: Unit
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Unit pack (no Postgres service)
        run: |
          mapfile -t paths < .github/ci/unit-paths.txt
          if [ "${#paths[@]}" -eq 0 ]; then
            echo "unit-paths.txt is empty" >&2
            exit 1
          fi
          uv run pytest "${paths[@]}" -q --tb=short

  contract-smoke:
    name: Contract smoke
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read
    env:
      REMEMBERSTACK_DATABASE_URL: postgresql+psycopg://rememberstack:rememberstack_test@localhost:5432/rememberstack_test
    steps:
      - uses: actions/checkout@v4
      - name: Start PostgreSQL 19 with required extensions
        run: .github/ci/start-postgres.sh
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Contract smoke pack
        run: |
          mapfile -t paths < .github/ci/contract-smoke-paths.txt
          uv run pytest "${paths[@]}" -v --tb=short

  integration-workers:
    name: Integration (workers)
    needs: changes
    if: >-
      github.event_name == 'push'
      || needs.changes.outputs.workers == 'true'
      || needs.changes.outputs.shared_core == 'true'
      || needs.changes.outputs.ci_control == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read
    env:
      REMEMBERSTACK_DATABASE_URL: postgresql+psycopg://rememberstack:rememberstack_test@localhost:5432/rememberstack_test
    steps:
      - uses: actions/checkout@v4
      - name: Start PostgreSQL 19 with required extensions
        run: .github/ci/start-postgres.sh
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Workers + spine integration
        run: uv run pytest src/tests/workers src/tests/spine -q --tb=short

  integration-surfaces:
    name: Integration (surfaces)
    needs: changes
    if: >-
      github.event_name == 'push'
      || needs.changes.outputs.surfaces == 'true'
      || needs.changes.outputs.shared_core == 'true'
      || needs.changes.outputs.ci_control == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 50
    permissions:
      contents: read
    env:
      REMEMBERSTACK_DATABASE_URL: postgresql+psycopg://rememberstack:rememberstack_test@localhost:5432/rememberstack_test
    steps:
      - uses: actions/checkout@v4
      - name: Start PostgreSQL 19 with required extensions
        run: .github/ci/start-postgres.sh
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Surfaces + eval integration
        run: uv run pytest src/tests/surfaces src/tests/eval -q --tb=short

  integration-adapters:
    name: Integration (adapters)
    needs: changes
    if: >-
      github.event_name == 'push'
      || needs.changes.outputs.adapters == 'true'
      || needs.changes.outputs.shared_core == 'true'
      || needs.changes.outputs.ci_control == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
    env:
      REMEMBERSTACK_DATABASE_URL: postgresql+psycopg://rememberstack:rememberstack_test@localhost:5432/rememberstack_test
    steps:
      - uses: actions/checkout@v4
      - name: Start PostgreSQL 19 with required extensions
        run: .github/ci/start-postgres.sh
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.6"
          enable-cache: true
      - name: Set up Python 3.13
        run: uv python install 3.13
      - name: Install dependencies (locked)
        run: uv sync --locked --python 3.13
      - name: Adapters + profiles + spikes integration
        run: |
          uv run pytest src/tests/adapters src/tests/profiles src/tests/spikes -q --tb=short

  compose-quickstart:
    name: Compose quickstart
    needs: changes
    if: >-
      github.event_name == 'push'
      || needs.changes.outputs.compose == 'true'
      || needs.changes.outputs.ci_control == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 25
    permissions:
      contents: read
    env:
      QUICKSTART_TARGET_SECONDS: "600"
      REMEMBERSTACK_BUILD_REVISION: ${{ github.sha }}
    steps:
      - uses: actions/checkout@v4

      - name: Validate Compose model
        run: docker compose --env-file .env.example config --quiet

      - name: Start fresh self-host deployment
        run: |
          started_at="$(date +%s)"
          docker compose --env-file .env.example up --build --detach --wait
          curl --fail http://localhost:8000/operations
          served_revision="$(curl --fail --silent http://localhost:8000/deployment \
            | python3 -c 'import json,sys; print(json.load(sys.stdin)["build_revision"])')"
          test "$served_revision" = "${REMEMBERSTACK_BUILD_REVISION}"
          query_result="$(curl --fail --silent \
            --header 'Content-Type: application/json' \
            --data '{"sql":"SELECT count(*) AS document_count FROM documents_live","parameters":[],"max_rows":1}' \
            http://localhost:8000/query/sql)"
          printf '%s' "$query_result" | python3 -c \
            'import json,sys; result=json.load(sys.stdin); assert result["termination_reason"] == "completed", result'
          elapsed_seconds="$(($(date +%s) - started_at))"
          echo "Quickstart cold start to first query: ${elapsed_seconds}s / ${QUICKSTART_TARGET_SECONDS}s target" \
            | tee -a "$GITHUB_STEP_SUMMARY"
          test "$elapsed_seconds" -le "$QUICKSTART_TARGET_SECONDS"

      - name: Prove MinIO conditional create
        run: |
          docker compose --env-file .env.example exec -T api python - <<'PY'
          from rememberstack.adapters.selfhost import MinIOObjectStore, MinIOSettings
          from rememberstack.model import ObjectAlreadyExistsError, ObjectKey

          store = MinIOObjectStore(
              bucket="remember-raw", settings=MinIOSettings.model_validate({})
          )
          key = ObjectKey("ci/immutable-object")
          store.write_bytes(key=key, content=b"first")
          try:
              store.write_bytes(key=key, content=b"replacement")
          except ObjectAlreadyExistsError:
              pass
          else:
              raise RuntimeError("MinIO replaced an immutable object key")
          PY

      - name: Prove the zero-cost full pipeline
        run: |
          running_services="$(
            docker compose --env-file .env.example ps --status running --services
          )"
          for service in \
            api worker-convert worker-structure worker-chunk worker-embed-chunk \
            worker-extract-claims worker-normalize-relations \
            worker-adjudicate-observations worker-adjudicate-supersession \
            worker-embed-claim worker-reconcile worker-label-relation; do
            printf '%s\n' "$running_services" | grep -qx "$service"
          done
          ingested="$(
            curl --fail --header 'Content-Type: application/octet-stream' \
              --data-binary ' ' \
              'http://localhost:8000/ingest?filename=empty.md&mime=text%2Fmarkdown'
          )"
          version_id="$(
            printf '%s' "$ingested" |
              docker compose --env-file .env.example exec -T api \
                python -c 'import json, sys; print(json.load(sys.stdin)["version_id"])'
          )"
          # D84/D88/D90: extract is chunk-scoped; empty input skips claim
          # normalize and takes durable empty_complete (no document_version
          # adjudicate_observations row at the entity-fanout generation).
          # Version-level ledger stages:
          # convert→structure→chunk→embed_chunk→adjudicate_supersession→
          # embed_claim→label→reconcile (8).
          observed=''
          for attempt in {1..60}; do
            observed="$(docker compose --env-file .env.example exec -T postgres \
              psql -U rememberstack -d rememberstack -Atc \
              "SELECT count(*), count(*) FILTER (WHERE status='succeeded'), count(DISTINCT stage) FROM processing_state WHERE target_id='$version_id'")"
            if [ "$observed" = '8|8|8' ]; then
              break
            fi
            sleep 1
          done
          test "$observed" = '8|8|8'
          test "$(docker compose --env-file .env.example exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT count(*) FROM cost_ledger')" = '0'
          docker compose --env-file .env.example --profile operations \
            run --rm projections
          readiness="$(
            curl --fail --header 'Content-Type: application/json' \
              --data "{\"version_ids\":[\"$version_id\"],\"require\":{\"pipeline\":true,\"p1\":true,\"live_graph\":true,\"p3\":true}}" \
              'http://localhost:8000/readiness'
          )"
          printf '%s' "$readiness" |
            docker compose --env-file .env.example exec -T api python -c \
              'import json, sys; report = json.load(sys.stdin); assert report["ready"]; assert all(item["ready"] for item in report["versions"]); assert all(item["ready"] for item in report["capabilities"].values())'

      - name: Stage an upgrade from the prior schema
        run: |
          docker compose --env-file .env.example stop \
            api worker-convert worker-structure worker-chunk worker-embed-chunk \
            worker-extract-claims worker-normalize-relations \
            worker-adjudicate-observations worker-adjudicate-supersession \
            worker-embed-claim worker-reconcile worker-label-relation
          docker compose --env-file .env.example run --rm --no-deps \
            --entrypoint alembic setup downgrade p7_02_0016
          test "$(docker compose --env-file .env.example exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT version_num FROM alembic_version')" = 'p7_02_0016'
          test "$(docker compose --env-file .env.example exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT count(*) FROM deployments')" = '1'
          docker compose --env-file .env.example rm --force setup

      - name: Run migrations before application processes restart
        run: |
          compose_args=(
            --env-file .env.example
            --file compose.yaml
            --file .github/compose/upgrade-gate.yaml
          )
          docker compose "${compose_args[@]}" up --detach --wait &
          compose_pid="$!"
          release_gate() {
            docker compose "${compose_args[@]}" exec -T setup \
              touch /tmp/rememberstack-upgrade-release
          }
          trap 'release_gate || true' EXIT

          setup_running=''
          for attempt in {1..30}; do
            setup_running="$(
              docker compose "${compose_args[@]}" ps --status running --services setup
            )"
            if [ "$setup_running" = 'setup' ]; then
              break
            fi
            sleep 1
          done
          test "$setup_running" = 'setup'

          running_services="$(
            docker compose "${compose_args[@]}" ps --status running --services
          )"
          test -z "$(printf '%s\n' "$running_services" \
            | grep -E '^(api|worker-(convert|structure|chunk|embed-chunk|extract-claims|normalize-relations|adjudicate-observations|adjudicate-supersession|embed-claim|reconcile|label-relation))$' || true)"
          test "$(docker compose "${compose_args[@]}" exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT version_num FROM alembic_version')" = 'p7_02_0016'

          release_gate
          wait "$compose_pid"
          trap - EXIT
          test "$(docker compose --env-file .env.example exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT version_num FROM alembic_version')" = 'p9_26_0047'
          test "$(docker compose --env-file .env.example exec -T postgres \
            psql -U rememberstack -d rememberstack -Atc \
            'SELECT count(*) FROM deployments')" = '1'
          running_services="$(
            docker compose "${compose_args[@]}" ps --status running --services
          )"
          for service in \
            api worker-convert worker-structure worker-chunk worker-embed-chunk \
            worker-extract-claims worker-normalize-relations \
            worker-adjudicate-observations worker-adjudicate-supersession \
            worker-embed-claim worker-reconcile worker-label-relation; do
            printf '%s\n' "$running_services" | grep -qx "$service"
          done
          curl --fail http://localhost:8000/healthz

      - name: Show service logs after failure
        if: failure()
        run: docker compose --env-file .env.example logs --no-color

      - name: Remove quickstart deployment
        if: always()
        run: docker compose --env-file .env.example down --volumes

  # Single always-present required check. Integration path jobs are soft.
  pr-gate:
    name: PR gate
    runs-on: ubuntu-latest
    needs: [changes, quality, unit, contract-smoke]
    if: always()
    steps:
      - name: Require hard lanes (incl. path selector)
        run: |
          echo "changes=${{ needs.changes.result }}"
          echo "quality=${{ needs.quality.result }}"
          echo "unit=${{ needs.unit.result }}"
          echo "contract-smoke=${{ needs.contract-smoke.result }}"
          test "${{ needs.changes.result }}" = "success"
          test "${{ needs.quality.result }}" = "success"
          test "${{ needs.unit.result }}" = "success"
          test "${{ needs.contract-smoke.result }}" = "success"