[PROD RELEASE] - WIP

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 
 defaults: &defaults
   docker:
-  - image: cimg/python:3.13.2-browsers
+    - image: cimg/python:3.13.2-browsers
 install_dependency: &install_dependency
   name: Installation of build and deployment dependencies.
   command: |
@@ -35,41 +35,44 @@ builddeploy_steps: &builddeploy_steps
 
 jobs:
   # Build & Deploy against development backend
-  "build-dev":
+  'build-dev':
     !!merge <<: *defaults
     environment:
-      DEPLOY_ENV: "DEV"
-      LOGICAL_ENV: "dev"
-      APPNAME: "tc-ai-api"
+      DEPLOY_ENV: 'DEV'
+      LOGICAL_ENV: 'dev'
+      APPNAME: 'tc-ai-api'
       DEPLOYMENT_ENVIRONMENT: 'dev'
     steps: *builddeploy_steps
 
-  "build-prod":
+  'build-prod':
     !!merge <<: *defaults
     environment:
-      DEPLOY_ENV: "PROD"
-      LOGICAL_ENV: "prod"
-      APPNAME: "tc-ai-api"
+      DEPLOY_ENV: 'PROD'
+      LOGICAL_ENV: 'prod'
+      APPNAME: 'tc-ai-api'
       DEPLOYMENT_ENVIRONMENT: 'prod'
     steps: *builddeploy_steps
 
 workflows:
   version: 2
   build:
     jobs:
-    # Development builds are executed on "develop" branch only.
-    - "build-dev":
-        context: org-global
-        filters:
-          branches:
-            only:
-              - develop
-
-    # Production builds are exectuted only on tagged commits to the
-    # master branch.
-    - "build-prod":
-        context: org-global
-        filters:
-          branches:
-            only:
-              - master
+      # Development builds are executed on "develop" branch only.
+      - 'build-dev':
+          context: org-global
+          filters:
+            branches:
+              only:
+                - develop
+                - challenges-rag
+            tags:
+                only: /^dev-.*/
+
+      # Production builds are exectuted only on tagged commits to the
+      # master branch.
+      - 'build-prod':
+          context: org-global
+          filters:
+            branches:
+              only:
+                - master

.env.sample

@@ -24,9 +24,29 @@ MASTRA_STUDIO_PATH="[ABS PATH_TO_MASTRA_STUDIO_OUTPUT_FOLDER]"
 RAG_EMBEDDING_PROVIDER="[TC-Ollama|AWSBedrock — default TC-Ollama]"
 RAG_EMBEDDING_MODEL_ID="[nomic-embed-text|amazon.titan-embed-text-v2:0 — default nomic-embed-text]"
 VECTOR_INDEX_NAME="[SQL identifier — default challenge_embeddings]"
-VECTOR_SEARCH_THRESHOLD="[0-1 — default 0.5]"
+VECTOR_SEARCH_THRESHOLD="[0-1 — default 0.25]"
 RAG_CHUNK_MAX_SIZE="[characters — default 512]"
 RAG_CHUNK_OVERLAP="[characters — default 50]"
 RAG_TOP_K="[default 10]"
 CHALLENGE_SEARCH_AI_PROVIDER="[TC-Ollama|WiproAI|AWSBedrock|OpenAI — default AWSBedrock]"
-CHALLENGE_SEARCH_AI_MODEL_ID="[default us.anthropic.claude-haiku-4-5]"
+CHALLENGE_SEARCH_AI_MODEL_ID="[default us.anthropic.claude-haiku-4-5]"
+BEDROCK_PROMPT_CACHE_ENABLED="[true|false — default true]"
+BEDROCK_PROMPT_CACHE_TTL="[5m|1h — default 5m]"
+
+# ============== Access control (all optional, defaults shown) ==============
+# See README "Access control" and docs/adr/0004-role-based-access-for-agents-workflows-tools.md
+ACCESS_CONTROL_DEFAULT_POLICY="[public|deny — default public]"
+ACCESS_CONTROL_ROLES_CLAIM="[JWT claim key for member roles — default https://<TC_API_BASE domain>/roles]"
+
+# Per-target override — only needed to diverge from the code default / global default.
+# <CATEGORY> = AGENT | WORKFLOW | TOOL, <TARGET_KEY> = the target's own .id, upper-snake-cased.
+#   ACCESS_POLICY_<CATEGORY>_<TARGET_KEY>_MODE="[public|deny — omit to use ROLES/SCOPES below]"
+#   ACCESS_POLICY_<CATEGORY>_<TARGET_KEY>_ROLES="[comma-separated member roles]"
+#   ACCESS_POLICY_<CATEGORY>_<TARGET_KEY>_SCOPES="[comma-separated M2M scopes]"
+#
+# The two ingestion workflows are restricted by DEFAULT_ACCESS_POLICIES in code, so the
+# following are redundant unless you are overriding them (e.g. loosening for staging):
+#   ACCESS_POLICY_WORKFLOW_CHALLENGE_INGESTION_ROLES="administrator"
+#   ACCESS_POLICY_WORKFLOW_CHALLENGE_INGESTION_SCOPES="challengesRAG:admin"
+#   ACCESS_POLICY_WORKFLOW_CHALLENGE_BULK_INGESTION_ROLES="administrator"
+#   ACCESS_POLICY_WORKFLOW_CHALLENGE_BULK_INGESTION_SCOPES="challengesRAG:admin"

docs/adr/0001-integrate-challenges-vector-rag.md

@@ -79,7 +79,7 @@ Neither can be copied as-is.
 | D9 | Event-driven ingestion (triggering on challenge activation/update) is **deferred**; the first release is on-demand invocation only. | Keeps the initial surface small; the workflow is already idempotent per challenge, so an event trigger can be layered on later without reworking it. |
 | D10 | Metadata carries **only `projectId`** as an opaque project reference (nullable, stored as a string). No project attributes are denormalized into metadata, no project text is indexed, and ingestion makes no call to projects-api. Consumers that need project detail resolve it in a **subsequent** call to `GET /v6/projects/:projectId`. | Keeps ingestion dependent on a single upstream API, and eliminates the denormalization staleness problem outright: there is no copied project field that can drift when a project is renamed, re-typed, or reassigned to a different billing account, so no refresh mechanism, no re-ingestion trigger, and no staleness signal are needed. It also keeps customer-identifying commercial data out of a store searched by similarity, leaving authorization where it belongs — on the projects-api call, which is already scope-guarded. `projectId` alone still supports project-scoped filtering (`{ projectId: { $in: [...] } }`) and roll-up of challenge hits by project. |
 | D11 | The **Challenge Search API** (`GET /v6/challenges`) is the primary bulk ingestion source, not CSV. It supports `projectId`/`projectIds`, `status`, `approvalStatus`, `types`/`tracks`, `tags`/`groups`, `updatedDateStart`/`updatedDateEnd`, and `page`/`perPage` pagination, enabling project-scoped fan-out, status-filtered corpus building, and incremental sync by `updatedDateStart`. CSV backfill remains as a secondary path for offline/air-gapped environments. | The search endpoint is already M2M-authenticated (`scopes: [READ]`) and returns the full challenge payload (including `description`) when `isLightweight` is false (the default). It eliminates the need to export and ship CSV files, and its `updatedDateStart` filter makes incremental sync a single paginated call rather than a full re-export. |
-| D12 | `type` and `track` are stored and filtered as **free-form strings**, not Zod enums. `ChallengeType` is a reference table (`model ChallengeType` with `name`, `isActive`, `isTask`, `isLegacy`), not an enum — new types can be added at runtime. `ChallengeTrackEnum` has four values (`DESIGN`, `DATA_SCIENCE`, `DEVELOPMENT`, `QUALITY_ASSURANCE`) but the API returns `track.name` (human-readable, e.g. "Quality Assurance"), not the enum value, and tracks can be deactivated via `isActive`. | The prototype hardcoded `['Challenge', 'First2Finish', 'Marathon Match']` as a Zod enum for type and `['Data Science', 'Design', 'Development']` for track, missing `Quality Assurance` and rejecting any future type. Treating both as strings (with the known values documented for reference but not enforced) is forward-compatible with the reference-table model and avoids ingestion failures when a new type or track is added. |
+| D12 | `type` and `track` are stored as **free-form strings**, not Zod enums. `ChallengeType` is a reference table (`model ChallengeType` with `name`, `isActive`, `isTask`, `isLegacy`), not an enum — new types can be added at runtime. `ChallengeTrackEnum` has four values (`DESIGN`, `DATA_SCIENCE`, `DEVELOPMENT`, `QUALITY_ASSURANCE`) but the API returns `track.name` (human-readable, e.g. "Quality Assurance"), not the enum value, and tracks can be deactivated via `isActive`. **Amended (2026-08-27):** both the `type` and `track` *query filters* (on `challengeVectorQueryTool` and the `challenge-search` workflow input) are now Zod enums — `type` restricted to `['Challenge', 'Marathon Match']`, `track` restricted to `['Data Science', 'Design', 'Quality Assurance', 'Development']` — a deliberate reversal of the free-form-filter half of this decision for those two callers. Storage/ingestion is unaffected: it still accepts and indexes any `ChallengeType` value (including `First2Finish`, `Task`) and any `track.name`. | The prototype hardcoded `['Challenge', 'First2Finish', 'Marathon Match']` as a Zod enum for type and `['Data Science', 'Design', 'Development']` for track, missing `Quality Assurance` and rejecting any future type. Treating both as strings (with the known values documented for reference but not enforced) is forward-compatible with the reference-table model and avoids ingestion failures when a new type or track is added. The amendment narrows the *query* surface to the values users actually filter by in practice, accepting that a future new `ChallengeType` (or `First2Finish`/`Task`) becomes unfilterable via `type` until the enum is revisited, and that a deactivated or renamed track (`ChallengeTrackEnum`'s `isActive` flag) would need the same revisit for `track` — it does not touch ingestion, so no re-indexing risk. |
 
 ## Implementation plan
 
@@ -108,11 +108,15 @@ Neither can be copied as-is.
   default, per D2) — chunk sizes,
   `VECTOR_SEARCH_THRESHOLD`, `VECTOR_INDEX_NAME` (SQL-identifier validated, as in
   the original), `RAG_TOP_K`. Per D12, `type` and `track` are **not** hardcoded
-  enums — the config documents the known `ChallengeTrackEnum` values
-  (`DESIGN`, `DATA_SCIENCE`, `DEVELOPMENT`, `QUALITY_ASSURANCE`) and the current
-  `ChallengeType` reference-table names for readability, but the query tool
-  accepts any string. Database settings reuse `MASTRA_DB_CONNECTION` and
-  `MASTRA_DB_SCHEMA` (default `ai`).
+  enums at the storage/config layer — the config documents the known
+  `ChallengeTrackEnum` values (`DESIGN`, `DATA_SCIENCE`, `DEVELOPMENT`,
+  `QUALITY_ASSURANCE`) and the current `ChallengeType` reference-table names for
+  readability but does not enforce them at the storage/config layer. The `type`
+  and `track` query filters are Zod enums restricted to `['Challenge', 'Marathon
+  Match']` and `['Data Science', 'Design', 'Quality Assurance', 'Development']`
+  respectively (D12 amendment, 2026-08-27) — narrower than storage on purpose.
+  Database settings reuse `MASTRA_DB_CONNECTION` and `MASTRA_DB_SCHEMA` (default
+  `ai`).
 - **`src/utils/providers/embedding-factory.ts`** — `createEmbeddingModel(provider, modelId)`
   switch mirroring `createModel`, using `ollama.embedding(modelId)` and
   `createBedrockProvider().embedding(modelId)`, logging via `tcAILogger`. Re-exported
@@ -263,7 +267,7 @@ Supporting changes:
 - **`src/mastra/workflows/challenge/challenge-search-workflow.ts`**, id
   `challenge-search`, registered under `workflows` — the deterministic path from D8,
   with no agent and no LLM call:
-  - Input `{ query?: string, skills?: string[], type?: string, track?: string, groups?: string[], projectId?: string | string[], groupBy?: 'chunk' | 'challenge' | 'project', topK?: number, minScore?: number }`.
+  - Input `{ query?: string, skills?: string[], type?: 'Challenge' | 'Marathon Match', track?: 'Data Science' | 'Design' | 'Quality Assurance' | 'Development', groups?: string[], projectId?: string | string[], groupBy?: 'chunk' | 'challenge' | 'project', topK?: number, minScore?: number }` (`type`/`track` enums per the D12 amendment above).
     Filters are supplied explicitly by the caller; unlike the agent path, nothing is
     inferred from natural language.
   - Single step `search-challenges` executing `challengeVectorQueryTool` with the