chore(schemas): track generated JSON Schemas + add drift CI gate

.github/workflows/ci.yml

@@ -106,6 +106,30 @@ jobs:
       - name: Run gitleaks
         run: gitleaks detect --source . --redact --exit-code 1
 
+  # ---------------------------------------------------------------------------
+  # Schema-drift gate — regenerates dist/schemas/ and fails if anything in the
+  # tracked schema artifacts is modified, staged, or untracked. Catches both
+  # "forgot to regenerate" and "forgot to git add" mistakes.
+  # ---------------------------------------------------------------------------
+  schema-drift:
+    name: Schema drift (dist/schemas/)
+    needs: changes
+    if: needs.changes.outputs.python == 'true'
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+      - run: uv sync --all-packages
+      - name: Regenerate JSON schemas
+        env:
+          PYTHONPATH: packages/core/src
+        run: uv run python -m rag_core.gen_schemas dist/schemas/
+      - name: Verify no drift
+        run: uv run python scripts/check_schema_drift.py
+
   # ---------------------------------------------------------------------------
   # Logging gates — schema + PII + event-registry (activated in Step 0.7b)
   # ---------------------------------------------------------------------------
@@ -186,7 +210,7 @@ jobs:
   # ---------------------------------------------------------------------------
   ci-pass:
     name: CI passed
-    needs: [lint-test, secrets-scan, log-gates, audit, iac]
+    needs: [lint-test, secrets-scan, schema-drift, log-gates, audit, iac]
     if: always()
     runs-on: ubuntu-22.04
     steps:

.gitignore

@@ -10,10 +10,11 @@ __pycache__/
 .Python
 build/
 develop-eggs/
-dist/
+# Use dist/* (not dist/) so the re-includes below take effect — git cannot
+# re-include a path whose parent directory is fully excluded.
+dist/*
 # Track generated JSON Schema artifacts (committed for schema-drift CI gate)
 !dist/schemas/
-!dist/schemas/*.json
 !dist/rag.schema.json
 !dist/rag.schema.yaml
 downloads/

Taskfile.yml

@@ -116,6 +116,14 @@ tasks:
     cmds:
       - "{{.PYTHON}} -m rag_core.gen_schemas dist/schemas/"
 
+  lint:schemas:
+    desc: "Schema-drift gate — fails if dist/schemas/ is out of sync with rag_core types"
+    env:
+      PYTHONPATH: "packages/core/src"
+    cmds:
+      - "{{.PYTHON}} -m rag_core.gen_schemas dist/schemas/"
+      - "{{.PYTHON}} scripts/check_schema_drift.py"
+
   # ---------------------------------------------------------------------------
   # Local dev stack
   # ---------------------------------------------------------------------------

dist/schemas/BlobRef.json

@@ -0,0 +1,36 @@
+{
+  "description": "Reference to a blob stored in the ``Storage`` SPI rather than inline.\n\nUsed by chunks whose text exceeds the inline-storage threshold (see\nADR-0007 tiered storage).  Callers must hydrate via ``Storage.get(uri)``\nonly when the text is actually needed.",
+  "properties": {
+    "uri": {
+      "title": "Uri",
+      "type": "string"
+    },
+    "size_bytes": {
+      "title": "Size Bytes",
+      "type": "integer"
+    },
+    "content_type": {
+      "default": "text/plain; charset=utf-8",
+      "title": "Content Type",
+      "type": "string"
+    },
+    "sha256": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Sha256"
+    }
+  },
+  "required": [
+    "uri",
+    "size_bytes"
+  ],
+  "title": "BlobRef",
+  "type": "object"
+}

dist/schemas/Budget.json

@@ -0,0 +1,55 @@
+{
+  "description": "Per-request resource envelope consulted by the cost-aware planner.",
+  "properties": {
+    "max_tokens": {
+      "anyOf": [
+        {
+          "type": "integer"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Max Tokens"
+    },
+    "max_dollars": {
+      "anyOf": [
+        {
+          "type": "number"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Max Dollars"
+    },
+    "max_wall_ms": {
+      "anyOf": [
+        {
+          "type": "integer"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Max Wall Ms"
+    },
+    "max_iter": {
+      "anyOf": [
+        {
+          "type": "integer"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Max Iter"
+    }
+  },
+  "title": "Budget",
+  "type": "object"
+}

dist/schemas/ChunkRef.json

@@ -0,0 +1,49 @@
+{
+  "description": "Lightweight reference to a chunk returned by ID-only retrieval paths.\n\nUsed by `RetrievalBackend.retrieve_ids` (Step 1.1b) to avoid hydrating\nfull chunk content for results that may be rejected by the reranker or\nPolicyEngine.  Carries only what downstream stages need to fuse, rerank,\nand policy-check before hydration.",
+  "properties": {
+    "chunk_id": {
+      "title": "Chunk Id",
+      "type": "string"
+    },
+    "tenant_id": {
+      "title": "Tenant Id",
+      "type": "string"
+    },
+    "score": {
+      "title": "Score",
+      "type": "number"
+    },
+    "acl_labels": {
+      "items": {
+        "type": "string"
+      },
+      "title": "Acl Labels",
+      "type": "array",
+      "uniqueItems": true
+    },
+    "corpus_id": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Corpus Id"
+    },
+    "metadata": {
+      "additionalProperties": true,
+      "title": "Metadata",
+      "type": "object"
+    }
+  },
+  "required": [
+    "chunk_id",
+    "tenant_id",
+    "score"
+  ],
+  "title": "ChunkRef",
+  "type": "object"
+}

dist/schemas/Cost.json

@@ -0,0 +1,22 @@
+{
+  "description": "Best-effort cost envelope for a plan node (ADR-0008).",
+  "properties": {
+    "ms_estimate": {
+      "default": 0.0,
+      "title": "Ms Estimate",
+      "type": "number"
+    },
+    "tokens_estimate": {
+      "default": 0,
+      "title": "Tokens Estimate",
+      "type": "integer"
+    },
+    "dollars_estimate": {
+      "default": 0.0,
+      "title": "Dollars Estimate",
+      "type": "number"
+    }
+  },
+  "title": "Cost",
+  "type": "object"
+}

dist/schemas/IndexHint.json

@@ -0,0 +1,38 @@
+{
+  "$defs": {
+    "WriteVolume": {
+      "description": "Expected write traffic class for an index, used by ``IndexHint``.\n\nBackends use this to balance build-time cost against query latency:\n\n``low``     \u2014 mostly read; rebuild on schedule is fine.\n``medium``  \u2014 incremental updates expected; favour incremental indexes.\n``high``    \u2014 heavy concurrent writes; favour lock-free / append structures.\n\nSee ADR-0009.",
+      "enum": [
+        "low",
+        "medium",
+        "high"
+      ],
+      "title": "WriteVolume",
+      "type": "string"
+    }
+  },
+  "description": "Hint passed by callers to an ``IndexBackend`` at write / initialise time.\n\nBackends pick the index implementation (flat / ivfflat / HNSW / IVF-PQ /\nDiskANN, etc.) based on these fields. See ADR-0009 for the default mapping\nper backend.  Callers may omit the hint \u2014 backends fall back to a sensible\ndefault for the current corpus size.",
+  "properties": {
+    "estimated_size": {
+      "default": 0,
+      "title": "Estimated Size",
+      "type": "integer"
+    },
+    "recall_target": {
+      "default": 0.95,
+      "title": "Recall Target",
+      "type": "number"
+    },
+    "latency_target_ms": {
+      "default": 50.0,
+      "title": "Latency Target Ms",
+      "type": "number"
+    },
+    "write_volume": {
+      "$ref": "#/$defs/WriteVolume",
+      "default": "low"
+    }
+  },
+  "title": "IndexHint",
+  "type": "object"
+}

dist/schemas/PiiPolicy.json

@@ -0,0 +1,39 @@
+{
+  "$defs": {
+    "PiiAction": {
+      "description": "Per-tenant PII handling policy.",
+      "enum": [
+        "block",
+        "redact",
+        "mask",
+        "encrypt",
+        "tag_only",
+        "allow"
+      ],
+      "title": "PiiAction",
+      "type": "string"
+    }
+  },
+  "description": "Per-tenant PII enforcement contract attached to a `RequestContext`.",
+  "properties": {
+    "action": {
+      "$ref": "#/$defs/PiiAction",
+      "default": "redact"
+    },
+    "entities": {
+      "items": {
+        "type": "string"
+      },
+      "title": "Entities",
+      "type": "array",
+      "uniqueItems": true
+    },
+    "min_score": {
+      "default": 0.5,
+      "title": "Min Score",
+      "type": "number"
+    }
+  },
+  "title": "PiiPolicy",
+  "type": "object"
+}

dist/schemas/PlanNode.json

@@ -0,0 +1,51 @@
+{
+  "$defs": {
+    "Cost": {
+      "description": "Best-effort cost envelope for a plan node (ADR-0008).",
+      "properties": {
+        "ms_estimate": {
+          "default": 0.0,
+          "title": "Ms Estimate",
+          "type": "number"
+        },
+        "tokens_estimate": {
+          "default": 0,
+          "title": "Tokens Estimate",
+          "type": "integer"
+        },
+        "dollars_estimate": {
+          "default": 0.0,
+          "title": "Dollars Estimate",
+          "type": "number"
+        }
+      },
+      "title": "Cost",
+      "type": "object"
+    }
+  },
+  "description": "Single step in a `QueryPlan`.",
+  "properties": {
+    "op": {
+      "title": "Op",
+      "type": "string"
+    },
+    "backend": {
+      "title": "Backend",
+      "type": "string"
+    },
+    "params": {
+      "additionalProperties": true,
+      "title": "Params",
+      "type": "object"
+    },
+    "estimated_cost": {
+      "$ref": "#/$defs/Cost"
+    }
+  },
+  "required": [
+    "op",
+    "backend"
+  ],
+  "title": "PlanNode",
+  "type": "object"
+}