From ab53883d9969b28c876dffc5df814ef6819bb06a Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Mon, 8 Jun 2026 01:37:28 +0530 Subject: [PATCH] =?UTF-8?q?feat(policy):=20ACL=20egress=20verifier=20?= =?UTF-8?q?=E2=80=94=20defense-in-depth=20re-check=20(Step=206.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a post-retrieval ACL re-check that backstops the Step 6.3 push-down. AclEgressVerifier.verify(ctx, refs) keeps only returned ChunkRefs whose acl_labels overlap the principal's — same overlap semantics as the push-down (no-op on correct results), reading ChunkRef.acl_labels (no re-hydration), and independent of the PolicyEngine — so a filter-translation bug, a backend that ignores the predicate, or a path wired without the engine cannot leak an over-privileged chunk past the boundary. Wired at the gateway as a SupportsRoute wrapper (AclEgressVerifyingRouter) around app.state.retrieval_router, so query/retrieve/corpus/OpenAI/agent all inherit it a layer above HybridRetriever. Opt-in via cfg.acl.verify_egress (default on, gated by cfg.acl.enabled). Emits acl.egress_violation on a caught leak (PII-free: ids + counts). A red-team gate bypasses the push-down (no engine + leaky backend) and proves a zero escaped-violation rate. Co-Authored-By: Claude Opus 4.7 --- TRACKER.md | 23 ++- apps/gateway/src/rag_gateway/_acl_egress.py | 64 ++++++ apps/gateway/src/rag_gateway/app.py | 19 +- apps/gateway/src/rag_gateway/wiring.py | 5 +- apps/gateway/tests/test_acl.py | 105 +++++++++- dist/rag.schema.json | 7 +- dist/rag.schema.yaml | 23 ++- docs/README.md | 5 +- docs/adr/ADR-0036-acl-egress-verifier.md | 89 +++++++++ docs/architecture/policy-engine.md | 12 +- docs/reference/tenancy.md | 36 ++++ packages/config/src/rag_config/schema.py | 12 +- packages/core/src/rag_core/events.py | 2 + .../src/rag_observability/events.py | 5 + packages/policy/src/rag_policy/__init__.py | 2 + packages/policy/src/rag_policy/egress.py | 80 ++++++++ tests/logs/test_event_schema.py | 2 + tests/policy/test_acl_egress.py | 151 ++++++++++++++ tests/redteam/test_acl_egress_verifier.py | 189 ++++++++++++++++++ 19 files changed, 816 insertions(+), 15 deletions(-) create mode 100644 apps/gateway/src/rag_gateway/_acl_egress.py create mode 100644 docs/adr/ADR-0036-acl-egress-verifier.md create mode 100644 packages/policy/src/rag_policy/egress.py create mode 100644 tests/policy/test_acl_egress.py create mode 100644 tests/redteam/test_acl_egress_verifier.py diff --git a/TRACKER.md b/TRACKER.md index 016cc9f..ce507eb 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -14,12 +14,13 @@ | | | |---|---| | **Last updated** | 2026-06-08 | -| **Current phase** | Phase 6 — Governance & Tenancy (**3 / 10 steps**) | -| **Overall** | **67 / 84 steps** — Phases 0–5 complete | -| **Next action** | **Step 6.4 — ACL egress verifier**: post-retrieval re-check of returned chunks against the principal's ACLs (defense-in-depth); zero-ACL-violation-rate gate. Backstops the 6.3 push-down. | +| **Current phase** | Phase 6 — Governance & Tenancy (**4 / 10 steps**) | +| **Overall** | **68 / 84 steps** — Phases 0–5 complete | +| **Next action** | **Step 6.5 — PII policies**: per-tenant PII enforcement at egress (block / redact / allow) through the PolicyEngine `egress_text` decision; `pii.egress_blocked` event. Builds on the per-tenant `pii_policy` already resolved onto `RequestContext` in 6.1. | **Recently shipped** +- **6.4** ✅ ACL egress verifier — `AclEgressVerifier` re-checks returned `ChunkRef`s against the principal's labels at the gateway router boundary (defense-in-depth, same overlap semantics, independent of the PDP); `acl.egress_violation` event; red-team zero-violation-rate gate — [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) - **6.3** ✅ ACL push-down — opt-in `AclPolicyEngine` And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` query (overlap, fail-closed); `acl.egress_denied` event — [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) - **6.2** ✅ Physical tenancy — per-tenant *dedicated* vector index/collection; `TenantConfig.dedicated_index` → `ctx.physical_index` → backend `-` (Noop/Pinecone/Qdrant); cross-tenant probe gate — [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) - **6.1** ✅ Logical multi-tenancy — `TenantResolver` resolves per-tenant `rag.yaml` config (namespace / PII / ACL labels) → `TenantSettings` applied at the gateway boundary; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant` — [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) @@ -57,9 +58,9 @@ | 3 | Gateway & Agent Runtime | 11 | **11** | 0 | | 4 | Reliability | 6 | **6** | 0 | | 5 | Eval & Observability | 7 | **7** | 0 | -| 6 | Governance & Tenancy | 10 | **3** | 7 | +| 6 | Governance & Tenancy | 10 | **4** | 6 | | 7 | Pilot, Harden, GA | 10 | 0 | 10 | -| **Total** | | **84** | **67** | **17** | +| **Total** | | **84** | **68** | **16** | --- @@ -649,7 +650,7 @@ | 6.1 | Logical multi-tenancy | ✅ | [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) — `TenantResolver` → `TenantSettings` (namespace / PII / ACL labels) applied at the boundary; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant` | | 6.2 | Physical tenancy (dedicated index) | ✅ | [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) — `dedicated_index` → `ctx.physical_index` → backend `-` (Noop/Pinecone/Qdrant, lazy create); cross-tenant probe gate | | 6.3 | ACL push-down at retrieval | ✅ | [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) — opt-in `AclPolicyEngine` And-merges `any_in("acl_labels", …)` into every `read_chunk` push-down (overlap, fail-closed); `acl.egress_denied` | -| 6.4 | ACL egress verifier | ⏳ | Post-retrieval re-check; defense-in-depth; zero-ACL-violation-rate gate | +| 6.4 | ACL egress verifier | ✅ | [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) — `AclEgressVerifier` re-checks returned chunks at the gateway router boundary (defense-in-depth above the 6.3 push-down); `acl.egress_violation`; zero-violation-rate red-team gate | | 6.5 | PII policies | ⏳ | Per-tenant PII enforcement (block / redact / allow); egress redaction; `pii.egress_blocked` event | | 6.6 | Immutable audit log | ⏳ | Hash-chain audit log; WORM export; tamper-evident verification; `GET /v1/audit` | | 6.7 | BYOK (Bring Your Own Key) | ⏳ | KMS integration (AWS KMS, GCP KMS, HashiCorp Vault); envelope encryption for embeddings | @@ -685,6 +686,15 @@ - `AclConfig` → `rag.schema`; ~17 new tests incl. an **end-to-end ACL red-team** through `HybridRetriever` (overlap / disjoint / fail-closed / public-label / no-tenant-bypass) + engine unit + event + wiring; all gates green (ruff, mypy --strict, RAG001, schema-drift, policy-coverage, log-schema) - **Scope:** push-down enforcement only — post-retrieval egress re-verification is 6.4, graph **edge** ACLs deferred. [ADR-0035](docs/adr/ADR-0035-acl-pushdown.md), [architecture/policy-engine.md](docs/architecture/policy-engine.md), [reference/tenancy.md](docs/reference/tenancy.md) +### 6.4 — ACL egress verifier ✅ [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) + +- The 6.3 push-down filters ACLs at the source (one layer, re-implemented by each backend translator); 6.4 adds the **defense-in-depth second layer** that the `tests/policy/coverage.py` linter docstring already anticipated. New **`AclEgressVerifier`** (`rag_policy.egress`) — `verify(ctx, refs)` keeps only the returned `ChunkRef`s whose `acl_labels` **overlap** the principal's, dropping the rest — so a filter-translation bug, a backend that ignores the predicate, or a path wired without the policy engine **cannot leak an over-privileged chunk past the boundary** +- **Identical overlap semantics to the push-down** (same `∩ ≠ ∅`, same fail-closed / "public is a shared label" model), so it's a **no-op on a correctly-filtered result** and only ever acts on a genuine leak. Reads `ChunkRef.acl_labels` (which every backend populates regardless of the filter applied) → **no re-hydration**, an O(results) set-intersection per call (hot-path-safe) +- **Independent of the PDP** — it consults only `ctx.principal.acl_labels`, so it backstops the push-down even when the push-down isn't wired. **Layered at the retrieval router boundary**: a thin `SupportsRoute` wrapper (`AclEgressVerifyingRouter`, in the gateway so `rag-retrieval` keeps its no-`rag-policy` property) decorates `app.state.retrieval_router` — the single attribute every surface reads — so query / retrieve / corpus / OpenAI / agent all inherit it, a layer **above** the `HybridRetriever` where the push-down merges +- **Opt-in, on-by-default behind ACL**: new `cfg.acl.verify_egress` (default **true**) gated on `cfg.acl.enabled` — turning ACLs on gives **both** layers; set it false to run the push-down alone (the strict drop is meaningless without the label model, so it never fires for a non-ACL deployment). `build_app(acl_verify_egress=…)` / `build_app_from_config` wire it; `app.state.acl_verify_egress` reports the effective state +- **`acl.egress_violation`** (pre-registered `EVT_ACL_EGRESS_VIOLATION`, `error` level, PII-free: tenant / principal / counts / dropped chunk **ids** only) fires once per call that drops ≥1 ref — the *unexpected* push-down failure, distinct from 6.3's *expected* `acl.egress_denied`; a clean pass is silent +- **Scope:** gateway retrieval surfaces (everything reading `app.state.retrieval_router`); trusts the labels the backend reports on each `ChunkRef` (catching a *mislabelling* backend needs authoritative re-hydration — deferred), per-tenant/per-label violation metrics deferred to the 6.x governance dashboards. `AclConfig` → `rag.schema` regenerated; ~21 new tests incl. a **red-team zero-violation-rate gate** (`tests/redteam/test_acl_egress_verifier.py` — bypassed push-down + leaky backend → verifier drops every violation) + verifier unit + gateway wiring + event-schema; all gates green (ruff, mypy --strict 296 files, RAG001, schema-drift, policy-coverage, log-schema). [ADR-0036](docs/adr/ADR-0036-acl-egress-verifier.md), [architecture/policy-engine.md](docs/architecture/policy-engine.md), [reference/tenancy.md](docs/reference/tenancy.md) + --- ## Phase 7 — Pilot, Harden, GA (Weeks 34–40) ⏳ @@ -839,6 +849,7 @@ Complete log of every PR. Routine Dependabot bumps are grouped; everything else | [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) | 2026-06-08 | feat(tenancy): logical multi-tenancy — per-tenant config resolution (Step 6.1) | | [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) | 2026-06-08 | feat(tenancy): physical tenancy — dedicated index per tenant (Step 6.2) | | [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) | 2026-06-08 | feat(policy): ACL push-down at retrieval — AclPolicyEngine (Step 6.3) | +| [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) | 2026-06-08 | feat(policy): ACL egress verifier — defense-in-depth re-check (Step 6.4) | | #78–#80, #116–#118 | Open | Dependabot bumps — awaiting merge | | #81 | Closed | Dependabot bump — superseded | diff --git a/apps/gateway/src/rag_gateway/_acl_egress.py b/apps/gateway/src/rag_gateway/_acl_egress.py new file mode 100644 index 0000000..af0d00c --- /dev/null +++ b/apps/gateway/src/rag_gateway/_acl_egress.py @@ -0,0 +1,64 @@ +"""Gateway glue: wrap the retrieval router with the Step 6.4 ACL egress verifier. + +A thin :class:`~rag_retrieval.router.SupportsRoute` decorator that runs the inner +router, then re-checks the returned chunks with an +:class:`~rag_policy.AclEgressVerifier` before they leave the retrieval boundary. +Wrapping ``app.state.retrieval_router`` — the single attribute every retrieval +surface reads — gives query / retrieve / corpus / OpenAI / agent the +defense-in-depth backstop uniformly, a layer above the ``HybridRetriever`` where +the 6.3 push-down is merged. Lives in the gateway (the composition root) so +``rag-retrieval`` stays free of any ``rag-policy`` dependency. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from rag_core.filter import FilterExpr +from rag_core.types import ChunkRef, CorpusId, RequestContext, RoutingDecision +from rag_policy import AclEgressVerifier +from rag_retrieval.router import SupportsRoute + +__all__ = ["AclEgressVerifyingRouter"] + + +class AclEgressVerifyingRouter: + """Decorate a ``SupportsRoute`` retriever with a post-retrieval ACL re-check.""" + + def __init__(self, *, inner: SupportsRoute, verifier: AclEgressVerifier) -> None: + self._inner = inner + self._verifier = verifier + + @property + def inner(self) -> SupportsRoute: + return self._inner + + async def route( + self, + ctx: RequestContext, + *, + text: str, + expansion_terms: dict[str, list[str]] | None = None, + hyde_vector: list[float] | None = None, + vector: list[float] | None = None, + graph_seeds: list[str] | None = None, + keyword_query: str | None = None, + corpus_ids: Sequence[CorpusId] | None = None, + top_k: int = 10, + filters: FilterExpr | None = None, + graph_hops: int = 1, + ) -> tuple[RoutingDecision, list[ChunkRef]]: + decision, refs = await self._inner.route( + ctx, + text=text, + expansion_terms=expansion_terms, + hyde_vector=hyde_vector, + vector=vector, + graph_seeds=graph_seeds, + keyword_query=keyword_query, + corpus_ids=corpus_ids, + top_k=top_k, + filters=filters, + graph_hops=graph_hops, + ) + return decision, self._verifier.verify(ctx, refs) diff --git a/apps/gateway/src/rag_gateway/app.py b/apps/gateway/src/rag_gateway/app.py index 3746b38..7908538 100644 --- a/apps/gateway/src/rag_gateway/app.py +++ b/apps/gateway/src/rag_gateway/app.py @@ -369,6 +369,7 @@ def build_app( ab_router: Any | None = None, tenant_resolver: Any | None = None, acl_enabled: bool = False, + acl_verify_egress: bool = True, enable_cors: bool = True, default_tenant_id: TenantId | None = None, ) -> FastAPI: @@ -435,11 +436,27 @@ def build_app( # consumer (corpus router, direct path, OpenAI surface, agent loop) reads # this one attribute, so all of them gain fallback transparently. base_router = retrieval_router or build_default_retrieval_router() - app.state.retrieval_router = ( + routed = ( build_default_fallback_chain(base_router, config=fallback) if fallback_enabled else base_router ) + # ACL egress verifier (Step 6.4) — defense-in-depth backstop *above* the + # router, where the 6.3 push-down lives inside HybridRetriever. When wired, + # it re-checks the chunks each ``route`` returns against the principal's ACL + # labels and drops any over-privileged leak (emitting ``acl.egress_violation``). + # Gated on ``acl_enabled`` because the strict overlap drop only makes sense + # under the ACL label model; every downstream consumer (corpus router, agent, + # OpenAI surface) reads this one attribute, so all gain it transparently. + acl_verify_egress = acl_enabled and acl_verify_egress + if acl_verify_egress: + from rag_policy import AclEgressVerifier + + from rag_gateway._acl_egress import AclEgressVerifyingRouter + + routed = AclEgressVerifyingRouter(inner=routed, verifier=AclEgressVerifier()) + app.state.retrieval_router = routed + app.state.acl_verify_egress = acl_verify_egress app.state.reranker = reranker or build_default_reranker() app.state.packer = packer or build_default_packer() app.state.llm = llm if llm is not None else NoopLLM() diff --git a/apps/gateway/src/rag_gateway/wiring.py b/apps/gateway/src/rag_gateway/wiring.py index 442e0fb..dce661d 100644 --- a/apps/gateway/src/rag_gateway/wiring.py +++ b/apps/gateway/src/rag_gateway/wiring.py @@ -702,8 +702,10 @@ def build_app_from_config(cfg: RagConfig, **overrides: Any) -> FastAPI: # process (the instrument is global, the callback reads current state). _register_platform_metrics_once(overrides.get("drift_registry"), overrides.get("cost_tracker")) - # Label-based ACL push-down (Step 6.3) — opt-in; off by default. + # Label-based ACL push-down (Step 6.3) + egress verifier (Step 6.4) — opt-in; + # off by default. The verifier defaults on but only fires when ``enabled``. acl_enabled = overrides.pop("acl_enabled", cfg.acl.enabled) + acl_verify_egress = overrides.pop("acl_verify_egress", cfg.acl.verify_egress) return build_app( corpus_store=corpus_store, @@ -715,6 +717,7 @@ def build_app_from_config(cfg: RagConfig, **overrides: Any) -> FastAPI: breaker_registry=breaker_registry, quota_enforcer=quota_enforcer, acl_enabled=acl_enabled, + acl_verify_egress=acl_verify_egress, **overrides, ) diff --git a/apps/gateway/tests/test_acl.py b/apps/gateway/tests/test_acl.py index 01d7c8a..54827ce 100644 --- a/apps/gateway/tests/test_acl.py +++ b/apps/gateway/tests/test_acl.py @@ -1,14 +1,31 @@ -"""ACL push-down config + gateway wiring (Step 6.3).""" +"""ACL push-down (Step 6.3) + egress verifier (Step 6.4) config + gateway wiring.""" from __future__ import annotations +from typing import Any + from rag_config import RagConfig from rag_config.schema import AclConfig +from rag_core.types import ( + ChunkId, + ChunkRef, + Principal, + PrincipalId, + PrincipalKind, + QueryShape, + RequestContext, + RoutingDecision, + TenantId, +) from rag_gateway import build_app +from rag_gateway._acl_egress import AclEgressVerifyingRouter from rag_gateway.wiring import build_app_from_config from rag_policy import AclPolicyEngine, NoopPolicyEngine +# --------------------------------------------------------------------------- +# Step 6.3 — push-down config + engine wrapping +# --------------------------------------------------------------------------- def test_acl_config_defaults_off() -> None: assert RagConfig().acl.enabled is False @@ -37,3 +54,89 @@ def test_acl_decorates_without_absorbing_inner() -> None: # the inner engine is preserved (here the default Noop), so ACL composes with # — rather than replaces — the production PDP. assert isinstance(engine.inner, NoopPolicyEngine) + + +# --------------------------------------------------------------------------- +# Step 6.4 — egress verifier config + router wrapping +# --------------------------------------------------------------------------- +def test_verify_egress_config_defaults_on() -> None: + assert RagConfig().acl.verify_egress is True + + +def test_verify_egress_inert_by_default() -> None: + # ACL off (default) → verifier off regardless of its own flag. + app = build_app() + assert app.state.acl_verify_egress is False + assert not isinstance(app.state.retrieval_router, AclEgressVerifyingRouter) + + +def test_verify_egress_wraps_router_when_acl_enabled() -> None: + app = build_app_from_config(RagConfig(acl=AclConfig(enabled=True))) + assert app.state.acl_verify_egress is True + assert isinstance(app.state.retrieval_router, AclEgressVerifyingRouter) + + +def test_verify_egress_off_runs_pushdown_only() -> None: + app = build_app_from_config(RagConfig(acl=AclConfig(enabled=True, verify_egress=False))) + assert app.state.acl_verify_egress is False + assert not isinstance(app.state.retrieval_router, AclEgressVerifyingRouter) + # push-down (6.3) is still wired + assert isinstance(app.state.policy_engine, AclPolicyEngine) + + +def test_verify_egress_inert_when_acl_disabled() -> None: + # verify_egress defaults True, but acl.enabled=False gates it off. + app = build_app_from_config(RagConfig(acl=AclConfig(enabled=False, verify_egress=True))) + assert app.state.acl_verify_egress is False + assert not isinstance(app.state.retrieval_router, AclEgressVerifyingRouter) + + +# --------------------------------------------------------------------------- +# Step 6.4 — wrapped router actually drops over-privileged chunks +# --------------------------------------------------------------------------- +class _StubRouter: + """A SupportsRoute stub that returns whatever refs it's given (push-down bypassed).""" + + def __init__(self, refs: list[ChunkRef]) -> None: + self._refs = refs + + async def route( + self, ctx: RequestContext, *, text: str, **_: Any + ) -> tuple[RoutingDecision, list[ChunkRef]]: + decision = RoutingDecision( + shape=QueryShape.MIXED, use_vector=True, use_keyword=False, use_graph=False + ) + return decision, list(self._refs) + + +def _ref(chunk_id: str, labels: list[str]) -> ChunkRef: + return ChunkRef( + chunk_id=ChunkId(chunk_id), + tenant_id=TenantId("acme"), + score=1.0, + acl_labels=frozenset(labels), + ) + + +def _ctx(labels: list[str]) -> RequestContext: + tid = TenantId("acme") + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), + kind=PrincipalKind.user, + display_name="p", + tenant_id=tid, + acl_labels=frozenset(labels), + ), + ) + + +async def test_wrapped_router_drops_over_privileged_chunks() -> None: + leaky = _StubRouter([_ref("c-eng", ["eng"]), _ref("c-fin", ["fin"]), _ref("c-none", [])]) + app = build_app(acl_enabled=True, retrieval_router=leaky) + router = app.state.retrieval_router + assert isinstance(router, AclEgressVerifyingRouter) + + _, refs = await router.route(_ctx(["eng"]), text="q", top_k=10) + assert {str(r.chunk_id) for r in refs} == {"c-eng"} # c-fin, c-none dropped at egress diff --git a/dist/rag.schema.json b/dist/rag.schema.json index 3ebc3b9..248af5f 100644 --- a/dist/rag.schema.json +++ b/dist/rag.schema.json @@ -2,12 +2,17 @@ "$defs": { "AclConfig": { "additionalProperties": false, - "description": "Label-based ACL push-down enforcement (Step 6.3).\n\nWhen ``enabled`` the gateway wraps its PolicyEngine in an ``AclPolicyEngine``\nthat injects ``any_in(\"acl_labels\", principal.acl_labels)`` into every\n``read_chunk`` push-down \u2014 a chunk is retrievable only when its labels overlap\nthe principal's (the labels resolved per tenant in Step 6.1).\n\n**Disabled by default** \u2014 turning it on changes which chunks a principal can\nretrieve, and is **fail-closed**: a principal carrying no ACL labels matches\nno labeled chunk, so it retrieves nothing (model \"public\" data as a shared\nlabel granted to all principals). Each label-less request emits a PII-free\n``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4.", + "description": "Label-based ACL push-down enforcement (Step 6.3).\n\nWhen ``enabled`` the gateway wraps its PolicyEngine in an ``AclPolicyEngine``\nthat injects ``any_in(\"acl_labels\", principal.acl_labels)`` into every\n``read_chunk`` push-down \u2014 a chunk is retrievable only when its labels overlap\nthe principal's (the labels resolved per tenant in Step 6.1).\n\n**Disabled by default** \u2014 turning it on changes which chunks a principal can\nretrieve, and is **fail-closed**: a principal carrying no ACL labels matches\nno labeled chunk, so it retrieves nothing (model \"public\" data as a shared\nlabel granted to all principals). Each label-less request emits a PII-free\n``acl.egress_denied`` event.\n\n``verify_egress`` adds the Step 6.4 **egress verifier** \u2014 a post-retrieval\nre-check of the chunks that actually came back, dropping any whose labels do\nnot overlap the principal's (defense-in-depth behind the push-down; emits\n``acl.egress_violation`` when it catches one). It uses the same overlap\nsemantics as the push-down, so it is a no-op on correctly-filtered results.\nEnabled by default **but only takes effect when ``enabled`` is true** \u2014 the\nstrict drop only makes sense under the ACL label model; set it false to run\nthe push-down alone.", "properties": { "enabled": { "default": false, "title": "Enabled", "type": "boolean" + }, + "verify_egress": { + "default": true, + "title": "Verify Egress", + "type": "boolean" } }, "title": "AclConfig", diff --git a/dist/rag.schema.yaml b/dist/rag.schema.yaml index 0c1c2cf..d144141 100644 --- a/dist/rag.schema.yaml +++ b/dist/rag.schema.yaml @@ -21,12 +21,33 @@ $defs: label granted to all principals). Each label-less request emits a PII-free - ``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4.' + ``acl.egress_denied`` event. + + + ``verify_egress`` adds the Step 6.4 **egress verifier** — a post-retrieval + + re-check of the chunks that actually came back, dropping any whose labels do + + not overlap the principal''s (defense-in-depth behind the push-down; emits + + ``acl.egress_violation`` when it catches one). It uses the same overlap + + semantics as the push-down, so it is a no-op on correctly-filtered results. + + Enabled by default **but only takes effect when ``enabled`` is true** — the + + strict drop only makes sense under the ACL label model; set it false to run + + the push-down alone.' properties: enabled: default: false title: Enabled type: boolean + verify_egress: + default: true + title: Verify Egress + type: boolean title: AclConfig type: object AuthConfig: diff --git a/docs/README.md b/docs/README.md index fae6339..a5172a2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -103,7 +103,7 @@ | [agent.md](reference/agent.md) | `rag-agent` + agent surface (Step 3.6) — `AgentLoop` / `AgentConfig`, `Controller` (`Scripted` / `Heuristic` / `LLM`), `Tool` / `ToolRegistry` / `RetrieveTool` / `Retriever`, `CheckpointStore` / `InMemoryCheckpointStore` / `AgentSnapshot`, the `rag_core.agent_types` wire models, `POST /v1/agent` (SSE) + gRPC `Converse` + `ragctl agent`, governance boundary, observability, extension points | | [sdks.md](reference/sdks.md) | Official SDKs (Step 3.7) — Python (`agentcontextos`) + TypeScript (`@agentcontextos/sdk`) hand-written clients, generated Go/Java/.NET, identity model, usage per language, the `task openapi:gen` / `sdk:gen` pipeline, extension points | | [admin-ui.md](reference/admin-ui.md) | Admin console (Step 3.10) — Next.js 14 operator GUI (`apps/admin-ui`); 9 pages (dashboard, corpora, connectors, glossary, webhooks, audit, API keys, tenants, config), live-vs-seed hybrid + `NEXT_PUBLIC_GATEWAY_URL`, header identity, running it, internals (shell/primitives/data layer), extension points | -| [tenancy.md](reference/tenancy.md) | Logical multi-tenancy (Step 6.1) — per-tenant `rag.yaml` config (`namespace` / `acl_labels` / `pii_policy` / `quota`); `TenantResolver.resolve(id) → TenantSettings`; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant list` / `resolve`; config table + scope/boundaries (6.2/6.3/6.5) + extension points | +| [tenancy.md](reference/tenancy.md) | Logical multi-tenancy (Step 6.1) — per-tenant `rag.yaml` config (`namespace` / `acl_labels` / `pii_policy` / `quota`); `TenantResolver.resolve(id) → TenantSettings`; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant list` / `resolve`; config table + scope/boundaries (6.2/6.3/6.5) + extension points; physical tenancy (6.2), ACL push-down (6.3) + egress verifier (6.4 — `cfg.acl.verify_egress`) sections | | [webhooks.md](reference/webhooks.md) | Outbound webhooks (Step 3.9) — event catalogue (`ingest.completed` / `audit.policy_violation` / `drift.detected` / `eval.regression`), event envelope, HMAC signing + `verify()`, at-least-once delivery, `/v1/webhooks/subscriptions` CRUD + test, `rag.yaml` block, `ragctl webhooks demo`, internals + extension points | | [integrations.md](reference/integrations.md) | Framework adapters (Step 3.8) — `agentcontextos.integrations.*` for LangChain / LlamaIndex / Haystack / DSPy / LangGraph / CrewAI / AutoGen / Semantic Kernel; per-framework extras, shared config + chunk metadata, usage per framework, internals + extension points | | [status-api.md](reference/status-api.md) | Status & Metrics API (Step 3.11) — `/v1/status/health` / `metrics` / `logs` (+ SSE `logs/stream`), `WS /v1/status/ws`, `/v1/connectors/status`; metric catalogue + request-timing middleware, the `MetricsCollector` / `LogTail` read-side, CORS + query-param identity for browser streams, extension points | @@ -147,7 +147,7 @@ broken, and what to fix before committing to the next phase. | [ADR-0002-eval-framework.md](adr/ADR-0002-eval-framework.md) | Decision: pure-Python Tier 1 metrics always-on; RAGAS as optional Tier 2 | | [ADR-0003-iac-kubernetes-native.md](adr/ADR-0003-iac-kubernetes-native.md) | Decision: Kubernetes-native Terraform modules over cloud-provider-specific RDS/ElastiCache | | [ADR-0004-storage-backends.md](adr/ADR-0004-storage-backends.md) | Decision: single `rag-backends` package, MinIO for S3-compatible dev storage, graceful integration test skip | -| [ADR-0005-policy-engine.md](adr/ADR-0005-policy-engine.md) | Decision: `rag-policy` package as central PDP for ACL/PII/quotas/redaction; supersedes Step 6.4 egress verifier | +| [ADR-0005-policy-engine.md](adr/ADR-0005-policy-engine.md) | Decision: `rag-policy` package as central PDP for ACL/PII/quotas/redaction; the Step 6.4 ACL egress verifier ships as a complementary independent second layer ([ADR-0036](adr/ADR-0036-acl-egress-verifier.md)) | | [ADR-0006-two-stage-rerank.md](adr/ADR-0006-two-stage-rerank.md) | Decision: `Reranker` SPI ships as two-stage cascade (fast bi-encoder → cross-encoder) from day 1 | | [ADR-0007-tiered-storage.md](adr/ADR-0007-tiered-storage.md) | Decision: `Chunk.text: str \| BlobRef` enables lazy hydration and hot/warm/cold tiering | | [ADR-0008-cost-aware-planner.md](adr/ADR-0008-cost-aware-planner.md) | Decision: `QueryPlan` carries `estimated_cost`; fallback triggered planner-side before dispatch, not by post-hoc timeout | @@ -174,6 +174,7 @@ broken, and what to fix before committing to the next phase. | [ADR-0033-logical-multi-tenancy.md](adr/ADR-0033-logical-multi-tenancy.md) | Decision (Step 6.1): make per-tenant `rag.yaml` config drive requests. `TenantResolver` (`rag_config`) maps a tenant id → frozen `TenantSettings` (`rag_core`), applied once at the gateway boundary; resolver in config / settings type in core keeps the `config → core` direction; unknown tenants → safe defaults (isolated not privileged); `RequestContext.namespace` defaults to `tenant_id` (a backend-partition primitive — Pinecone uses it — not a chunk field, so `filter_pushdown` is unchanged); scope stops at resolution + threading (ACL push-down 6.3, PII egress 6.5, physical tenancy 6.2); additive + inert in `build_app` | | [ADR-0034-physical-multi-tenancy.md](adr/ADR-0034-physical-multi-tenancy.md) | Decision (Step 6.2): a *dedicated* vector index/collection per tenant. `TenantConfig.dedicated_index` resolves to a `physical_index` key on `TenantSettings`, threaded onto `RequestContext.physical_index`; backends read only `ctx` (graph is backends→core, never rag-config) and namespace their base under it (`-`), lazily creating it; one instance + per-tenant derivation (no per-tenant instances, no SPI change); Noop is the CI conformance oracle (keyed by `physical_index`) for a cross-tenant probe gate that proves isolation independent of the tenant filter; Noop + Pinecone + Qdrant this step, others later | | [ADR-0035-acl-pushdown.md](adr/ADR-0035-acl-pushdown.md) | Decision (Step 6.3): label-based ACL push-down at retrieval. `AclPolicyEngine` (a decorator like `QuotaPolicyEngine`) And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` push-down at the canonical `HybridRetriever` PDP site — overlap semantics via the existing `AnyIn` predicate (zero backend/translator changes), **fail-closed** (label-less principal matches nothing; "public" = a shared label), **opt-in** via `cfg.acl.enabled`; emits `acl.egress_denied` on a request-level denial; graph edge ACLs + post-retrieval re-verification (6.4) deferred | +| [ADR-0036-acl-egress-verifier.md](adr/ADR-0036-acl-egress-verifier.md) | Decision (Step 6.4): a post-retrieval ACL re-check as a **defense-in-depth second layer** behind the 6.3 push-down. `AclEgressVerifier.verify(ctx, refs)` drops any returned `ChunkRef` whose labels don't overlap the principal's — same overlap semantics (no-op on correct results), reading `ChunkRef.acl_labels` (no re-hydration), **independent of the PDP** (consults only `ctx.principal.acl_labels`) so a push-down bug/bypass can't disable both; wired at the gateway as a `SupportsRoute` wrapper around `app.state.retrieval_router` (covers query/retrieve/corpus/OpenAI/agent); `cfg.acl.verify_egress` default on but gated by `enabled`; emits `acl.egress_violation` on a caught leak; a red-team gate proves a zero escaped-violation rate when the push-down is bypassed; backend-mislabel re-hydration + per-tenant violation metrics deferred | | [ADR-0031-cost-anomaly.md](adr/ADR-0031-cost-anomaly.md) | Decision (Step 5.6c): detect per-tenant spend spikes with a rolling `CostTracker` (not the cumulative quota counter); detect scale-free on the token series (cost = tokens × a constant price) so detection is decoupled from quota pricing and works with quotas off; two gates (ratio + z-score, z relaxed on a flat baseline) → tri-state verdict; put it in `rag-observability` as a `dataclass` (gateway wraps it in a Pydantic `CostStatusResponse`) so there's **no `rag-core` type / `dist/schemas` churn**; feed O(1) from `record_request_usage` before the quota block; pull-based `GET /v1/status/cost` (no per-request span/event); rejected folding into the infra-scoped drift registry, a new package, a `cost.anomaly_detected` push event (deferred), per-model pricing, a time-series DB | | [ADR-0030-drift-monitors.md](adr/ADR-0030-drift-monitors.md) | Decision (Step 5.5): detect retrieval degradation with five drift monitors in a new `rag-drift` package (mirroring rag-feedback); two statistics — PSI (pure, binned, dependency-free) for the distribution monitors + mean-drop for the rate/score monitors — over one scalar-window `DriftMonitor`; infra-scoped registry (like breakers) fed via `observe` from the signals the gateway already computes (query length / retrieval score / HyDE-embedding norm / guard grounded-claim fraction / feedback citation clicks); detection on dashboard-poll `evaluate()` with transition-edge `drift.detected` (structured event + the Step 3.9 webhook, targeting `alert_tenant`); observe-only / inert-by-default / rebaseline; rejected per-tenant monitors, per-dimension embedding PSI, a stats library, a background scheduler, hot-path detection | | [ADR-0029-online-feedback.md](adr/ADR-0029-online-feedback.md) | Decision (Step 5.4): capture online feedback + implicit signals in a new `rag-feedback` package mirroring `rag-provenance` (SPI + types in rag-core; recorder + pure aggregator in the package); one polymorphic `POST /v1/feedback` (a `signal` enum spanning explicit thumbs/rating/comment + implicit citation-click/copy/regenerate/dwell, `kind` inferred); normalise every signal to a `[-1,1]` score so the dashboard has one satisfaction number; **redact-don't-hash** free-text comments via an injected `PIIDetector` (default `NoopPIIDetector` seam, `comment_redacted` flag, PII-free event) — opposite of provenance's hashing; body identity like `/v1/query`; degrade-open + inert-by-default; `GET /v1/status/feedback` dashboard (event-only, no per-call span); admin-UI card deferred to 5.6; rejected separate per-signal endpoints, header-auth, hashing/raw comments, an OTel span per submission, folding into provenance | diff --git a/docs/adr/ADR-0036-acl-egress-verifier.md b/docs/adr/ADR-0036-acl-egress-verifier.md new file mode 100644 index 0000000..f4bb9b4 --- /dev/null +++ b/docs/adr/ADR-0036-acl-egress-verifier.md @@ -0,0 +1,89 @@ +# ADR-0036 — ACL egress verifier + +**Status:** Accepted +**Date:** 2026-06-08 +**Step:** 6.4 — ACL egress verifier (Phase 6 — Governance & Tenancy) +**Related:** [ADR-0035](ADR-0035-acl-pushdown.md) (ACL push-down), [ADR-0005](ADR-0005-policy-engine.md) (PolicyEngine PDP), [ADR-0033](ADR-0033-logical-multi-tenancy.md) (logical tenancy), [architecture/policy-engine.md](../architecture/policy-engine.md), [reference/tenancy.md](../reference/tenancy.md) + +## Context + +Step 6.3 enforces label-based ACLs by *push-down*: `AclPolicyEngine` And-merges +`any_in("acl_labels", principal.acl_labels)` into the `read_chunk` query so the +backend filters at the source. That is the right primary mechanism, but it is a +single layer, and the `tests/policy/coverage.py` linter already names the risk: +governance leaks come from *path-coverage failures* — a filter translation bug +(pgvector `&&`, Qdrant `MatchAny`, Cypher `ANY()` each re-implement the clause), a +backend that silently drops the predicate, or a retrieval path wired without the +policy engine. When the first layer fails, an over-privileged chunk leaves the +backend and there is nothing to catch it. + +Step 6.4 adds the **second layer**: re-check the chunks that actually came back, +at the retrieval egress boundary, and drop any the principal isn't cleared for. + +## Decision + +**1. A standalone verifier, not a PDP decision.** `AclEgressVerifier` +(`rag_policy.egress`) exposes `verify(ctx, refs) -> list[ChunkRef]`. It is *not* +another `PolicyDecision` branch: the whole point is to be an **independent** +mechanism from the push-down, so a bug in the PDP path can't disable both. It +consults only `ctx.principal.acl_labels`, so it backstops the push-down even when +the push-down isn't wired. + +**2. Identical overlap semantics to the push-down.** A ref is kept iff +`ref.acl_labels ∩ principal.acl_labels ≠ ∅` — the same overlap the 6.3 clause +encodes, the same fail-closed model (a label-less principal keeps nothing; +"public" is a shared label). Because the two layers agree, the verifier is a +**no-op on a correctly-filtered result** and only ever acts on a genuine leak. + +**3. Check `ChunkRef.acl_labels`, no re-hydration.** Every backend populates +`ChunkRef.acl_labels` from the stored chunk *independently of the filter it +applied*, so a leaked ref still carries its true labels. The verifier is a cheap +O(results) set-intersection per call — no extra round-trip, hot-path-safe. + +**4. Layered at the router boundary, above `HybridRetriever`.** A thin +`SupportsRoute` decorator (`AclEgressVerifyingRouter`, in the gateway) wraps +`app.state.retrieval_router` — the single attribute every retrieval surface +reads — so query / retrieve / corpus / OpenAI / agent all inherit the backstop +uniformly, a *different* layer from the `HybridRetriever` where the push-down is +merged. The wrapper lives in the gateway (the composition root) so `rag-retrieval` +keeps its no-`rag-policy`-dependency property. + +**5. Opt-in, on-by-default behind ACL.** `cfg.acl.verify_egress` defaults **true** +but is gated on `cfg.acl.enabled` — enabling ACLs turns on *both* layers +(defense-in-depth by default); set `verify_egress: false` to run the push-down +alone. The strict drop is meaningless without the ACL label model, so it never +fires for a non-ACL deployment. + +**6. `acl.egress_violation` on a caught leak.** When the verifier drops ≥1 ref it +emits one PII-free event (`error` level — a caught violation means the first +layer failed) carrying `tenant_id`, `principal_id`, `violation_count`, +`kept_count`, `dropped_chunk_ids` (opaque ids, never label or chunk text), and a +`reason`. A clean pass is silent. This complements 6.3's `acl.egress_denied` +(the *expected* fail-closed denial of a label-less principal); a violation is the +*unexpected* push-down failure. + +## Consequences + +**Positive** +- True defense-in-depth: a push-down bug / bypass cannot leak a chunk past the + retrieval boundary, and the caught failure is alertable. +- Zero coupling to backends or the PDP: works regardless of which translator ran + or whether the policy engine was wired — verified by a red-team gate that + bypasses the push-down and asserts a zero escaped-violation rate. +- No-op on healthy traffic (shared overlap semantics), so it's safe to default on + with ACLs. + +**Negative / deferred** +- Trusts `ChunkRef.acl_labels` as reported by the backend; a backend that + *mislabels* a ref (not just mis-filters) is out of scope — catching that needs + authoritative re-hydration, deferred. +- Covers the gateway retrieval surfaces (everything reading + `app.state.retrieval_router`); direct `HybridRetriever` / `ragctl` smoke paths + are dev tooling and not wrapped. +- Per-tenant / per-label violation metrics beyond the single event are deferred + to the Step 6.x governance dashboards. + +## See also +- [architecture/policy-engine.md](../architecture/policy-engine.md) — the two enforcement layers +- [reference/tenancy.md](../reference/tenancy.md) — ACL labels + enforcement (6.3 push-down, 6.4 egress) +- [ADR-0035](ADR-0035-acl-pushdown.md) — the push-down this verifier backstops diff --git a/docs/architecture/policy-engine.md b/docs/architecture/policy-engine.md index edab6d9..e250ea4 100644 --- a/docs/architecture/policy-engine.md +++ b/docs/architecture/policy-engine.md @@ -103,6 +103,16 @@ class MyOrgPolicyEngine(PolicyEngine): > (a label-less principal matches nothing). It composes with any inner engine, so a > custom PDP only needs to override `read_chunk` semantics if it wants more than > label overlap. See [ADR-0035](../adr/ADR-0035-acl-pushdown.md). +> +> **ACL egress verifier (Step 6.4).** The push-down is the primary, source-side +> layer; `rag_policy.AclEgressVerifier` is the **second layer** — a post-retrieval +> re-check (`verify(ctx, refs)`) that drops any returned `ChunkRef` whose labels +> don't overlap the principal's, using the *same* overlap semantics so it's a no-op +> on a correctly-filtered result. It is deliberately **independent** of the PDP +> (it reads only `ctx.principal.acl_labels`), so a push-down bug / bypass can't +> disable both. Wired at the gateway as a `SupportsRoute` wrapper around the +> retrieval router (`cfg.acl.verify_egress`, default on, gated by `enabled`); emits +> `acl.egress_violation` on a caught leak. See [ADR-0036](../adr/ADR-0036-acl-egress-verifier.md). Register at the composition root (`apps/gateway/`): @@ -120,4 +130,4 @@ Future built-ins on the roadmap: `OpaPolicyEngine` (delegates to an OPA sidecar) - [ADR-0005](../adr/ADR-0005-policy-engine.md) — the decision. - [request-context.md](request-context.md) — the envelope every policy call receives. - TRACKER.md Step 1.1c — implementation step. -- Steps 1.7, 4.5, 6.3, 6.5 — consumers; Step 6.4 superseded. +- Steps 1.7, 4.5, 6.3, 6.5 — consumers; Step 6.4 ships the egress verifier as an independent second layer ([ADR-0036](../adr/ADR-0036-acl-egress-verifier.md)). diff --git a/docs/reference/tenancy.md b/docs/reference/tenancy.md index 34d1a25..504fd56 100644 --- a/docs/reference/tenancy.md +++ b/docs/reference/tenancy.md @@ -98,6 +98,7 @@ lands in later Phase-6 steps: - **Physical tenancy** (a dedicated index per tenant) — **6.2** (below). - **ACL push-down** at retrieval (inject the labels into every backend query) — **6.3** ✅: enable `cfg.acl.enabled` to wrap the PolicyEngine in an `AclPolicyEngine` that And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` push-down. **Fail-closed** (a label-less principal retrieves nothing; model "public" as a shared label) and emits `acl.egress_denied`. See [ADR-0035](../adr/ADR-0035-acl-pushdown.md). +- **ACL egress verifier** — a post-retrieval re-check that backstops the push-down — **6.4** ✅ (below). - **PII egress** enforcement (block / redact answers per policy) — **6.5**. ## Physical tenancy (Step 6.2) @@ -138,6 +139,41 @@ filter*; live-service isolation is covered by the Pinecone / Qdrant integration tests. See [architecture/multi-tenancy.md](../architecture/multi-tenancy.md) and [ADR-0034](../adr/ADR-0034-physical-multi-tenancy.md). +## ACL egress verifier (Step 6.4) + +The 6.3 push-down filters ACLs *at the source*. The egress verifier is the +**defense-in-depth second layer**: it re-checks the chunks that actually came +back and drops any whose labels don't overlap the principal's — so a filter +translation bug, a backend that ignores the predicate, or a retrieval path wired +without the policy engine can't leak an over-privileged chunk past the boundary. + +- **Same overlap semantics as the push-down** (`ref.acl_labels ∩ + principal.acl_labels ≠ ∅`, same fail-closed / "public is a shared label" + model), so it is a **no-op on correctly-filtered results** and only acts on a + real leak. +- **Cheap + hydration-free** — it reads `ChunkRef.acl_labels` (which every + backend populates regardless of the filter applied), an O(results) + set-intersection per query. +- **Layered at the retrieval router boundary** — a `SupportsRoute` wrapper around + `app.state.retrieval_router`, so query / retrieve / corpus / OpenAI / agent all + inherit it, a layer *above* the `HybridRetriever` where the push-down lives. +- **Emits `acl.egress_violation`** (PII-free: ids + counts) when it catches a + leak — the *unexpected* push-down failure, distinct from 6.3's expected + `acl.egress_denied`. + +```yaml +acl: + enabled: true # 6.3 push-down + verify_egress: true # 6.4 egress verifier (default true; only fires when enabled) +``` + +`verify_egress` defaults **on** but is gated on `enabled` — turning ACLs on gives +both layers; set it `false` to run the push-down alone. The **zero-violation-rate +gate** (`tests/redteam/test_acl_egress_verifier.py`) bypasses the push-down (no +policy engine, and a backend that drops the filter) and proves the verifier +reduces the escaped-violation rate to zero across a battery of principals. See +[ADR-0036](../adr/ADR-0036-acl-egress-verifier.md). + ## Extension points - **Namespace strategy** — backends that namespace natively partition on diff --git a/packages/config/src/rag_config/schema.py b/packages/config/src/rag_config/schema.py index 94b3a51..750ae45 100644 --- a/packages/config/src/rag_config/schema.py +++ b/packages/config/src/rag_config/schema.py @@ -547,10 +547,20 @@ class AclConfig(_StrictBase): retrieve, and is **fail-closed**: a principal carrying no ACL labels matches no labeled chunk, so it retrieves nothing (model "public" data as a shared label granted to all principals). Each label-less request emits a PII-free - ``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4. + ``acl.egress_denied`` event. + + ``verify_egress`` adds the Step 6.4 **egress verifier** — a post-retrieval + re-check of the chunks that actually came back, dropping any whose labels do + not overlap the principal's (defense-in-depth behind the push-down; emits + ``acl.egress_violation`` when it catches one). It uses the same overlap + semantics as the push-down, so it is a no-op on correctly-filtered results. + Enabled by default **but only takes effect when ``enabled`` is true** — the + strict drop only makes sense under the ACL label model; set it false to run + the push-down alone. """ enabled: bool = False + verify_egress: bool = True class QuotaConfig(_StrictBase): diff --git a/packages/core/src/rag_core/events.py b/packages/core/src/rag_core/events.py index d68645f..505c772 100644 --- a/packages/core/src/rag_core/events.py +++ b/packages/core/src/rag_core/events.py @@ -6,6 +6,7 @@ """ from rag_observability.events import EVT_ACL_DENIED as EVT_ACL_DENIED +from rag_observability.events import EVT_ACL_EGRESS_VIOLATION as EVT_ACL_EGRESS_VIOLATION from rag_observability.events import EVT_BREAKER_OPENED as EVT_BREAKER_OPENED from rag_observability.events import EVT_CACHE_HIT as EVT_CACHE_HIT from rag_observability.events import EVT_CACHE_INVALIDATED as EVT_CACHE_INVALIDATED @@ -67,6 +68,7 @@ "EVT_PII_DETECTED", "EVT_PII_EGRESS_BLOCKED", "EVT_ACL_DENIED", + "EVT_ACL_EGRESS_VIOLATION", "EVT_GUARD_CLAIM_BLOCKED", "EVT_SPI_CALL", "EVT_BREAKER_OPENED", diff --git a/packages/observability/src/rag_observability/events.py b/packages/observability/src/rag_observability/events.py index a144632..c2ad31e 100644 --- a/packages/observability/src/rag_observability/events.py +++ b/packages/observability/src/rag_observability/events.py @@ -53,6 +53,7 @@ "EVT_PII_DETECTED", "EVT_PII_EGRESS_BLOCKED", "EVT_ACL_DENIED", + "EVT_ACL_EGRESS_VIOLATION", "EVT_GUARD_CLAIM_BLOCKED", # event name constants — SPI / reliability "EVT_SPI_CALL", @@ -147,6 +148,10 @@ def _register(cls, name: str) -> str: EVT_PII_DETECTED: str = RagEvent._register("pii.detected") EVT_PII_EGRESS_BLOCKED: str = RagEvent._register("pii.egress_blocked") EVT_ACL_DENIED: str = RagEvent._register("acl.egress_denied") +# Step 6.4 — the post-retrieval ACL egress verifier dropped a returned chunk +# whose labels did not overlap the principal's: a caught defense-in-depth +# violation (the 6.3 push-down should have filtered it at the source). +EVT_ACL_EGRESS_VIOLATION: str = RagEvent._register("acl.egress_violation") EVT_GUARD_CLAIM_BLOCKED: str = RagEvent._register("guard.claim_blocked") # SPI / reliability diff --git a/packages/policy/src/rag_policy/__init__.py b/packages/policy/src/rag_policy/__init__.py index fec8d32..de4148e 100644 --- a/packages/policy/src/rag_policy/__init__.py +++ b/packages/policy/src/rag_policy/__init__.py @@ -6,6 +6,7 @@ """ from rag_policy.acl import AclPolicyEngine +from rag_policy.egress import AclEgressVerifier from rag_policy.engine import PolicyEngine from rag_policy.filter import ( And, @@ -39,6 +40,7 @@ "PolicyEngine", "NoopPolicyEngine", "AclPolicyEngine", + "AclEgressVerifier", "PolicyWriter", # decisions + results "PolicyDecision", diff --git a/packages/policy/src/rag_policy/egress.py b/packages/policy/src/rag_policy/egress.py new file mode 100644 index 0000000..5394aab --- /dev/null +++ b/packages/policy/src/rag_policy/egress.py @@ -0,0 +1,80 @@ +"""AclEgressVerifier — post-retrieval ACL re-check (Step 6.4). + +Defense-in-depth backstop for the Step 6.3 ACL *push-down*. The push-down +(:class:`~rag_policy.acl.AclPolicyEngine`) And-merges an ``acl_labels`` overlap +clause into the ``read_chunk`` query so the backend filters at the source. This +verifier is the **second layer**: it re-checks the chunks that actually came +back against the principal's ACL labels and drops any that should not have been +returned — so a bug or bypass in the first layer (a wrong filter translation, a +backend that ignores the predicate, a retriever wired without the policy engine) +cannot leak a chunk past the boundary. + +The two layers share **identical overlap semantics** so they never disagree on a +correctly-filtered result: a chunk is cleared iff its ``acl_labels`` intersect +the principal's (the same ``any_in`` overlap the push-down uses, and the same +"public is a shared label" / fail-closed model — a label-less principal is +cleared for nothing). On a healthy request the verifier therefore drops nothing +and stays silent; it only acts — and emits one PII-free +``acl.egress_violation`` event — when it actually catches an over-privileged +chunk, which is by definition a first-layer failure worth alerting on. + +The check reads :attr:`ChunkRef.acl_labels` (which every backend populates +independently of the filter it applied), so it needs no re-hydration and is a +cheap O(results) set-intersection per call. It is independent of the +``PolicyEngine``: it consults only ``ctx.principal.acl_labels``, so it backstops +the push-down even when the push-down isn't wired. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from rag_core.events import EVT_ACL_EGRESS_VIOLATION +from rag_core.types import ChunkRef, RequestContext +from rag_observability.logging import get_logger + +_log = get_logger(__name__) + +__all__ = ["AclEgressVerifier"] + + +class AclEgressVerifier: + """Re-check returned chunks against the principal's ACL labels (Step 6.4).""" + + def verify(self, ctx: RequestContext, refs: Sequence[ChunkRef]) -> list[ChunkRef]: + """Return only the ``refs`` the principal is cleared for. + + A ref is kept iff its ``acl_labels`` overlap ``ctx.principal.acl_labels`` + — the same overlap the 6.3 push-down enforces, so a correctly-filtered + result is unchanged. Anything else is dropped (fail-closed: a label-less + principal keeps nothing) and counted into a single ``acl.egress_violation`` + event so the caught leak is observable. + """ + cleared = ctx.principal.acl_labels + kept: list[ChunkRef] = [] + dropped: list[str] = [] + for ref in refs: + if ref.acl_labels & cleared: + kept.append(ref) + else: + dropped.append(str(ref.chunk_id)) + if dropped: + self._emit_violation(ctx, dropped=dropped, kept_n=len(kept)) + return kept + + @staticmethod + def _emit_violation(ctx: RequestContext, *, dropped: list[str], kept_n: int) -> None: + # A caught violation means the push-down (first layer) let an + # over-privileged chunk through — error level, PII-free (ids only). + _log.error( + EVT_ACL_EGRESS_VIOLATION, + extra={ + "event_kind": EVT_ACL_EGRESS_VIOLATION, + "tenant_id": str(ctx.tenant_id), + "principal_id": str(ctx.principal.id), + "violation_count": len(dropped), + "kept_count": kept_n, + "dropped_chunk_ids": dropped, + "reason": "acl_overlap_failed", + }, + ) diff --git a/tests/logs/test_event_schema.py b/tests/logs/test_event_schema.py index 03be835..1f403cc 100644 --- a/tests/logs/test_event_schema.py +++ b/tests/logs/test_event_schema.py @@ -8,6 +8,7 @@ from pydantic import ValidationError from rag_core.events import ( EVT_ACL_DENIED, + EVT_ACL_EGRESS_VIOLATION, EVT_BREAKER_OPENED, EVT_CACHE_HIT, EVT_CACHE_INVALIDATED, @@ -51,6 +52,7 @@ EVT_PII_DETECTED, EVT_PII_EGRESS_BLOCKED, EVT_ACL_DENIED, + EVT_ACL_EGRESS_VIOLATION, EVT_GUARD_CLAIM_BLOCKED, EVT_SPI_CALL, EVT_BREAKER_OPENED, diff --git a/tests/policy/test_acl_egress.py b/tests/policy/test_acl_egress.py new file mode 100644 index 0000000..f122701 --- /dev/null +++ b/tests/policy/test_acl_egress.py @@ -0,0 +1,151 @@ +"""Unit tests for AclEgressVerifier — post-retrieval ACL re-check (Step 6.4).""" + +from __future__ import annotations + +from typing import Any + +import pytest +from rag_core.types import ( + ChunkId, + ChunkRef, + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) +from rag_policy import AclEgressVerifier + + +def _ctx(labels: list[str], *, tenant: str = "acme") -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), + kind=PrincipalKind.user, + display_name="p", + tenant_id=tid, + acl_labels=frozenset(labels), + ), + ) + + +def _ref(chunk_id: str, labels: list[str], *, tenant: str = "acme") -> ChunkRef: + return ChunkRef( + chunk_id=ChunkId(chunk_id), + tenant_id=TenantId(tenant), + score=1.0, + acl_labels=frozenset(labels), + ) + + +def _ids(refs: list[ChunkRef]) -> list[str]: + return [str(r.chunk_id) for r in refs] + + +# --------------------------------------------------------------------------- +# overlap semantics — mirrors the 6.3 push-down exactly +# --------------------------------------------------------------------------- +def test_keeps_only_overlapping_refs() -> None: + verifier = AclEgressVerifier() + refs = [ + _ref("c-eng", ["eng"]), + _ref("c-fin", ["fin"]), + _ref("c-pub", ["public"]), + _ref("c-none", []), + ] + kept = verifier.verify(_ctx(["eng", "public"]), refs) + assert _ids(kept) == ["c-eng", "c-pub"] # not c-fin, not the label-less c-none + + +def test_disjoint_labels_drop_everything() -> None: + verifier = AclEgressVerifier() + refs = [_ref("c-eng", ["eng"]), _ref("c-fin", ["fin"])] + assert verifier.verify(_ctx(["hr"]), refs) == [] + + +def test_label_less_principal_is_fail_closed() -> None: + verifier = AclEgressVerifier() + refs = [_ref("c-eng", ["eng"]), _ref("c-pub", ["public"]), _ref("c-none", [])] + # No clearance → nothing, not even the unlabeled chunk. + assert verifier.verify(_ctx([]), refs) == [] + + +def test_label_less_chunk_is_dropped() -> None: + verifier = AclEgressVerifier() + # An unlabeled chunk overlaps nobody (matches the push-down: any_in over [] is false). + assert verifier.verify(_ctx(["eng"]), [_ref("c-none", [])]) == [] + + +def test_public_label_is_the_shared_grant_idiom() -> None: + verifier = AclEgressVerifier() + refs = [_ref("c-pub", ["public"]), _ref("c-eng", ["eng"])] + assert _ids(verifier.verify(_ctx(["public"]), refs)) == ["c-pub"] + + +def test_empty_input_returns_empty() -> None: + assert AclEgressVerifier().verify(_ctx(["eng"]), []) == [] + + +def test_kept_refs_are_unchanged_identity_and_order() -> None: + verifier = AclEgressVerifier() + a, b, c = _ref("a", ["eng"]), _ref("b", ["fin"]), _ref("c", ["eng"]) + kept = verifier.verify(_ctx(["eng"]), [a, b, c]) + # same objects, original order preserved (verifier filters, never reorders/copies). + assert kept == [a, c] + assert kept[0] is a and kept[1] is c + + +# --------------------------------------------------------------------------- +# acl.egress_violation event +# --------------------------------------------------------------------------- +def _capture(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, Any]]]: + import rag_policy.egress as egress_mod + + calls: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr(egress_mod._log, "error", lambda msg, **kw: calls.append((msg, kw))) + return calls + + +def test_violation_emits_one_event_with_counts_and_ids( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = _capture(monkeypatch) + refs = [_ref("c-eng", ["eng"]), _ref("c-fin", ["fin"]), _ref("c-hr", ["hr"])] + AclEgressVerifier().verify(_ctx(["eng"]), refs) + + events = [kw for msg, kw in calls if msg == "acl.egress_violation"] + assert len(events) == 1 + extra = events[0]["extra"] + assert extra["event_kind"] == "acl.egress_violation" + assert extra["tenant_id"] == "acme" + assert extra["violation_count"] == 2 # c-fin, c-hr + assert extra["kept_count"] == 1 # c-eng + assert set(extra["dropped_chunk_ids"]) == {"c-fin", "c-hr"} + assert extra["reason"] == "acl_overlap_failed" + + +def test_clean_pass_is_silent(monkeypatch: pytest.MonkeyPatch) -> None: + calls = _capture(monkeypatch) + refs = [_ref("c-eng", ["eng"]), _ref("c-pub", ["public"])] + AclEgressVerifier().verify(_ctx(["eng", "public"]), refs) + assert [msg for msg, _ in calls if msg == "acl.egress_violation"] == [] + + +def test_event_is_pii_free(monkeypatch: pytest.MonkeyPatch) -> None: + calls = _capture(monkeypatch) + AclEgressVerifier().verify(_ctx(["eng"]), [_ref("c-1", ["topsecret"])]) + extra = next(kw for msg, kw in calls if msg == "acl.egress_violation")["extra"] + # Only identifiers / counts / reason — never label text or chunk content. + assert set(extra) == { + "event_kind", + "tenant_id", + "principal_id", + "violation_count", + "kept_count", + "dropped_chunk_ids", + "reason", + } + # the dropped chunk's label value is never carried in the event payload + assert "topsecret" not in str(extra) diff --git a/tests/redteam/test_acl_egress_verifier.py b/tests/redteam/test_acl_egress_verifier.py new file mode 100644 index 0000000..43c9d67 --- /dev/null +++ b/tests/redteam/test_acl_egress_verifier.py @@ -0,0 +1,189 @@ +"""Red-team gate: the ACL egress verifier catches every push-down leak (Step 6.4). + +The Step 6.3 push-down filters ACLs at the source; Step 6.4 is the +defense-in-depth backstop that re-checks what actually came back. This gate +proves the backstop works **when the first layer fails** — a retriever wired +without the policy engine (the ACL clause never injected), and a backend that +silently drops the filter (a translation bug). In both cases the raw retrieval +leaks over-privileged chunks, and in both cases +:class:`~rag_policy.AclEgressVerifier` reduces the *escaped-violation rate to +zero*: every surviving chunk overlaps the principal's labels. +""" + +from __future__ import annotations + +from rag_core.filter import FilterExpr +from rag_core.spi.noop.vector_store import NoopVectorStore +from rag_core.types import ( + ChunkId, + ChunkRef, + CorpusId, + Embedding, + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) +from rag_policy import AclEgressVerifier, AclPolicyEngine, NoopPolicyEngine +from rag_retrieval import HybridRetriever, HybridWeights + +_VEC = [1.0, 0.0] + +# tenant "acme" corpus: chunk_id -> acl_labels +_CHUNKS: dict[str, list[str]] = { + "c-eng": ["eng"], + "c-fin": ["fin"], + "c-pub": ["public"], + "c-none": [], + "c-multi": ["eng", "fin"], +} + +# A battery of principal clearances spanning overlap / disjoint / empty / multi. +_PRINCIPALS: list[list[str]] = [ + [], + ["eng"], + ["fin"], + ["public"], + ["eng", "fin"], + ["hr"], + ["eng", "hr"], +] + + +def _ctx(labels: list[str], *, tenant: str = "acme") -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), + kind=PrincipalKind.user, + display_name="p", + tenant_id=tid, + acl_labels=frozenset(labels), + ), + ) + + +def _emb(chunk_id: str, tenant: str, labels: list[str]) -> Embedding: + return Embedding( + chunk_id=ChunkId(chunk_id), + tenant_id=TenantId(tenant), + vector=_VEC, + dimension=2, + model="m", + acl_labels=frozenset(labels), + ) + + +def _cleared(labels: list[str]) -> set[str]: + """The chunks a principal *should* be able to see — overlap semantics.""" + clr = set(labels) + return {cid for cid, lbls in _CHUNKS.items() if set(lbls) & clr} + + +class _LeakyVectorStore(NoopVectorStore): + """A backend that silently drops the ACL filter — simulates a translator bug. + + Tenant scoping (the partition check) is left intact; only the pushed-down + ``FilterExpr`` is discarded, so an ACL-cleared query still returns every + chunk in the tenant — exactly the leak Step 6.4 must catch. + """ + + async def retrieve_ids( + self, + ctx: RequestContext, + vector: list[float], + top_k: int, + corpus_ids: list[CorpusId], + filters: FilterExpr | None = None, + ) -> list[ChunkRef]: + return await super().retrieve_ids(ctx, vector, top_k, corpus_ids, filters=None) + + +async def _seed(store: NoopVectorStore) -> NoopVectorStore: + await store.bulk_index( + _ctx([]), + [_emb(cid, "acme", lbls) for cid, lbls in _CHUNKS.items()], + ) + await store.bulk_index(_ctx([], tenant="globex"), [_emb("g-eng", "globex", ["eng"])]) + return store + + +async def _retriever_without_policy() -> HybridRetriever: + """Push-down never wired — the ACL clause is never injected (first layer absent).""" + store = await _seed(NoopVectorStore()) + return HybridRetriever(vector_backend=store, weights=HybridWeights()) + + +async def _retriever_with_leaky_backend() -> HybridRetriever: + """Push-down wired, but the backend ignores the filter (first layer bypassed).""" + store = await _seed(_LeakyVectorStore()) + return HybridRetriever( + vector_backend=store, + weights=HybridWeights(), + policy_engine=AclPolicyEngine(inner=NoopPolicyEngine()), + ) + + +async def _ids(hybrid: HybridRetriever, ctx: RequestContext) -> set[str]: + refs = await hybrid.retrieve(ctx, vector=_VEC, top_k=10) + return {str(r.chunk_id) for r in refs} + + +async def _verified_ids(hybrid: HybridRetriever, ctx: RequestContext) -> set[str]: + refs = await hybrid.retrieve(ctx, vector=_VEC, top_k=10) + return {str(r.chunk_id) for r in AclEgressVerifier().verify(ctx, refs)} + + +# --------------------------------------------------------------------------- +# The gate: zero escaped violations across the whole principal battery +# --------------------------------------------------------------------------- +async def test_verifier_zero_violation_rate_when_pushdown_absent() -> None: + hybrid = await _retriever_without_policy() + for labels in _PRINCIPALS: + ctx = _ctx(labels) + verified = await _verified_ids(hybrid, ctx) + expected = _cleared(labels) + assert verified == expected, f"labels={labels}: {verified} != {expected}" + # Headline gate: nothing the principal isn't cleared for survives. + assert verified <= expected + + +async def test_verifier_zero_violation_rate_when_backend_leaks() -> None: + hybrid = await _retriever_with_leaky_backend() + for labels in _PRINCIPALS: + ctx = _ctx(labels) + verified = await _verified_ids(hybrid, ctx) + expected = _cleared(labels) + assert verified == expected, f"labels={labels}: {verified} != {expected}" + + +async def test_gate_is_meaningful_raw_retrieval_actually_leaks() -> None: + # Sanity: without the verifier the bypassed push-down really does over-return, + # so the gate above is exercising a real leak rather than a no-op. + hybrid = await _retriever_without_policy() + raw = await _ids(hybrid, _ctx(["eng"])) + assert _cleared(["eng"]) < raw # strict superset → there were violations to catch + + +async def test_verifier_is_noop_on_correctly_pushed_down_results() -> None: + # With a correct push-down, the backstop changes nothing (no double-filtering bug). + store = await _seed(NoopVectorStore()) + hybrid = HybridRetriever( + vector_backend=store, + weights=HybridWeights(), + policy_engine=AclPolicyEngine(inner=NoopPolicyEngine()), + ) + for labels in _PRINCIPALS: + ctx = _ctx(labels) + pushed = await _ids(hybrid, ctx) + verified = await _verified_ids(hybrid, ctx) + assert verified == pushed == _cleared(labels) + + +async def test_verifier_never_bypasses_tenant_scoping() -> None: + hybrid = await _retriever_without_policy() + # A globex principal (even with a matching label) only ever sees globex data, + # and never acme's 'eng' chunk — tenant scoping is upstream of the ACL check. + assert await _verified_ids(hybrid, _ctx(["eng"], tenant="globex")) == {"g-eng"}