From 712c53d43d1baf9394c4fa46698afe2e6cee5622 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Mon, 8 Jun 2026 01:02:49 +0530 Subject: [PATCH] =?UTF-8?q?feat(policy):=20ACL=20push-down=20at=20retrieva?= =?UTF-8?q?l=20=E2=80=94=20AclPolicyEngine=20(Step=206.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enforce label-based ACLs at retrieval. Step 6.1 resolved each tenant's acl_labels onto the principal, but nothing enforced them — filter_pushdown scoped tenant_id only, so any principal in a tenant could read every chunk. New AclPolicyEngine (a decorator like QuotaPolicyEngine) And-merges any_in("acl_labels", sorted(principal.acl_labels)) into every read_chunk push-down: a chunk is retrievable only when its labels overlap the principal's. - Overlap semantics via the existing AnyIn predicate — the semantics the codebase already assumed and every backend translator already speaks (pgvector &&, Qdrant MatchAny, Cypher ANY(), noop evaluate), so zero new predicate / translator work. Injected at the canonical HybridRetriever read_chunk PDP site (no new call site). - Fail-closed: any_in([]) matches nothing, so a label-less principal retrieves nothing (model "public" as a shared label granted to all); each label-less request emits one PII-free acl.egress_denied event (EVT_ACL_DENIED). - Opt-in via cfg.acl.enabled (default off → pre-6.3 behaviour); build_app (acl_enabled=...) wraps the engine, build_app_from_config from config. Decorates without absorbing the inner engine. Scope is push-down enforcement only — post-retrieval egress re-verification is 6.4, graph edge ACLs deferred. ~17 new tests incl. an end-to-end ACL red-team through HybridRetriever. All gates green (ruff, mypy --strict 294 files, RAG001, schema-drift, policy-coverage, log-schema). Co-Authored-By: Claude Opus 4.7 --- TRACKER.md | 25 +++-- apps/gateway/src/rag_gateway/app.py | 10 ++ apps/gateway/src/rag_gateway/wiring.py | 4 + apps/gateway/tests/test_acl.py | 39 ++++++++ dist/rag.schema.json | 16 ++++ dist/rag.schema.yaml | 32 +++++++ docs/README.md | 1 + docs/adr/ADR-0035-acl-pushdown.md | 73 ++++++++++++++ docs/architecture/policy-engine.md | 7 ++ docs/reference/tenancy.md | 2 +- packages/config/src/rag_config/__init__.py | 3 + packages/config/src/rag_config/schema.py | 19 ++++ packages/policy/src/rag_policy/__init__.py | 2 + packages/policy/src/rag_policy/acl.py | 89 +++++++++++++++++ tests/policy/test_acl_engine.py | 104 ++++++++++++++++++++ tests/redteam/test_acl_isolation.py | 106 +++++++++++++++++++++ 16 files changed, 524 insertions(+), 8 deletions(-) create mode 100644 apps/gateway/tests/test_acl.py create mode 100644 docs/adr/ADR-0035-acl-pushdown.md create mode 100644 packages/policy/src/rag_policy/acl.py create mode 100644 tests/policy/test_acl_engine.py create mode 100644 tests/redteam/test_acl_isolation.py diff --git a/TRACKER.md b/TRACKER.md index ab24740..016cc9f 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -14,12 +14,13 @@ | | | |---|---| | **Last updated** | 2026-06-08 | -| **Current phase** | Phase 6 — Governance & Tenancy (**2 / 10 steps**) | -| **Overall** | **66 / 84 steps** — Phases 0–5 complete | -| **Next action** | **Step 6.3 — ACL push-down at retrieval**: inject the ACL filter into every vector / BM25 / graph query; `acl.egress_denied` event. Consumes the per-tenant `acl_labels` resolved in 6.1. | +| **Current phase** | Phase 6 — Governance & Tenancy (**3 / 10 steps**) | +| **Overall** | **67 / 84 steps** — Phases 0–5 complete | +| **Next action** | **Step 6.4 — ACL egress verifier**: post-retrieval re-check of returned chunks against the principal's ACLs (defense-in-depth); zero-ACL-violation-rate gate. Backstops the 6.3 push-down. | **Recently shipped** +- **6.3** ✅ ACL push-down — opt-in `AclPolicyEngine` And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` query (overlap, fail-closed); `acl.egress_denied` event — [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) - **6.2** ✅ Physical tenancy — per-tenant *dedicated* vector index/collection; `TenantConfig.dedicated_index` → `ctx.physical_index` → backend `-` (Noop/Pinecone/Qdrant); cross-tenant probe gate — [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) - **6.1** ✅ Logical multi-tenancy — `TenantResolver` resolves per-tenant `rag.yaml` config (namespace / PII / ACL labels) → `TenantSettings` applied at the gateway boundary; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant` — [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) - **5.7d** ✅ Experiments console card + Phase-5 close-out — admin Live-Status A/B card over `GET /v1/status/experiments` (lift + CI per experiment) — [#147](https://github.com/officialCodeWork/AgentContextOS/pull/147) @@ -56,9 +57,9 @@ | 3 | Gateway & Agent Runtime | 11 | **11** | 0 | | 4 | Reliability | 6 | **6** | 0 | | 5 | Eval & Observability | 7 | **7** | 0 | -| 6 | Governance & Tenancy | 10 | **2** | 8 | +| 6 | Governance & Tenancy | 10 | **3** | 7 | | 7 | Pilot, Harden, GA | 10 | 0 | 10 | -| **Total** | | **84** | **66** | **18** | +| **Total** | | **84** | **67** | **17** | --- @@ -641,13 +642,13 @@ - **Phase-5 close-out:** Step 5.7 ✅ → **Phase 5 complete (7 / 7)**; deferred items remain documented (per-tenant drift / per-dimension embedding PSI; feedback/breaker/quota Grafana export + Loki-events dashboard; gRPC proto mirror of `corpus_decision` + `experiment`; sequential / multi-metric experiments) - [reference/experiments.md](docs/reference/experiments.md), [reference/admin-ui.md](docs/reference/admin-ui.md) -## Phase 6 — Governance & Tenancy (Weeks 28–34) 🚧 (2 / 10) +## Phase 6 — Governance & Tenancy (Weeks 28–34) 🚧 (3 / 10) | Step | Title | Status | Planned deliverables | |------|-------|:------:|----------------------| | 6.1 | Logical multi-tenancy | ✅ | [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) — `TenantResolver` → `TenantSettings` (namespace / PII / ACL labels) applied at the boundary; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant` | | 6.2 | Physical tenancy (dedicated index) | ✅ | [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) — `dedicated_index` → `ctx.physical_index` → backend `-` (Noop/Pinecone/Qdrant, lazy create); cross-tenant probe gate | -| 6.3 | ACL push-down at retrieval | ⏳ | ACL filter injected into every vector / BM25 / graph query; `acl.egress_denied` event | +| 6.3 | ACL push-down at retrieval | ✅ | [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) — opt-in `AclPolicyEngine` And-merges `any_in("acl_labels", …)` into every `read_chunk` push-down (overlap, fail-closed); `acl.egress_denied` | | 6.4 | ACL egress verifier | ⏳ | Post-retrieval re-check; defense-in-depth; zero-ACL-violation-rate gate | | 6.5 | PII policies | ⏳ | Per-tenant PII enforcement (block / redact / allow); egress redaction; `pii.egress_blocked` event | | 6.6 | Immutable audit log | ⏳ | Hash-chain audit log; WORM export; tamper-evident verification; `GET /v1/audit` | @@ -675,6 +676,15 @@ - One instance + per-tenant derivation (**no** per-tenant backend instances, **no** SPI change); `GET /v1/status/tenant` + `ragctl tenant resolve` report `dedicated_index` / `physical_index`; `TenantConfig` fields → `rag.schema`, `TenantSettings` + `RequestContext` regenerated, `/v1/status/tenant` → `dist/openapi`; ~30 new/updated tests; all gates green - [ADR-0034](docs/adr/ADR-0034-physical-multi-tenancy.md), [reference/tenancy.md](docs/reference/tenancy.md#physical-tenancy-step-62), [architecture/multi-tenancy.md](docs/architecture/multi-tenancy.md) +### 6.3 — ACL push-down at retrieval ✅ [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) + +- Enforces **label-based ACLs** at retrieval: 6.1 resolved each tenant's `acl_labels` onto the principal, but nothing enforced them. New **`AclPolicyEngine`** (`rag-policy`) — a decorator like `QuotaPolicyEngine` — And-merges **`any_in("acl_labels", sorted(principal.acl_labels))`** into every `read_chunk` push-down; a chunk is retrievable only when its labels **overlap** the principal's +- **Overlap via the existing `AnyIn` predicate** — the semantics the codebase already assumed and every backend translator already speaks (pgvector `&&`, Qdrant `MatchAny`, Cypher `ANY()`, noop `evaluate`), so **zero new predicate / translator changes**. Injected at the canonical `HybridRetriever` `read_chunk` PDP site (already merges `filter_pushdown`), so no new call site / coverage-linter entry +- **Fail-closed**: the clause is uniform — `any_in([])` matches nothing, so a label-less principal retrieves nothing (model "public" as a shared label granted to all); each label-less request emits one PII-free **`acl.egress_denied`** event (pre-registered `EVT_ACL_DENIED`) +- **Opt-in** via new `cfg.acl.enabled` (default off → pre-6.3 behaviour, tenant scoping only); `build_app(acl_enabled=…)` wraps the engine, `build_app_from_config` from config. Decorates without absorbing the inner engine (composes with a production PDP) +- `AclConfig` → `rag.schema`; ~17 new tests incl. an **end-to-end ACL red-team** through `HybridRetriever` (overlap / disjoint / fail-closed / public-label / no-tenant-bypass) + engine unit + event + wiring; all gates green (ruff, mypy --strict, RAG001, schema-drift, policy-coverage, log-schema) +- **Scope:** push-down enforcement only — post-retrieval egress re-verification is 6.4, graph **edge** ACLs deferred. [ADR-0035](docs/adr/ADR-0035-acl-pushdown.md), [architecture/policy-engine.md](docs/architecture/policy-engine.md), [reference/tenancy.md](docs/reference/tenancy.md) + --- ## Phase 7 — Pilot, Harden, GA (Weeks 34–40) ⏳ @@ -828,6 +838,7 @@ Complete log of every PR. Routine Dependabot bumps are grouped; everything else | [#147](https://github.com/officialCodeWork/AgentContextOS/pull/147) | 2026-06-07 | feat(admin-ui): A/B experiments console card + Phase-5 close-out (Step 5.7d) | | [#148](https://github.com/officialCodeWork/AgentContextOS/pull/148) | 2026-06-08 | feat(tenancy): logical multi-tenancy — per-tenant config resolution (Step 6.1) | | [#149](https://github.com/officialCodeWork/AgentContextOS/pull/149) | 2026-06-08 | feat(tenancy): physical tenancy — dedicated index per tenant (Step 6.2) | +| [#150](https://github.com/officialCodeWork/AgentContextOS/pull/150) | 2026-06-08 | feat(policy): ACL push-down at retrieval — AclPolicyEngine (Step 6.3) | | #78–#80, #116–#118 | Open | Dependabot bumps — awaiting merge | | #81 | Closed | Dependabot bump — superseded | diff --git a/apps/gateway/src/rag_gateway/app.py b/apps/gateway/src/rag_gateway/app.py index 5b6b559..3746b38 100644 --- a/apps/gateway/src/rag_gateway/app.py +++ b/apps/gateway/src/rag_gateway/app.py @@ -368,6 +368,7 @@ def build_app( shadow_runner: Any | None = None, ab_router: Any | None = None, tenant_resolver: Any | None = None, + acl_enabled: bool = False, enable_cors: bool = True, default_tenant_id: TenantId | None = None, ) -> FastAPI: @@ -492,7 +493,16 @@ def build_app( # quotas and the entry checks resolve to the inner engine's allow. if quota_enforcer is not None: policy_engine = QuotaPolicyEngine(quota_enforcer, inner=policy_engine) + # Label-based ACL push-down (Step 6.3) — when enabled, decorate the engine so + # every ``read_chunk`` filter is And-merged with an ``acl_labels`` overlap + # clause; fail-closed (a label-less principal retrieves nothing). Opt-in, so + # the default keeps the pre-6.3 behaviour (tenant scoping only). + if acl_enabled: + from rag_policy import AclPolicyEngine + + policy_engine = AclPolicyEngine(inner=policy_engine) app.state.policy_engine = policy_engine + app.state.acl_enabled = acl_enabled app.state.quota_enforcer = quota_enforcer # Hallucination guard (Step 4.3) — post-generation faithfulness check over diff --git a/apps/gateway/src/rag_gateway/wiring.py b/apps/gateway/src/rag_gateway/wiring.py index ff4cbe1..442e0fb 100644 --- a/apps/gateway/src/rag_gateway/wiring.py +++ b/apps/gateway/src/rag_gateway/wiring.py @@ -702,6 +702,9 @@ def build_app_from_config(cfg: RagConfig, **overrides: Any) -> FastAPI: # process (the instrument is global, the callback reads current state). _register_platform_metrics_once(overrides.get("drift_registry"), overrides.get("cost_tracker")) + # Label-based ACL push-down (Step 6.3) — opt-in; off by default. + acl_enabled = overrides.pop("acl_enabled", cfg.acl.enabled) + return build_app( corpus_store=corpus_store, retrieval_router=retrieval_router, @@ -711,6 +714,7 @@ def build_app_from_config(cfg: RagConfig, **overrides: Any) -> FastAPI: guard_enabled=guard_enabled, breaker_registry=breaker_registry, quota_enforcer=quota_enforcer, + acl_enabled=acl_enabled, **overrides, ) diff --git a/apps/gateway/tests/test_acl.py b/apps/gateway/tests/test_acl.py new file mode 100644 index 0000000..01d7c8a --- /dev/null +++ b/apps/gateway/tests/test_acl.py @@ -0,0 +1,39 @@ +"""ACL push-down config + gateway wiring (Step 6.3).""" + +from __future__ import annotations + +from rag_config import RagConfig +from rag_config.schema import AclConfig +from rag_gateway import build_app +from rag_gateway.wiring import build_app_from_config +from rag_policy import AclPolicyEngine, NoopPolicyEngine + + +def test_acl_config_defaults_off() -> None: + assert RagConfig().acl.enabled is False + + +def test_inert_by_default() -> None: + app = build_app() + assert app.state.acl_enabled is False + assert not isinstance(app.state.policy_engine, AclPolicyEngine) + + +def test_build_from_config_wraps_when_enabled() -> None: + app = build_app_from_config(RagConfig(acl=AclConfig(enabled=True))) + assert app.state.acl_enabled is True + assert isinstance(app.state.policy_engine, AclPolicyEngine) + + +def test_build_from_config_inert_when_disabled() -> None: + app = build_app_from_config(RagConfig()) + assert not isinstance(app.state.policy_engine, AclPolicyEngine) + + +def test_acl_decorates_without_absorbing_inner() -> None: + app = build_app_from_config(RagConfig(acl=AclConfig(enabled=True))) + engine = app.state.policy_engine + assert isinstance(engine, AclPolicyEngine) + # the inner engine is preserved (here the default Noop), so ACL composes with + # — rather than replaces — the production PDP. + assert isinstance(engine.inner, NoopPolicyEngine) diff --git a/dist/rag.schema.json b/dist/rag.schema.json index 98896e3..3ebc3b9 100644 --- a/dist/rag.schema.json +++ b/dist/rag.schema.json @@ -1,5 +1,18 @@ { "$defs": { + "AclConfig": { + "additionalProperties": false, + "description": "Label-based ACL push-down enforcement (Step 6.3).\n\nWhen ``enabled`` the gateway wraps its PolicyEngine in an ``AclPolicyEngine``\nthat injects ``any_in(\"acl_labels\", principal.acl_labels)`` into every\n``read_chunk`` push-down \u2014 a chunk is retrievable only when its labels overlap\nthe principal's (the labels resolved per tenant in Step 6.1).\n\n**Disabled by default** \u2014 turning it on changes which chunks a principal can\nretrieve, and is **fail-closed**: a principal carrying no ACL labels matches\nno labeled chunk, so it retrieves nothing (model \"public\" data as a shared\nlabel granted to all principals). Each label-less request emits a PII-free\n``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4.", + "properties": { + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + } + }, + "title": "AclConfig", + "type": "object" + }, "AuthConfig": { "additionalProperties": false, "properties": { @@ -1600,6 +1613,9 @@ "quotas": { "$ref": "#/$defs/QuotaConfig" }, + "acl": { + "$ref": "#/$defs/AclConfig" + }, "webhooks": { "$ref": "#/$defs/WebhooksConfig" }, diff --git a/dist/rag.schema.yaml b/dist/rag.schema.yaml index b3ace92..0c1c2cf 100644 --- a/dist/rag.schema.yaml +++ b/dist/rag.schema.yaml @@ -1,4 +1,34 @@ $defs: + AclConfig: + additionalProperties: false + description: 'Label-based ACL push-down enforcement (Step 6.3). + + + When ``enabled`` the gateway wraps its PolicyEngine in an ``AclPolicyEngine`` + + that injects ``any_in("acl_labels", principal.acl_labels)`` into every + + ``read_chunk`` push-down — a chunk is retrievable only when its labels overlap + + the principal''s (the labels resolved per tenant in Step 6.1). + + + **Disabled by default** — turning it on changes which chunks a principal can + + retrieve, and is **fail-closed**: a principal carrying no ACL labels matches + + no labeled chunk, so it retrieves nothing (model "public" data as a shared + + label granted to all principals). Each label-less request emits a PII-free + + ``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4.' + properties: + enabled: + default: false + title: Enabled + type: boolean + title: AclConfig + type: object AuthConfig: additionalProperties: false properties: @@ -1444,6 +1474,8 @@ properties: $ref: '#/$defs/BreakerConfig' quotas: $ref: '#/$defs/QuotaConfig' + acl: + $ref: '#/$defs/AclConfig' webhooks: $ref: '#/$defs/WebhooksConfig' provenance: diff --git a/docs/README.md b/docs/README.md index e37f465..fae6339 100644 --- a/docs/README.md +++ b/docs/README.md @@ -173,6 +173,7 @@ broken, and what to fix before committing to the next phase. | [ADR-0032-ab-testing-shadow-mode.md](adr/ADR-0032-ab-testing-shadow-mode.md) | Decision (Step 5.7): compare two configs on live traffic, delivered in slices (5.7a analyzer+tracker+surface, 5.7b shadow, 5.7c routing, 5.7d console); the analyzer is pure + stdlib-only in `rag_config.eval` (normal-approx Welch via `statistics.NormalDist`, no numpy/scipy — same spirit as drift PSI / cost z-score); the `ABExperimentTracker` is a pure *sample holder* in `rag_observability` (so it doesn't import `rag_config`), and the gateway composes the two for `GET /v1/status/experiments`; opt-in by default (A/B routing can change responses); `ABAnalysisResult` additive in `rag_core.eval` (not in `dist/schemas`); rejected numpy/scipy, a new package, putting the analyzer in observability, defaulting on | | [ADR-0033-logical-multi-tenancy.md](adr/ADR-0033-logical-multi-tenancy.md) | Decision (Step 6.1): make per-tenant `rag.yaml` config drive requests. `TenantResolver` (`rag_config`) maps a tenant id → frozen `TenantSettings` (`rag_core`), applied once at the gateway boundary; resolver in config / settings type in core keeps the `config → core` direction; unknown tenants → safe defaults (isolated not privileged); `RequestContext.namespace` defaults to `tenant_id` (a backend-partition primitive — Pinecone uses it — not a chunk field, so `filter_pushdown` is unchanged); scope stops at resolution + threading (ACL push-down 6.3, PII egress 6.5, physical tenancy 6.2); additive + inert in `build_app` | | [ADR-0034-physical-multi-tenancy.md](adr/ADR-0034-physical-multi-tenancy.md) | Decision (Step 6.2): a *dedicated* vector index/collection per tenant. `TenantConfig.dedicated_index` resolves to a `physical_index` key on `TenantSettings`, threaded onto `RequestContext.physical_index`; backends read only `ctx` (graph is backends→core, never rag-config) and namespace their base under it (`-`), lazily creating it; one instance + per-tenant derivation (no per-tenant instances, no SPI change); Noop is the CI conformance oracle (keyed by `physical_index`) for a cross-tenant probe gate that proves isolation independent of the tenant filter; Noop + Pinecone + Qdrant this step, others later | +| [ADR-0035-acl-pushdown.md](adr/ADR-0035-acl-pushdown.md) | Decision (Step 6.3): label-based ACL push-down at retrieval. `AclPolicyEngine` (a decorator like `QuotaPolicyEngine`) And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` push-down at the canonical `HybridRetriever` PDP site — overlap semantics via the existing `AnyIn` predicate (zero backend/translator changes), **fail-closed** (label-less principal matches nothing; "public" = a shared label), **opt-in** via `cfg.acl.enabled`; emits `acl.egress_denied` on a request-level denial; graph edge ACLs + post-retrieval re-verification (6.4) deferred | | [ADR-0031-cost-anomaly.md](adr/ADR-0031-cost-anomaly.md) | Decision (Step 5.6c): detect per-tenant spend spikes with a rolling `CostTracker` (not the cumulative quota counter); detect scale-free on the token series (cost = tokens × a constant price) so detection is decoupled from quota pricing and works with quotas off; two gates (ratio + z-score, z relaxed on a flat baseline) → tri-state verdict; put it in `rag-observability` as a `dataclass` (gateway wraps it in a Pydantic `CostStatusResponse`) so there's **no `rag-core` type / `dist/schemas` churn**; feed O(1) from `record_request_usage` before the quota block; pull-based `GET /v1/status/cost` (no per-request span/event); rejected folding into the infra-scoped drift registry, a new package, a `cost.anomaly_detected` push event (deferred), per-model pricing, a time-series DB | | [ADR-0030-drift-monitors.md](adr/ADR-0030-drift-monitors.md) | Decision (Step 5.5): detect retrieval degradation with five drift monitors in a new `rag-drift` package (mirroring rag-feedback); two statistics — PSI (pure, binned, dependency-free) for the distribution monitors + mean-drop for the rate/score monitors — over one scalar-window `DriftMonitor`; infra-scoped registry (like breakers) fed via `observe` from the signals the gateway already computes (query length / retrieval score / HyDE-embedding norm / guard grounded-claim fraction / feedback citation clicks); detection on dashboard-poll `evaluate()` with transition-edge `drift.detected` (structured event + the Step 3.9 webhook, targeting `alert_tenant`); observe-only / inert-by-default / rebaseline; rejected per-tenant monitors, per-dimension embedding PSI, a stats library, a background scheduler, hot-path detection | | [ADR-0029-online-feedback.md](adr/ADR-0029-online-feedback.md) | Decision (Step 5.4): capture online feedback + implicit signals in a new `rag-feedback` package mirroring `rag-provenance` (SPI + types in rag-core; recorder + pure aggregator in the package); one polymorphic `POST /v1/feedback` (a `signal` enum spanning explicit thumbs/rating/comment + implicit citation-click/copy/regenerate/dwell, `kind` inferred); normalise every signal to a `[-1,1]` score so the dashboard has one satisfaction number; **redact-don't-hash** free-text comments via an injected `PIIDetector` (default `NoopPIIDetector` seam, `comment_redacted` flag, PII-free event) — opposite of provenance's hashing; body identity like `/v1/query`; degrade-open + inert-by-default; `GET /v1/status/feedback` dashboard (event-only, no per-call span); admin-UI card deferred to 5.6; rejected separate per-signal endpoints, header-auth, hashing/raw comments, an OTel span per submission, folding into provenance | diff --git a/docs/adr/ADR-0035-acl-pushdown.md b/docs/adr/ADR-0035-acl-pushdown.md new file mode 100644 index 0000000..bec1f3d --- /dev/null +++ b/docs/adr/ADR-0035-acl-pushdown.md @@ -0,0 +1,73 @@ +# ADR-0035 — ACL push-down at retrieval + +**Status:** Accepted +**Date:** 2026-06-08 +**Step:** 6.3 — ACL push-down at retrieval (Phase 6 — Governance & Tenancy) +**Related:** [ADR-0005](ADR-0005-policy-engine.md) (PolicyEngine PDP), [ADR-0033](ADR-0033-logical-multi-tenancy.md) (logical tenancy), [architecture/policy-engine.md](../architecture/policy-engine.md), [reference/tenancy.md](../reference/tenancy.md) + +## Context + +Step 6.1 resolved each tenant's `acl_labels` onto `RequestContext.principal.acl_labels`, +but nothing enforced them — `PolicyEngine.filter_pushdown` scoped only `tenant_id`, +so any principal in a tenant could retrieve every chunk. Step 6.3 enforces +*label-based ACLs* at retrieval: a principal should only retrieve chunks whose +`acl_labels` they are cleared for. + +## Decision + +**1. Overlap semantics via the existing `AnyIn` predicate.** A chunk is visible +when `chunk.acl_labels ∩ principal.acl_labels ≠ ∅`, expressed as +`any_in("acl_labels", sorted(principal.acl_labels))`. This is the semantics the +codebase already assumed (the `FilterExpr` docstring, the policy-engine extension +example, the filter tests), and **every backend translator already speaks it** +(pgvector `&&`, Qdrant `MatchAny`, Cypher `ANY()`, the noop `evaluate`). So 6.3 +needs **no new predicate and no translator changes**. + +**2. A decorator engine — `AclPolicyEngine`.** Mirroring `QuotaPolicyEngine` +(Step 4.5), a thin decorator wraps an inner `PolicyEngine` and And-merges the ACL +clause into the `read_chunk` push-down; everything else delegates. It composes +with any production PDP rather than absorbing it. The canonical injection site is +unchanged — `HybridRetriever` already merges `filter_pushdown` into every +vector / keyword / graph query — so no new PDP call site and no coverage-linter +entry. + +**3. Fail-closed, with "public" as a shared label.** The clause is uniform: +`any_in("acl_labels", sorted(principal.acl_labels))` matches **nothing** when the +principal has no labels, so an unconfigured principal retrieves nothing rather +than everything. There is no separate "empty labels = public" branch (which would +need an emptiness predicate); public data is modeled as a shared label (e.g. +`public`) granted to all principals. + +**4. Opt-in.** Gated by `cfg.acl.enabled` (default **false**), because turning it +on changes which chunks a principal can retrieve and would otherwise break +deployments whose principals/chunks carry no labels. When off, the pre-6.3 +behaviour (tenant scoping only) is unchanged. + +**5. `acl.egress_denied` on request-level denial.** The pre-registered event fires +once, PII-free (`tenant_id`, `principal_id`, `decision`, `reason`), when a +label-less principal is denied under enforcement — a real, alertable +misconfiguration signal. Per-chunk narrowing by the filter is the normal path and +stays silent. (A full post-retrieval re-verification is the Step 6.4 egress +verifier.) + +## Consequences + +**Positive** +- ACLs are enforced at the source (push-down), so denied chunks never leave the + backend; the existing `AnyIn` support means zero backend work. +- Fail-closed by construction; opt-in keeps existing deployments unchanged. +- A decorator keeps the one-PDP invariant and composes with a production engine. + +**Negative / deferred** +- Overlap (not subset) semantics: a principal needs only *one* matching label, not + all of a chunk's labels. Subset/"all-of" semantics would need a new predicate; + not required by the model here. +- Graph **edge** ACLs (per-hop `edge_filter`) are deferred; only `node_filter` + carries the ACL clause today. +- Push-down can't *count* denied chunks (they're filtered at the source), so the + event is request-level, not per-chunk; thorough post-retrieval re-checking is + Step 6.4. + +## See also +- [architecture/policy-engine.md](../architecture/policy-engine.md) — PDP + push-down +- [reference/tenancy.md](../reference/tenancy.md) — ACL labels + enforcement diff --git a/docs/architecture/policy-engine.md b/docs/architecture/policy-engine.md index ff6cbcf..edab6d9 100644 --- a/docs/architecture/policy-engine.md +++ b/docs/architecture/policy-engine.md @@ -97,6 +97,13 @@ class MyOrgPolicyEngine(PolicyEngine): ) ``` +> **Built-in ACL push-down (Step 6.3).** This `acl_labels` overlap clause now ships +> as `rag_policy.AclPolicyEngine` — a decorator that And-merges it into every +> `read_chunk` push-down, **opt-in** via `cfg.acl.enabled` and **fail-closed** +> (a label-less principal matches nothing). It composes with any inner engine, so a +> custom PDP only needs to override `read_chunk` semantics if it wants more than +> label overlap. See [ADR-0035](../adr/ADR-0035-acl-pushdown.md). + Register at the composition root (`apps/gateway/`): ```python diff --git a/docs/reference/tenancy.md b/docs/reference/tenancy.md index 3506395..34d1a25 100644 --- a/docs/reference/tenancy.md +++ b/docs/reference/tenancy.md @@ -97,7 +97,7 @@ per-tenant isolation + governance primitives; the enforcement that builds on the lands in later Phase-6 steps: - **Physical tenancy** (a dedicated index per tenant) — **6.2** (below). -- **ACL push-down** at retrieval (inject the labels into every backend query) — **6.3**. +- **ACL push-down** at retrieval (inject the labels into every backend query) — **6.3** ✅: enable `cfg.acl.enabled` to wrap the PolicyEngine in an `AclPolicyEngine` that And-merges `any_in("acl_labels", principal.acl_labels)` into every `read_chunk` push-down. **Fail-closed** (a label-less principal retrieves nothing; model "public" as a shared label) and emits `acl.egress_denied`. See [ADR-0035](../adr/ADR-0035-acl-pushdown.md). - **PII egress** enforcement (block / redact answers per policy) — **6.5**. ## Physical tenancy (Step 6.2) diff --git a/packages/config/src/rag_config/__init__.py b/packages/config/src/rag_config/__init__.py index a4f25af..1937732 100644 --- a/packages/config/src/rag_config/__init__.py +++ b/packages/config/src/rag_config/__init__.py @@ -7,6 +7,7 @@ from rag_config.loader import load, loads, watch from rag_config.schema import ( + AclConfig, AuthConfig, AuthProvider, BackendsConfig, @@ -123,6 +124,8 @@ "BreakerConfig", # quotas & rate limiting (Step 4.5) "QuotaConfig", + # ACL push-down (Step 6.3) + "AclConfig", # provenance & per-query tracing (Step 5.1) "ProvenanceConfig", # online feedback & implicit signals (Step 5.4) diff --git a/packages/config/src/rag_config/schema.py b/packages/config/src/rag_config/schema.py index 6ef0ebf..94b3a51 100644 --- a/packages/config/src/rag_config/schema.py +++ b/packages/config/src/rag_config/schema.py @@ -535,6 +535,24 @@ class BreakerConfig(_StrictBase): # --------------------------------------------------------------------------- +class AclConfig(_StrictBase): + """Label-based ACL push-down enforcement (Step 6.3). + + When ``enabled`` the gateway wraps its PolicyEngine in an ``AclPolicyEngine`` + that injects ``any_in("acl_labels", principal.acl_labels)`` into every + ``read_chunk`` push-down — a chunk is retrievable only when its labels overlap + the principal's (the labels resolved per tenant in Step 6.1). + + **Disabled by default** — turning it on changes which chunks a principal can + retrieve, and is **fail-closed**: a principal carrying no ACL labels matches + no labeled chunk, so it retrieves nothing (model "public" data as a shared + label granted to all principals). Each label-less request emits a PII-free + ``acl.egress_denied`` event. ACL *egress* re-verification is Step 6.4. + """ + + enabled: bool = False + + class QuotaConfig(_StrictBase): """Per-tenant quota & rate-limit enforcement knobs (Step 4.5). @@ -845,6 +863,7 @@ class RagConfig(_StrictBase): guard: GuardConfig = Field(default_factory=GuardConfig) breakers: BreakerConfig = Field(default_factory=BreakerConfig) quotas: QuotaConfig = Field(default_factory=QuotaConfig) + acl: AclConfig = Field(default_factory=AclConfig) webhooks: WebhooksConfig = Field(default_factory=WebhooksConfig) provenance: ProvenanceConfig = Field(default_factory=ProvenanceConfig) feedback: FeedbackConfig = Field(default_factory=FeedbackConfig) diff --git a/packages/policy/src/rag_policy/__init__.py b/packages/policy/src/rag_policy/__init__.py index 455101a..fec8d32 100644 --- a/packages/policy/src/rag_policy/__init__.py +++ b/packages/policy/src/rag_policy/__init__.py @@ -5,6 +5,7 @@ and [ADR-0005](../../../docs/adr/ADR-0005-policy-engine.md). """ +from rag_policy.acl import AclPolicyEngine from rag_policy.engine import PolicyEngine from rag_policy.filter import ( And, @@ -37,6 +38,7 @@ # engine "PolicyEngine", "NoopPolicyEngine", + "AclPolicyEngine", "PolicyWriter", # decisions + results "PolicyDecision", diff --git a/packages/policy/src/rag_policy/acl.py b/packages/policy/src/rag_policy/acl.py new file mode 100644 index 0000000..1b96fe0 --- /dev/null +++ b/packages/policy/src/rag_policy/acl.py @@ -0,0 +1,89 @@ +"""AclPolicyEngine — ACL push-down at retrieval (Step 6.3). + +Per [ADR-0005](../../../docs/adr/ADR-0005-policy-engine.md) every governance +concern is answered by the single :class:`~rag_policy.engine.PolicyEngine` PDP. +This engine is a thin **decorator** that enforces *label-based ACLs* by injecting +a filter into the ``read_chunk`` push-down: a chunk is visible only when its +``acl_labels`` overlap the principal's (the ``AnyIn`` overlap semantics every +backend translator already speaks). Everything else delegates to the wrapped +inner engine, so it composes with any production PDP. + +Fail-closed by construction: the clause is ``any_in("acl_labels", sorted( +principal.acl_labels))``, which matches **nothing** when the principal carries no +labels — so an unconfigured principal sees nothing rather than everything. Model +"public" data as a shared label granted to all principals (no special predicate). +A label-less principal under enforcement is a definite denial, so one PII-free +``acl.egress_denied`` event is emitted per such request; per-chunk narrowing by +the filter is the normal path and stays silent. + +Opt-in: only wired when ``cfg.acl.enabled`` — the default keeps the pre-6.3 +behaviour (tenant scoping only). ACL *egress* re-verification is Step 6.4. +""" + +from __future__ import annotations + +from typing import Any + +from rag_core.events import EVT_ACL_DENIED +from rag_core.types import RequestContext +from rag_observability.logging import get_logger + +from rag_policy.engine import PolicyEngine +from rag_policy.filter import FilterExpr, and_, any_in +from rag_policy.types import PolicyDecision, PolicyResult + +_log = get_logger(__name__) + +__all__ = ["AclPolicyEngine"] + + +class AclPolicyEngine(PolicyEngine): + """Decorate an inner :class:`PolicyEngine`, enforcing label-based ACLs (Step 6.3).""" + + def __init__(self, *, inner: PolicyEngine) -> None: + self._inner = inner + + @property + def inner(self) -> PolicyEngine: + return self._inner + + async def evaluate( + self, + ctx: RequestContext, + decision: PolicyDecision, + subject: Any, + ) -> PolicyResult: + return await self._inner.evaluate(ctx, decision, subject) + + async def filter_pushdown( + self, + ctx: RequestContext, + decision: PolicyDecision, + ) -> FilterExpr: + base = await self._inner.filter_pushdown(ctx, decision) + if decision is not PolicyDecision.read_chunk: + return base + labels = sorted(ctx.principal.acl_labels) + if not labels: + # Fail-closed: no clearance → the overlap clause matches nothing, so + # this principal is denied every chunk. Flag it once (PII-free). + self._emit_denied(ctx) + # ``any_in([])`` is false for every chunk, so the empty-labels case needs + # no special filter — the same clause is the fail-closed deny. + return and_(base, any_in("acl_labels", labels)) + + async def health(self) -> bool: + return await self._inner.health() + + @staticmethod + def _emit_denied(ctx: RequestContext) -> None: + _log.warning( + EVT_ACL_DENIED, + extra={ + "event_kind": EVT_ACL_DENIED, + "tenant_id": str(ctx.tenant_id), + "principal_id": str(ctx.principal.id), + "decision": PolicyDecision.read_chunk.value, + "reason": "no_acl_clearance", + }, + ) diff --git a/tests/policy/test_acl_engine.py b/tests/policy/test_acl_engine.py new file mode 100644 index 0000000..2f4cb62 --- /dev/null +++ b/tests/policy/test_acl_engine.py @@ -0,0 +1,104 @@ +"""Unit tests for AclPolicyEngine — ACL push-down decorator (Step 6.3).""" + +from __future__ import annotations + +from typing import Any + +import pytest +from rag_core.types import ( + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) +from rag_policy import AclPolicyEngine, NoopPolicyEngine, PolicyDecision, evaluate + + +def _ctx(labels: list[str], *, tenant: str = "acme") -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), + kind=PrincipalKind.user, + display_name="p", + tenant_id=tid, + acl_labels=frozenset(labels), + ), + ) + + +def _engine() -> AclPolicyEngine: + return AclPolicyEngine(inner=NoopPolicyEngine()) + + +async def _filter(labels: list[str], decision: PolicyDecision) -> Any: + return await _engine().filter_pushdown(_ctx(labels), decision) + + +# --------------------------------------------------------------------------- +# filter_pushdown — overlap semantics +# --------------------------------------------------------------------------- +async def test_read_chunk_filter_enforces_label_overlap() -> None: + f = await _filter(["eng", "fin"], PolicyDecision.read_chunk) + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset(["eng"])}) is True + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset(["hr"])}) is False + # an unlabeled chunk overlaps nobody → hidden + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset()}) is False + + +async def test_label_less_principal_is_fail_closed() -> None: + f = await _filter([], PolicyDecision.read_chunk) + # the any_in([]) clause matches no chunk, labeled or not + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset(["eng"])}) is False + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset()}) is False + + +async def test_tenant_scoping_is_preserved() -> None: + f = await _filter(["eng"], PolicyDecision.read_chunk) + assert evaluate(f, {"tenant_id": "other", "acl_labels": frozenset(["eng"])}) is False + + +async def test_non_read_chunk_decisions_delegate_without_acl_clause() -> None: + # ingest/egress push-downs keep the inner (tenant-only) filter — no ACL clause. + for decision in (PolicyDecision.ingest_doc, PolicyDecision.egress_text): + f = await _filter([], decision) + assert evaluate(f, {"tenant_id": "acme", "acl_labels": frozenset()}) is True + + +async def test_health_delegates() -> None: + assert await _engine().health() is True + + +# --------------------------------------------------------------------------- +# acl.egress_denied event +# --------------------------------------------------------------------------- +async def test_label_less_read_chunk_emits_acl_egress_denied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import rag_policy.acl as acl_mod + + calls: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr(acl_mod._log, "warning", lambda msg, **kw: calls.append((msg, kw))) + + await _filter([], PolicyDecision.read_chunk) + + events = [kw for msg, kw in calls if msg == "acl.egress_denied"] + assert len(events) == 1 + extra = events[0]["extra"] + assert extra["event_kind"] == "acl.egress_denied" + assert extra["tenant_id"] == "acme" + assert extra["reason"] == "no_acl_clearance" + # PII-free: only identifiers / reason — no query / chunk / label text. + assert set(extra) == {"event_kind", "tenant_id", "principal_id", "decision", "reason"} + + +async def test_labeled_principal_does_not_emit(monkeypatch: pytest.MonkeyPatch) -> None: + import rag_policy.acl as acl_mod + + calls: list[tuple[str, dict[str, Any]]] = [] + monkeypatch.setattr(acl_mod._log, "warning", lambda msg, **kw: calls.append((msg, kw))) + + await _filter(["eng"], PolicyDecision.read_chunk) + assert [msg for msg, _ in calls if msg == "acl.egress_denied"] == [] diff --git a/tests/redteam/test_acl_isolation.py b/tests/redteam/test_acl_isolation.py new file mode 100644 index 0000000..793f20b --- /dev/null +++ b/tests/redteam/test_acl_isolation.py @@ -0,0 +1,106 @@ +"""Red-team probe: ACL push-down restricts retrieval to cleared chunks (Step 6.3). + +Exercises the *canonical* ``read_chunk`` PDP site — ``HybridRetriever`` merging +``AclPolicyEngine.filter_pushdown`` into the backend query — over the in-memory +``NoopVectorStore`` (whose ``evaluate`` is the conformance oracle). A principal +retrieves a chunk only when their ACL labels overlap its ``acl_labels``; +label-less principals retrieve nothing (fail-closed); and tenant scoping is never +bypassed. +""" + +from __future__ import annotations + +from rag_core.spi.noop.vector_store import NoopVectorStore +from rag_core.types import ( + ChunkId, + Embedding, + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) +from rag_policy import AclPolicyEngine, NoopPolicyEngine +from rag_retrieval import HybridRetriever, HybridWeights + +_VEC = [1.0, 0.0] + + +def _ctx(tenant: str, labels: list[str]) -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), + kind=PrincipalKind.user, + display_name="p", + tenant_id=tid, + acl_labels=frozenset(labels), + ), + ) + + +def _emb(chunk_id: str, tenant: str, labels: list[str]) -> Embedding: + return Embedding( + chunk_id=ChunkId(chunk_id), + tenant_id=TenantId(tenant), + vector=_VEC, + dimension=2, + model="m", + acl_labels=frozenset(labels), + ) + + +async def _seeded() -> HybridRetriever: + """A retriever over a Noop store pre-seeded with mixed-label chunks.""" + store = NoopVectorStore() + await store.bulk_index( + _ctx("acme", []), # writer; the index path isn't ACL-gated + [ + _emb("c-eng", "acme", ["eng"]), + _emb("c-fin", "acme", ["fin"]), + _emb("c-pub", "acme", ["public"]), + _emb("c-none", "acme", []), + ], + ) + await store.bulk_index(_ctx("globex", []), [_emb("g-eng", "globex", ["eng"])]) + return HybridRetriever( + vector_backend=store, + weights=HybridWeights(), + policy_engine=AclPolicyEngine(inner=NoopPolicyEngine()), + ) + + +async def _ids(hybrid: HybridRetriever, ctx: RequestContext) -> set[str]: + refs = await hybrid.retrieve(ctx, vector=_VEC, top_k=10) + return {str(r.chunk_id) for r in refs} + + +async def test_principal_sees_only_overlapping_labels() -> None: + hybrid = await _seeded() + got = await _ids(hybrid, _ctx("acme", ["eng", "public"])) + assert got == {"c-eng", "c-pub"} # not c-fin, not the label-less c-none + + +async def test_disjoint_labels_get_nothing() -> None: + hybrid = await _seeded() + assert await _ids(hybrid, _ctx("acme", ["hr"])) == set() + + +async def test_label_less_principal_is_fail_closed() -> None: + hybrid = await _seeded() + # No clearance → nothing, not even the unlabeled chunk. + assert await _ids(hybrid, _ctx("acme", [])) == set() + + +async def test_public_label_is_the_shared_grant_idiom() -> None: + hybrid = await _seeded() + # "public" is just a label everyone can be granted — no special predicate. + assert await _ids(hybrid, _ctx("acme", ["public"])) == {"c-pub"} + + +async def test_acl_does_not_bypass_tenant_scoping() -> None: + hybrid = await _seeded() + # A globex principal with the 'eng' label must not reach acme's 'eng' chunk, + # and only sees globex's own. + assert await _ids(hybrid, _ctx("globex", ["eng"])) == {"g-eng"}