From f13555a2ff51db5da6019ed7b34067e65013e879 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Mon, 8 Jun 2026 03:43:31 +0530 Subject: [PATCH] feat(crypto): BYOK envelope encryption library + local KMS (Step 6.7a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of BYOK. New KeyManager SPI (rag-core) over opaque bytes (encrypt/decrypt) + NoopKeyManager passthrough; EncryptingStorage decorator (crypto-free) that encrypts blobs on put / decrypts on get — the application seam over any Storage. EnvelopeKeyManager base (rag-backends, cryptography) does the data-key half once: a fresh AES-256-GCM DEK per payload with ctx.tenant_id bound as AAD, the small DEK wrapped/unwrapped by a subclass via the KEK. LocalKeyManager wraps with an in-process per-tenant KEK (dev/tests/air-gapped); cloud providers (6.7b) subclass and wrap via the KMS API so the KEK never leaves the customer. Per-tenant isolation via KEK-per-tenant + tenant-bound AAD (a shared KEK still can't read another tenant's blob); sealing via KeyUnavailableError (a tenant with no KEK is denied, others unaffected); tamper-evidence via the GCM auth tag. Adds EncryptionError / KeyUnavailableError + ragctl kms demo. Encryption targets chunk content / blobs at rest, NOT embedding vectors (ANN search needs plaintext vectors) — a deliberate, documented constraint. Deferred: cloud KMS providers behind [kms-*] extras + wiring EncryptingStorage into the ingest/storage path + per-tenant key config (6.7b); key rotation (6.7c). Co-Authored-By: Claude Opus 4.7 --- TRACKER.md | 18 ++- docs/README.md | 2 + docs/adr/ADR-0039-byok-envelope-encryption.md | 84 ++++++++++ docs/architecture/byok.md | 89 ++++++++++ docs/reference/encryption.md | 75 +++++++++ packages/backends/pyproject.toml | 3 + .../backends/src/rag_backends/__init__.py | 2 + .../backends/src/rag_backends/kms/__init__.py | 12 ++ .../backends/src/rag_backends/kms/envelope.py | 98 +++++++++++ .../backends/src/rag_backends/kms/local.py | 72 +++++++++ .../core/src/rag_core/encrypting_storage.py | 61 +++++++ packages/core/src/rag_core/errors.py | 26 +++ packages/core/src/rag_core/spi/__init__.py | 2 + packages/core/src/rag_core/spi/key_manager.py | 54 +++++++ .../core/src/rag_core/spi/noop/__init__.py | 2 + .../core/src/rag_core/spi/noop/key_manager.py | 26 +++ packages/ragctl/src/ragctl/main.py | 63 ++++++++ packages/ragctl/tests/test_kms.py | 20 +++ tests/contract/test_key_manager.py | 56 +++++++ tests/kms/test_envelope.py | 153 ++++++++++++++++++ uv.lock | 2 + 21 files changed, 918 insertions(+), 2 deletions(-) create mode 100644 docs/adr/ADR-0039-byok-envelope-encryption.md create mode 100644 docs/architecture/byok.md create mode 100644 docs/reference/encryption.md create mode 100644 packages/backends/src/rag_backends/kms/__init__.py create mode 100644 packages/backends/src/rag_backends/kms/envelope.py create mode 100644 packages/backends/src/rag_backends/kms/local.py create mode 100644 packages/core/src/rag_core/encrypting_storage.py create mode 100644 packages/core/src/rag_core/spi/key_manager.py create mode 100644 packages/core/src/rag_core/spi/noop/key_manager.py create mode 100644 packages/ragctl/tests/test_kms.py create mode 100644 tests/contract/test_key_manager.py create mode 100644 tests/kms/test_envelope.py diff --git a/TRACKER.md b/TRACKER.md index c158bd7..97b0c9c 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -16,10 +16,11 @@ | **Last updated** | 2026-06-08 | | **Current phase** | Phase 6 — Governance & Tenancy (**6 / 10 steps**) | | **Overall** | **70 / 84 steps** — Phases 0–5 complete | -| **Next action** | **Step 6.7 — BYOK (Bring Your Own Key)**: KMS integration (AWS KMS / GCP KMS / HashiCorp Vault) + envelope encryption for embeddings; per-tenant keys. New ground (no existing crypto seam beyond the HMAC signers from 5.1 / 6.6b). | +| **Next action** | **Step 6.7b — Cloud KMS providers + storage wiring**: AWS / GCP / Azure / Vault `EnvelopeKeyManager` subclasses behind `[kms-*]` extras (lazy SDK, wrap/unwrap via KMS) + wire `EncryptingStorage` into the ingest/storage path + per-tenant key config (`cfg.kms` + `TenantConfig` key ref). Completes Step 6.7 with 6.7c (key rotation). 6.7a — the encryption library + local KMS — shipped. | **Recently shipped** +- **6.7a** 🚧 BYOK envelope encryption library — `KeyManager` SPI + `NoopKeyManager` + `EncryptingStorage` (rag-core); `EnvelopeKeyManager` (AES-256-GCM DEK + `tenant_id` AAD) + `LocalKeyManager` (in-process per-tenant KEK) (rag-backends); per-tenant isolation + **sealing** (`KeyUnavailableError`) + tamper detection; `ragctl kms`; first slice of Step 6.7 (cloud KMS + wiring is 6.7b) — [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) - **6.6** ✅ Immutable audit log — **6.6a** shared `AuditWriter`/store on `app.state` + `GET /v1/audit` (tenant-scoped, `chain_verified`) + `GET /v1/audit/verify` (whole-log) + `cfg.audit.enabled` ([#153](https://github.com/officialCodeWork/AgentContextOS/pull/153)); **6.6b** `AuditExporter` self-verifying WORM bundle (SHA-256 `content_hash` + HMAC, offline `verify()`), `POST /v1/audit/export` (tenant-scoped) + `ragctl audit`, `cfg.audit.export_secret` ([#154](https://github.com/officialCodeWork/AgentContextOS/pull/154)) - **6.5** ✅ PII egress policies — `PiiPolicyEngine` (`rag-pii`) answers `egress_text` over the context (`list[Chunk]`) + agent answer (`str`) the gateway already passes, applying the per-tenant `pii_policy` (allow / redact / mask / block); reuses the Step 1.7 detector + rewriters; opt-in `cfg.pii.enabled`; emits `pii.egress_blocked` — [#152](https://github.com/officialCodeWork/AgentContextOS/pull/152) - **6.4** ✅ ACL egress verifier — `AclEgressVerifier` re-checks returned `ChunkRef`s against the principal's labels at the gateway router boundary (defense-in-depth, same overlap semantics, independent of the PDP); `acl.egress_violation` event; red-team zero-violation-rate gate — [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) @@ -655,7 +656,7 @@ | 6.4 | ACL egress verifier | ✅ | [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) — `AclEgressVerifier` re-checks returned chunks at the gateway router boundary (defense-in-depth above the 6.3 push-down); `acl.egress_violation`; zero-violation-rate red-team gate | | 6.5 | PII policies | ✅ | [#152](https://github.com/officialCodeWork/AgentContextOS/pull/152) — `PiiPolicyEngine` egress_text decorator (allow / redact / mask / block per tenant) over answer + context; reuses Step 1.7 detector; `pii.egress_blocked` | | 6.6 | Immutable audit log | ✅ | **6.6a** [#153](https://github.com/officialCodeWork/AgentContextOS/pull/153) — read API `GET /v1/audit` + `GET /v1/audit/verify` + shared store + `cfg.audit`. **6.6b** [#154](https://github.com/officialCodeWork/AgentContextOS/pull/154) — `AuditExporter` signed WORM bundle + `POST /v1/audit/export` + `ragctl audit` | -| 6.7 | BYOK (Bring Your Own Key) | ⏳ | KMS integration (AWS KMS, GCP KMS, HashiCorp Vault); envelope encryption for embeddings | +| 6.7 | BYOK (Bring Your Own Key) | 🚧 | **6.7a** ✅ [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) — `KeyManager` SPI + envelope encryption (`LocalKeyManager`) + `EncryptingStorage` + sealing/isolation/tamper + `ragctl kms`. **6.7b** ⏳ — cloud KMS providers + storage wiring + config; **6.7c** ⏳ — key rotation | | 6.8 | SSO / SCIM | ⏳ | OIDC + SAML IdP federation; SCIM 2.0 user provisioning; per-tenant IdP config | | 6.9 | Air-gapped install bundle | ⏳ | Signed tarball with all images + Helm chart; offline bootstrap; cosign verification | | 6.10 | Compliance posture | ⏳ | SOC 2 Type II control mapping; GDPR data-residency config; data-retention policies | @@ -727,6 +728,18 @@ Step 0.7c shipped the foundation — an append-only `AuditStore` SPI, a `NoopAud - New `AuditExport` / `AuditExportSignature` / `AuditExportVerification` core types (`dist/schemas` regenerated); `POST /v1/audit/export` → `dist/openapi`; **`ragctl audit`** (seed → export → verify round-trip, `--out` writes the bundle, `--verify FILE` checks one offline). ~23 new tests (exporter unit: content_hash determinism, sign→verify, unsigned, no_secret, wrong-secret, **content tamper → content_mismatch**, **signature tamper → signature_mismatch**, empty; gateway: tenant-scoped + signed + **offline-verifies** + cross-tenant isolation + unsigned + disabled→404 + no-auth→401; ragctl round-trip + file verify). All gates green (ruff, mypy --strict 299 files, RAG001, schema/openapi-drift, policy-coverage, log-schema). [ADR-0038](docs/adr/ADR-0038-immutable-audit-log.md), [architecture/audit-log.md](docs/architecture/audit-log.md), [reference/audit.md](docs/reference/audit.md) - **Deferred:** a durable live-store `AuditStore` backend (Postgres / append-only ledger) and expanding what gets audited beyond `corpus.route`. +### 6.7 — BYOK (Bring Your Own Key) 🚧 (sliced 6.7a + 6.7b + 6.7c) + +New ground — the only prior crypto was HMAC signing. The V1 plan calls for envelope encryption with per-tenant data keys wrapped by a customer-controlled KMS KEK, zero-downtime rotation, and **sealing** (KEK unavailable → tenant data unreadable). Delivered in slices. + +#### 6.7a — Envelope encryption library + local KMS ✅ [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) + +- **What's encrypted:** **chunk content / blobs at rest** — *not* embedding vectors (ANN search reads vectors directly, so encrypting them would break retrieval; vectors carry no raw text). A deliberate, documented constraint +- **`KeyManager` SPI** (`rag-core`) over opaque bytes — `encrypt(ctx, plaintext) → bytes` / `decrypt(ctx, ciphertext) → bytes`; ciphertext is a self-describing envelope. `NoopKeyManager` (passthrough) for wiring tests. **`EncryptingStorage`** (`rag-core`, crypto-free) is a `Storage` decorator that encrypts on `put` / decrypts on `get` — the application seam over any backend. `EncryptionError` + `KeyUnavailableError` (sealing) +- **`EnvelopeKeyManager`** base (`rag-backends`, `cryptography`) does the data-key half once for every provider: a fresh AES-256-GCM **DEK** per payload (with `ctx.tenant_id` bound as **AAD**), the small DEK handed to a subclass to wrap/unwrap with the **KEK**. **`LocalKeyManager`** wraps with an in-process per-tenant KEK (dev / tests / air-gapped); cloud providers (6.7b) subclass and wrap/unwrap via the KMS API so the KEK never leaves the customer +- **Per-tenant isolation** via KEK-per-tenant **and** tenant-bound AAD (even a shared KEK can't read another tenant's blob); **sealing** via `KeyUnavailableError` (a tenant with no KEK is denied, others unaffected); **tamper-evidence** via the GCM auth tag. `cryptography` in `rag-backends`; `rag-core` stays crypto-free (SPI + decorator + noop) +- **Scope:** library + local KMS only. **Deferred:** cloud KMS providers behind `[kms-*]` extras + wiring `EncryptingStorage` into the ingest/storage path + per-tenant key config (6.7b); zero-downtime key rotation (6.7c); vector encryption is out of scope by design. ~19 new tests (KeyManager contract over noop + local; envelope round-trip / non-determinism / tamper / malformed / cross-tenant / shared-KEK-still-blocked / sealing on encrypt+decrypt / wrong-size KEK / per-tenant non-impact / EncryptingStorage stores ciphertext + delegates key ops) + `ragctl kms` smoke; all gates green (ruff, mypy --strict 305 files, RAG001, policy-coverage, log-schema; no dist drift). [ADR-0039](docs/adr/ADR-0039-byok-envelope-encryption.md), [architecture/byok.md](docs/architecture/byok.md), [reference/encryption.md](docs/reference/encryption.md) + --- ## Phase 7 — Pilot, Harden, GA (Weeks 34–40) ⏳ @@ -885,6 +898,7 @@ Complete log of every PR. Routine Dependabot bumps are grouped; everything else | [#152](https://github.com/officialCodeWork/AgentContextOS/pull/152) | 2026-06-08 | feat(pii): PII egress policies — PiiPolicyEngine egress_text decorator (Step 6.5) | | [#153](https://github.com/officialCodeWork/AgentContextOS/pull/153) | 2026-06-08 | feat(gateway): audit read API + chain verification (Step 6.6a) | | [#154](https://github.com/officialCodeWork/AgentContextOS/pull/154) | 2026-06-08 | feat(audit): WORM signed export — AuditExporter + POST /v1/audit/export (Step 6.6b) | +| [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) | 2026-06-08 | feat(crypto): BYOK envelope encryption library + local KMS (Step 6.7a) | | #78–#80, #116–#118 | Open | Dependabot bumps — awaiting merge | | #81 | Closed | Dependabot bump — superseded | diff --git a/docs/README.md b/docs/README.md index 644e2c6..3a06a92 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ | [multi-tenancy.md](architecture/multi-tenancy.md) | Logical multi-tenancy (Step 6.1): make per-tenant `rag.yaml` config drive requests. `TenantResolver` (`rag_config.tenancy`) maps a tenant id → frozen `TenantSettings` (namespace / pii_policy / acl_labels), applied **once** at the gateway boundary (namespace + pii_policy onto the `RequestContext`, acl_labels unioned into the principal); unknown tenants resolve to safe defaults (namespace = id, default PII, no labels — isolated not privileged); `RequestContext.namespace` defaults to `tenant_id` (Pinecone partitions on it; `filter_pushdown` unchanged); resolves+threads only — ACL push-down is 6.3, PII egress 6.5, physical tenancy 6.2; `GET /v1/status/tenant`; inert in `build_app` | | [policy-engine.md](architecture/policy-engine.md) | `PolicyEngine` (PDP) — single decision point for ACL, PII, quotas, redaction; replaces scattered governance checks | | [audit-log.md](architecture/audit-log.md) | Immutable audit log (Step 6.6): tamper-evidence (SHA-256 hash chain) vs immutability-at-rest (WORM export, 6.6b); one shared `AuditWriter`/store on `app.state`; read-path tenant scoping (tenant-scoped list vs whole-log verify); why the read API defaults on | +| [byok.md](architecture/byok.md) | BYOK / envelope encryption (Step 6.7): what's encrypted (chunk content at rest) vs not (embedding vectors — search needs plaintext); DEK+KEK envelope (client-side AES-GCM DEK, provider wraps the DEK); per-tenant isolation via KEK + `tenant_id` AAD; sealing as a typed error; rag-core/rag-backends split; slicing (6.7a library, 6.7b cloud KMS + wiring, 6.7c rotation) | | [caching.md](architecture/caching.md) | Three-cache split: `EmbeddingCache`, `RetrievalCache`, `AnswerCache` — distinct invalidation rules | | [performance.md](architecture/performance.md) | Hot-path discipline, per-SPI p99 budgets, async telemetry, reviewer checklist | | [pipeline-batcher.md](architecture/pipeline-batcher.md) | `Pipeline` (async DAG, bounded queues, per-stage workers) + `Batcher` (DataLoader-pattern coalescing) primitives — Step 1.1d | @@ -106,6 +107,7 @@ | [admin-ui.md](reference/admin-ui.md) | Admin console (Step 3.10) — Next.js 14 operator GUI (`apps/admin-ui`); 9 pages (dashboard, corpora, connectors, glossary, webhooks, audit, API keys, tenants, config), live-vs-seed hybrid + `NEXT_PUBLIC_GATEWAY_URL`, header identity, running it, internals (shell/primitives/data layer), extension points | | [tenancy.md](reference/tenancy.md) | Logical multi-tenancy (Step 6.1) — per-tenant `rag.yaml` config (`namespace` / `acl_labels` / `pii_policy` / `quota`); `TenantResolver.resolve(id) → TenantSettings`; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant list` / `resolve`; config table + scope/boundaries (6.2/6.3/6.5) + extension points; physical tenancy (6.2), ACL push-down (6.3) + egress verifier (6.4 — `cfg.acl.verify_egress`) sections | | [audit.md](reference/audit.md) | Audit log (Step 6.6) — `AuditEvent` / `AuditStore` (append / events / verify_chain) / `NoopAuditStore` SHA-256 hash chain / `AuditWriter` (+ `.store`); read API `GET /v1/audit` (tenant-scoped, `chain_verified`) + `GET /v1/audit/verify` (whole-log); WORM signed export (6.6b) — `AuditExporter` (content_hash + HMAC), `POST /v1/audit/export`, offline `verify()`, `ragctl audit`; `cfg.audit.enabled` / `export_secret`; durable-store extension points | +| [encryption.md](reference/encryption.md) | BYOK envelope encryption (Step 6.7a) — `KeyManager` SPI (encrypt/decrypt opaque bytes) + `NoopKeyManager`; `EnvelopeKeyManager` (AES-256-GCM DEK + `tenant_id` AAD) + `LocalKeyManager` (in-process per-tenant KEK); `EncryptingStorage` decorator; `EncryptionError` / `KeyUnavailableError` (sealing); guarantees table (confidentiality / tamper / isolation / sealing); `ragctl kms`; cloud-provider extension points (6.7b) | | [webhooks.md](reference/webhooks.md) | Outbound webhooks (Step 3.9) — event catalogue (`ingest.completed` / `audit.policy_violation` / `drift.detected` / `eval.regression`), event envelope, HMAC signing + `verify()`, at-least-once delivery, `/v1/webhooks/subscriptions` CRUD + test, `rag.yaml` block, `ragctl webhooks demo`, internals + extension points | | [integrations.md](reference/integrations.md) | Framework adapters (Step 3.8) — `agentcontextos.integrations.*` for LangChain / LlamaIndex / Haystack / DSPy / LangGraph / CrewAI / AutoGen / Semantic Kernel; per-framework extras, shared config + chunk metadata, usage per framework, internals + extension points | | [status-api.md](reference/status-api.md) | Status & Metrics API (Step 3.11) — `/v1/status/health` / `metrics` / `logs` (+ SSE `logs/stream`), `WS /v1/status/ws`, `/v1/connectors/status`; metric catalogue + request-timing middleware, the `MetricsCollector` / `LogTail` read-side, CORS + query-param identity for browser streams, extension points | diff --git a/docs/adr/ADR-0039-byok-envelope-encryption.md b/docs/adr/ADR-0039-byok-envelope-encryption.md new file mode 100644 index 0000000..d518e44 --- /dev/null +++ b/docs/adr/ADR-0039-byok-envelope-encryption.md @@ -0,0 +1,84 @@ +# ADR-0039 — BYOK envelope encryption + +**Status:** Accepted +**Date:** 2026-06-08 +**Step:** 6.7 — BYOK (Bring Your Own Key) (Phase 6 — Governance & Tenancy) +**Related:** [ADR-0007](ADR-0007-tiered-storage.md) (BlobRef / tiered storage), [ADR-0026](ADR-0026-per-query-tracing-provenance.md) (HMAC signing), [architecture/byok.md](../architecture/byok.md), [reference/encryption.md](../reference/encryption.md) + +## Context + +The V1 plan (Step 6.7) calls for **envelope encryption** with **per-tenant data +keys** wrapped by a **customer-controlled KEK** in a KMS (AWS KMS / GCP KMS / +Azure Key Vault / HashiCorp Vault — "Bring Your Own Key"), plus zero-downtime key +rotation and **sealing** (if the KEK becomes unavailable, that tenant's data +becomes unreadable, with no cross-tenant impact). This is new ground — the only +crypto in the codebase was HMAC signing (provenance / webhooks / audit export). + +This ADR is delivered in slices: **6.7a** (this) — the encryption library + a +local KMS; **6.7b** — cloud KMS providers + wiring the encryption into the storage +path + per-tenant key config; **6.7c** — key rotation. + +## Decision + +**1. Encrypt content at rest, not embedding vectors.** Embedding *vectors* must +stay plaintext — approximate-nearest-neighbour search reads them directly, so +encrypting them would break retrieval (and homomorphic search is out of scope). +The encryption target is the **sensitive chunk content / blobs at rest** (the +text), stored behind the `Storage` SPI. Vectors carry no raw text; the text is +what BYOK protects. + +**2. A `KeyManager` SPI over opaque bytes.** `encrypt(ctx, plaintext) -> bytes` / +`decrypt(ctx, ciphertext) -> bytes`. The ciphertext is a self-describing envelope; +callers treat it as opaque and store it wherever they'd store the plaintext. Keeps +the SPI tiny and provider-agnostic. `NoopKeyManager` (passthrough, no crypto) is +the test/dev wiring stub. + +**3. Envelope encryption in a shared base.** `EnvelopeKeyManager` (rag-backends) +does the data-key half once for every provider: a fresh AES-256-GCM **DEK** per +payload encrypts the plaintext (with `ctx.tenant_id` bound in as **AAD**, so a +ciphertext can't be replayed under another tenant), and the small DEK is handed to +a subclass to **wrap / unwrap** with the **KEK**. Subclasses implement only +`_wrap_dek` / `_unwrap_dek`: `LocalKeyManager` wraps with an in-process KEK (6.7a); +cloud providers wrap via the KMS API (6.7b) — so the KEK never leaves the +customer's control. + +**4. `EncryptingStorage` is the application seam.** A crypto-free `Storage` +decorator (rag-core) that runs blobs through the `KeyManager` on `put` / `get`. It +composes with any storage backend and any provider, so wiring BYOK = wrapping the +production `Storage` when enabled. + +**5. Per-tenant isolation + sealing are first-class.** The KEK is resolved from +`ctx.tenant_id`; a tenant with no KEK is **sealed** — `encrypt` / `decrypt` raise +`KeyUnavailableError` (a typed error, distinct from `EncryptionError` for +tampering), with no cross-tenant impact. The tenant-bound AAD means even a shared +KEK can't read another tenant's blob. + +**6. `cryptography` in rag-backends.** AES-GCM comes from the well-maintained +`cryptography` library (don't roll your own crypto), declared in rag-backends +where the real provider impls live. rag-core stays crypto-free (SPI + decorator + +`NoopKeyManager` only). + +## Consequences + +**Positive** +- BYOK control: a customer holds the KEK; revoking it seals their data + (demonstrated by the sealing test) — independent of platform operators. +- Tamper-evident (AES-GCM auth tag) + tenant-bound (AAD), verified by tests. +- The encryption library is complete + tested now; wiring + cloud providers layer + on without changing the SPI. + +**Negative / deferred** +- **Embedding vectors are not encrypted** — a deliberate constraint (search needs + plaintext vectors). Deployments needing vector confidentiality rely on + infrastructure-level disk encryption for the vector store. +- Data is decrypted server-side for use, so plaintext exists in memory during a + request — BYOK protects data **at rest** + gates access via the KEK, not against + a compromised running server. +- **6.7b** (cloud KMS providers behind `[kms-*]` extras + wiring `EncryptingStorage` + into the ingest/storage path + per-tenant key config) and **6.7c** (zero-downtime + rotation, decrypt-only retention of old keys) are deferred. + +## See also +- [architecture/byok.md](../architecture/byok.md) — envelope design, AAD, sealing, slicing +- [reference/encryption.md](../reference/encryption.md) — `KeyManager` / `LocalKeyManager` / `EncryptingStorage` +- [ADR-0007](ADR-0007-tiered-storage.md) — the `BlobRef` / `Storage` path this encrypts diff --git a/docs/architecture/byok.md b/docs/architecture/byok.md new file mode 100644 index 0000000..5ed1da7 --- /dev/null +++ b/docs/architecture/byok.md @@ -0,0 +1,89 @@ +# BYOK / envelope encryption — architecture + +Design notes behind the [encryption reference](../reference/encryption.md): what +gets encrypted (and what deliberately doesn't), the envelope structure, how +per-tenant isolation and sealing work, and the package layout. + +## What is encrypted — and what isn't + +**Encrypted: chunk content / blobs at rest.** The sensitive payload in a RAG +system is the source *text*. It's stored behind the `Storage` SPI (inline text +moves to a `BlobRef` blob under tiered storage, ADR-0007), which is the natural +encryption boundary. + +**Not encrypted: embedding vectors.** Approximate-nearest-neighbour search reads +vectors directly, so encrypting them at rest would break retrieval (homomorphic +vector search is out of scope). Vectors carry no raw text — they're a lossy +numeric projection — so leaving them plaintext doesn't expose the content that +BYOK protects. Deployments that want vector-store confidentiality use +infrastructure disk encryption for that store. + +This is the key design call: BYOK here means **content-at-rest encryption gated by +a customer-controlled key**, not end-to-end encryption of every artifact. + +## Envelope encryption (DEK + KEK) + +Standard envelope encryption, split so the KEK never leaves the customer's KMS: + +``` +encrypt(plaintext): + DEK = random 256-bit key # one per payload + ct = AES-256-GCM(DEK, nonce, plaintext, aad=tenant_id) + wDEK = wrap(DEK) via the tenant's KEK # provider hook (_wrap_dek) + return RAGK | ver | len(wDEK) | wDEK | nonce | ct + +decrypt(envelope): + DEK = unwrap(wDEK) via the tenant's KEK # provider hook (_unwrap_dek) — sealing point + return AES-256-GCM-decrypt(DEK, nonce, ct, aad=tenant_id) +``` + +The DEK does the bulk encryption client-side (in `EnvelopeKeyManager`); only the +small DEK is wrapped/unwrapped by the KEK. A cloud provider implements just +`_wrap_dek` / `_unwrap_dek` (a KMS Encrypt/Decrypt call); `LocalKeyManager` wraps +with an in-process KEK. Everything else — DEK generation, AES-GCM, AAD, framing — +is shared. + +## Per-tenant isolation: KEK + AAD + +Two independent mechanisms, so isolation holds even if one is misconfigured: + +1. **KEK per tenant** — the wrap/unwrap key is resolved from `ctx.tenant_id`, so a + blob wrapped for tenant A can't be unwrapped without A's KEK. +2. **Tenant-bound AAD** — `tenant_id` is the AES-GCM additional authenticated + data, so even if two tenants *shared* a KEK, decrypting A's blob under B's + context fails the GCM tag. A blob is cryptographically pinned to its tenant. + +## Sealing + +"Sealing" = a tenant's KEK becoming unavailable (revoked / unreachable) renders +*that tenant's* data unreadable, with no cross-tenant impact. It's modelled as a +typed error: `_unwrap_dek` raises `KeyUnavailableError` (a subclass of +`EncryptionError`, so callers can distinguish "key gone" from "data tampered"). +`LocalKeyManager` seals any tenant with no configured KEK; a cloud provider seals +when the KMS denies/loses the key. The sealing test kills a tenant's key and +asserts its reads fail while another tenant's succeed. + +## Package layout + +- **rag-core** (crypto-free): `KeyManager` SPI, `NoopKeyManager` (passthrough), + `EncryptingStorage` decorator, `EncryptionError` / `KeyUnavailableError`. +- **rag-backends** (real crypto, `cryptography` dep): `EnvelopeKeyManager` base + + `LocalKeyManager`. Cloud KMS providers (AWS/GCP/Azure/Vault) land here behind + `[kms-*]` extras in 6.7b. This mirrors the SPI-in-core / impls-in-backends split + used for vector stores, storage, etc. + +## Slicing + +- **6.7a** (this) — the encryption library + `LocalKeyManager` + `EncryptingStorage` + + full crypto tests + `ragctl kms`. A complete, demonstrable BYOK foundation. +- **6.7b** — cloud KMS providers behind extras; wire `EncryptingStorage` into the + ingest/storage path; per-tenant key config (`cfg.kms` + `TenantConfig` key ref). +- **6.7c** — zero-downtime key rotation; old keys retained decrypt-only until + expiry; background re-encryption. + +## Boundary note + +BYOK protects data **at rest** and gates access via the customer's KEK. Data is +decrypted server-side for use, so plaintext exists in process memory during a +request — BYOK is not a defense against a compromised running server; it's control +over stored data and a kill-switch (sealing) the customer holds. diff --git a/docs/reference/encryption.md b/docs/reference/encryption.md new file mode 100644 index 0000000..bc5f646 --- /dev/null +++ b/docs/reference/encryption.md @@ -0,0 +1,75 @@ +# Encryption (BYOK) — reference + +Envelope encryption for data at rest with per-tenant, customer-controlled keys +(Step 6.7). 6.7a ships the library + a local KMS; cloud KMS providers + storage +wiring are 6.7b. + +## Overview + +- **`KeyManager`** (`rag_core.spi.key_manager`) — SPI over opaque bytes: + `encrypt(ctx, plaintext) -> bytes`, `decrypt(ctx, ciphertext) -> bytes`, + `health()`. The ciphertext is a self-describing envelope; treat it as opaque. +- **`NoopKeyManager`** (`rag_core.spi.noop`) — passthrough, **no encryption** + (test/dev wiring only). +- **`EnvelopeKeyManager`** (`rag_backends.kms`) — client-side envelope base: a + fresh AES-256-GCM DEK per payload (with `ctx.tenant_id` as AAD); subclasses + implement `_wrap_dek` / `_unwrap_dek`. +- **`LocalKeyManager`** (`rag_backends.kms`) — wraps the DEK with an **in-process** + per-tenant KEK (dev / tests / air-gapped). Cloud providers (AWS/GCP/Azure/Vault) + are 6.7b. +- **`EncryptingStorage`** (`rag_core.encrypting_storage`) — a `Storage` decorator + that encrypts on `put` / decrypts on `get` (crypto-free; uses the `KeyManager`). +- **Errors** (`rag_core.errors`): `EncryptionError` (malformed / tampered / wrong + tenant) and `KeyUnavailableError` (sealing — the KEK is revoked / unreachable). + +## Usage + +```python +import os +from rag_backends import LocalKeyManager +from rag_core.encrypting_storage import EncryptingStorage +from rag_core.spi.noop import NoopStorage + +km = LocalKeyManager(keks={"acme": os.urandom(32)}) # per-tenant KEKs +ciphertext = await km.encrypt(ctx, b"secret chunk text") # ctx.tenant_id = "acme" +assert await km.decrypt(ctx, ciphertext) == b"secret chunk text" + +# Encrypt blobs at rest behind any Storage backend: +store = EncryptingStorage(NoopStorage(), km) +await store.put(ctx, "doc/1", b"secret") # underlying store holds ciphertext +await store.get(ctx, "doc/1") # → b"secret" +``` + +### Guarantees + +| Property | How | +|----------|-----| +| Confidentiality | AES-256-GCM with a per-payload DEK; ciphertext ≠ plaintext | +| Tamper-evidence | GCM auth tag → `EncryptionError` on any bit flip | +| Per-tenant isolation | KEK per tenant **+** `tenant_id` bound as AAD (a blob can't be read under another tenant, even with a shared KEK) | +| Sealing | No/revoked KEK for a tenant → `KeyUnavailableError`; that tenant's data is unreadable, others unaffected | + +### CLI + +```bash +ragctl kms --tenant acme --text "secret" # round-trip + isolation + sealing demo +``` + +## Internals + +- **Envelope wire format:** `b"RAGK" | version | u32(len(wrapped_dek)) | + wrapped_dek | data_nonce(12) | ciphertext` — opaque to callers, parsed only by + `EnvelopeKeyManager`. +- **Vectors stay plaintext.** Encryption targets chunk content / blobs at rest, + not embedding vectors (ANN search needs plaintext vectors). See + [ADR-0039](../adr/ADR-0039-byok-envelope-encryption.md). +- **`cryptography`** provides AES-GCM (in `rag-backends`); rag-core stays + crypto-free (SPI + decorator + noop only). + +## Extension points + +Add a cloud KMS provider (6.7b): subclass `EnvelopeKeyManager` and implement +`_wrap_dek` / `_unwrap_dek` by calling the KMS Encrypt/Decrypt API for the +tenant's key (raise `KeyUnavailableError` when the key is denied / unreachable). +The DEK generation, AES-GCM, AAD, and envelope framing are inherited — a provider +is just the KEK wrap/unwrap. diff --git a/packages/backends/pyproject.toml b/packages/backends/pyproject.toml index ef67631..3e16a16 100644 --- a/packages/backends/pyproject.toml +++ b/packages/backends/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ "aioboto3>=13.0", "types-aiobotocore[s3]>=2.13", "aiofiles>=24.0", + # BYOK envelope encryption (Step 6.7) — AES-256-GCM for the LocalKeyManager + # + the EnvelopeKeyManager base the cloud KMS providers build on. + "cryptography>=43.0", ] [project.optional-dependencies] diff --git a/packages/backends/src/rag_backends/__init__.py b/packages/backends/src/rag_backends/__init__.py index 6dd7aa4..7b0a579 100644 --- a/packages/backends/src/rag_backends/__init__.py +++ b/packages/backends/src/rag_backends/__init__.py @@ -11,6 +11,7 @@ from rag_backends.cache.redis import RedisCache from rag_backends.cache.redis_typed import RedisAnswerCache, RedisRetrievalCache from rag_backends.corpus.pg_corpus_store import PgCorpusStore +from rag_backends.kms.local import LocalKeyManager from rag_backends.quota.redis import RedisQuotaStore from rag_backends.storage.local import LocalFileStorage from rag_backends.storage.s3 import S3Storage @@ -18,6 +19,7 @@ from rag_backends.vector.qdrant import QdrantVectorStore __all__ = [ + "LocalKeyManager", "PgCorpusStore", "PgVectorStore", "QdrantVectorStore", diff --git a/packages/backends/src/rag_backends/kms/__init__.py b/packages/backends/src/rag_backends/kms/__init__.py new file mode 100644 index 0000000..42a0505 --- /dev/null +++ b/packages/backends/src/rag_backends/kms/__init__.py @@ -0,0 +1,12 @@ +"""BYOK key managers (Step 6.7) — envelope encryption providers. + +``EnvelopeKeyManager`` is the client-side envelope-encryption base; +``LocalKeyManager`` wraps the data key with an in-process per-tenant KEK (dev / +tests / air-gapped). Cloud providers (AWS KMS / GCP KMS / Azure Key Vault / +HashiCorp Vault) land in Step 6.7b behind ``[kms-*]`` extras. +""" + +from rag_backends.kms.envelope import EnvelopeKeyManager +from rag_backends.kms.local import LocalKeyManager + +__all__ = ["EnvelopeKeyManager", "LocalKeyManager"] diff --git a/packages/backends/src/rag_backends/kms/envelope.py b/packages/backends/src/rag_backends/kms/envelope.py new file mode 100644 index 0000000..1eca3b3 --- /dev/null +++ b/packages/backends/src/rag_backends/kms/envelope.py @@ -0,0 +1,98 @@ +"""EnvelopeKeyManager — client-side envelope encryption base (Step 6.7). + +Implements the data-key half of envelope encryption once, for every provider: a +fresh AES-256-GCM data-encryption key (DEK) per payload encrypts the plaintext +(with ``ctx.tenant_id`` bound in as additional authenticated data, so a ciphertext +can't be replayed under another tenant), and the small DEK is handed to a +subclass to *wrap* / *unwrap* with the key-encryption key (KEK). + +Subclasses implement only ``_wrap_dek`` / ``_unwrap_dek``: + +* :class:`~rag_backends.kms.local.LocalKeyManager` wraps with an in-process KEK + (dev / tests / air-gapped). +* Cloud providers (AWS KMS / GCP KMS / Azure Key Vault / HashiCorp Vault, Step + 6.7b) wrap by calling the KMS Encrypt/Decrypt API — the KEK never leaves the + customer's control ("Bring Your Own Key"). + +The wire format is ``b"RAGK" | version | u32(len(wrapped_dek)) | wrapped_dek | +data_nonce(12) | ciphertext`` — opaque to callers. +""" + +from __future__ import annotations + +import abc +import os +import struct + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from rag_core.errors import EncryptionError +from rag_core.spi.key_manager import KeyManager +from rag_core.types import RequestContext + +_MAGIC = b"RAGK" +_VERSION = 1 +_DEK_BYTES = 32 # AES-256 +_NONCE_BYTES = 12 # GCM standard nonce + +__all__ = ["EnvelopeKeyManager"] + + +class EnvelopeKeyManager(KeyManager, abc.ABC): + """Envelope encryption; subclasses wrap/unwrap the DEK via a KEK.""" + + async def encrypt(self, ctx: RequestContext, plaintext: bytes) -> bytes: + dek = os.urandom(_DEK_BYTES) + data_nonce = os.urandom(_NONCE_BYTES) + ciphertext = AESGCM(dek).encrypt(data_nonce, plaintext, _aad(ctx)) + wrapped = await self._wrap_dek(ctx, dek) # may raise KeyUnavailableError + return ( + _MAGIC + + bytes([_VERSION]) + + struct.pack(">I", len(wrapped)) + + wrapped + + data_nonce + + ciphertext + ) + + async def decrypt(self, ctx: RequestContext, ciphertext: bytes) -> bytes: + wrapped, data_nonce, ct = _unpack(ciphertext) + dek = await self._unwrap_dek(ctx, wrapped) # may raise KeyUnavailableError + try: + return AESGCM(dek).decrypt(data_nonce, ct, _aad(ctx)) + except InvalidTag as exc: + raise EncryptionError( + "envelope decryption failed (tampered ciphertext or wrong tenant)" + ) from exc + + @abc.abstractmethod + async def _wrap_dek(self, ctx: RequestContext, dek: bytes) -> bytes: + """Wrap (encrypt) the data key with the tenant's KEK.""" + + @abc.abstractmethod + async def _unwrap_dek(self, ctx: RequestContext, wrapped: bytes) -> bytes: + """Unwrap (decrypt) the data key with the tenant's KEK.""" + + +def _aad(ctx: RequestContext) -> bytes: + # Bind every ciphertext to its tenant — a blob can't be moved cross-tenant. + return str(ctx.tenant_id).encode("utf-8") + + +def _unpack(blob: bytes) -> tuple[bytes, bytes, bytes]: + try: + if blob[:4] != _MAGIC or blob[4] != _VERSION: + raise EncryptionError("unrecognised envelope header") + offset = 5 + (wrapped_len,) = struct.unpack(">I", blob[offset : offset + 4]) + offset += 4 + wrapped = blob[offset : offset + wrapped_len] + offset += wrapped_len + data_nonce = blob[offset : offset + _NONCE_BYTES] + offset += _NONCE_BYTES + ciphertext = blob[offset:] + except (IndexError, struct.error) as exc: + raise EncryptionError("malformed envelope") from exc + if len(wrapped) != wrapped_len or len(data_nonce) != _NONCE_BYTES: + raise EncryptionError("truncated envelope") + return wrapped, data_nonce, ciphertext diff --git a/packages/backends/src/rag_backends/kms/local.py b/packages/backends/src/rag_backends/kms/local.py new file mode 100644 index 0000000..5590900 --- /dev/null +++ b/packages/backends/src/rag_backends/kms/local.py @@ -0,0 +1,72 @@ +"""LocalKeyManager — envelope encryption with in-process per-tenant KEKs (Step 6.7). + +A KeyManager whose key-encryption keys live **in process** rather than a remote +KMS — for dev, tests, and air-gapped deployments. Per-tenant KEKs are resolved +from a mapping (with an optional shared default); a tenant with **no** KEK is +*sealed*: both ``encrypt`` and ``decrypt`` raise +:class:`~rag_core.errors.KeyUnavailableError`, so that tenant's data is +unreadable with no cross-tenant impact. + +Cloud providers (AWS KMS / GCP KMS / Azure Key Vault / HashiCorp Vault, Step +6.7b) subclass :class:`~rag_backends.kms.envelope.EnvelopeKeyManager` the same way +but wrap/unwrap the DEK via the KMS API. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from rag_core.errors import EncryptionError, KeyUnavailableError +from rag_core.types import RequestContext + +from rag_backends.kms.envelope import _NONCE_BYTES, EnvelopeKeyManager, _aad + +_KEK_BYTES = 32 # AES-256 + +__all__ = ["LocalKeyManager"] + + +class LocalKeyManager(EnvelopeKeyManager): + """Envelope encryption with in-process, per-tenant AES-256 KEKs.""" + + def __init__( + self, + *, + keks: Mapping[str, bytes] | None = None, + default_kek: bytes | None = None, + ) -> None: + self._keks = dict(keks or {}) + self._default = default_kek + + def _kek_for(self, ctx: RequestContext) -> bytes: + kek = self._keks.get(str(ctx.tenant_id), self._default) + if kek is None: + raise KeyUnavailableError( + f"no KEK configured for tenant {ctx.tenant_id!r} (sealed)", + tenant_id=str(ctx.tenant_id), + ) + if len(kek) != _KEK_BYTES: + raise KeyUnavailableError( + f"KEK for tenant {ctx.tenant_id!r} must be {_KEK_BYTES} bytes (AES-256)", + tenant_id=str(ctx.tenant_id), + ) + return kek + + async def _wrap_dek(self, ctx: RequestContext, dek: bytes) -> bytes: + kek = self._kek_for(ctx) + nonce = os.urandom(_NONCE_BYTES) + return nonce + AESGCM(kek).encrypt(nonce, dek, _aad(ctx)) + + async def _unwrap_dek(self, ctx: RequestContext, wrapped: bytes) -> bytes: + kek = self._kek_for(ctx) # KeyUnavailableError when sealed + nonce, ct = wrapped[:_NONCE_BYTES], wrapped[_NONCE_BYTES:] + try: + return AESGCM(kek).decrypt(nonce, ct, _aad(ctx)) + except InvalidTag as exc: + raise EncryptionError("DEK unwrap failed (wrong key or tampered)") from exc + + async def health(self) -> bool: + return True diff --git a/packages/core/src/rag_core/encrypting_storage.py b/packages/core/src/rag_core/encrypting_storage.py new file mode 100644 index 0000000..2d41bc0 --- /dev/null +++ b/packages/core/src/rag_core/encrypting_storage.py @@ -0,0 +1,61 @@ +"""EncryptingStorage — envelope-encrypt blobs at rest (Step 6.7). + +A drop-in :class:`~rag_core.spi.storage.Storage` decorator: it runs ``data`` +through a :class:`~rag_core.spi.key_manager.KeyManager` on :meth:`put` (so the +underlying store only ever holds ciphertext) and reverses it on :meth:`get`. +Everything else (``delete`` / ``exists`` / ``list_keys`` / ``health``) operates +on keys, not payloads, so it delegates verbatim. + +The decorator is itself **crypto-free** — all encryption lives in the injected +``KeyManager`` — so it composes with any storage backend and any KMS provider. +Wrap the production ``Storage`` with this when ``cfg.kms.enabled`` to get +encryption at rest for chunk content / blobs (the sensitive payloads); embedding +vectors are stored separately and stay plaintext for search. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator + +from rag_core.spi.key_manager import KeyManager +from rag_core.spi.storage import Storage +from rag_core.types import RequestContext + +__all__ = ["EncryptingStorage"] + + +class EncryptingStorage(Storage): + """Decorate a ``Storage`` so blobs are envelope-encrypted at rest.""" + + def __init__(self, inner: Storage, key_manager: KeyManager) -> None: + self._inner = inner + self._km = key_manager + + @property + def inner(self) -> Storage: + return self._inner + + async def put( + self, + ctx: RequestContext, + key: str, + data: bytes, + content_type: str | None = None, + ) -> None: + ciphertext = await self._km.encrypt(ctx, data) + await self._inner.put(ctx, key, ciphertext, content_type) + + async def get(self, ctx: RequestContext, key: str) -> bytes: + return await self._km.decrypt(ctx, await self._inner.get(ctx, key)) + + async def delete(self, ctx: RequestContext, key: str) -> None: + await self._inner.delete(ctx, key) + + async def exists(self, ctx: RequestContext, key: str) -> bool: + return await self._inner.exists(ctx, key) + + def list_keys(self, ctx: RequestContext, prefix: str = "") -> AsyncIterator[str]: + return self._inner.list_keys(ctx, prefix) + + async def health(self) -> bool: + return await self._inner.health() and await self._km.health() diff --git a/packages/core/src/rag_core/errors.py b/packages/core/src/rag_core/errors.py index 8c8fea7..7cf5b50 100644 --- a/packages/core/src/rag_core/errors.py +++ b/packages/core/src/rag_core/errors.py @@ -291,3 +291,29 @@ class AuditNotFoundError(AuditError): """ code = "audit_not_found" + + +# --------------------------------------------------------------------------- +# Encryption / BYOK (Step 6.7) +# --------------------------------------------------------------------------- +class EncryptionError(RagError): + """Envelope encryption / decryption failed (Step 6.7). + + Raised when a ciphertext is malformed or fails authentication — tampering, or + a ciphertext presented under the wrong tenant. Distinct from + :class:`KeyUnavailableError`, which is the *sealing* case (the key itself is + unreachable). + """ + + code = "encryption_error" + + +class KeyUnavailableError(EncryptionError): + """A tenant's key-encryption key (KEK) is unavailable (Step 6.7 — sealing). + + Raised when the customer-controlled KMS key for ``ctx.tenant_id`` is revoked, + not configured, or unreachable. By design this **seals** that tenant's data + (it becomes unreadable) with no cross-tenant impact. + """ + + code = "key_unavailable" diff --git a/packages/core/src/rag_core/spi/__init__.py b/packages/core/src/rag_core/spi/__init__.py index 0a00399..77f328d 100644 --- a/packages/core/src/rag_core/spi/__init__.py +++ b/packages/core/src/rag_core/spi/__init__.py @@ -26,6 +26,7 @@ GraphRetrievalBackend, GraphStore, ) +from rag_core.spi.key_manager import KeyManager from rag_core.spi.keyword_store import ( KeywordIndexBackend, KeywordRetrievalBackend, @@ -68,6 +69,7 @@ "GraphIndexBackend", "GraphRetrievalBackend", "GraphStore", + "KeyManager", "KeywordIndexBackend", "KeywordRetrievalBackend", "KeywordStore", diff --git a/packages/core/src/rag_core/spi/key_manager.py b/packages/core/src/rag_core/spi/key_manager.py new file mode 100644 index 0000000..9853c97 --- /dev/null +++ b/packages/core/src/rag_core/spi/key_manager.py @@ -0,0 +1,54 @@ +"""KeyManager SPI — envelope encryption for BYOK (Step 6.7). + +A ``KeyManager`` encrypts and decrypts opaque byte payloads using **envelope +encryption**: a fresh data-encryption key (DEK) per payload, wrapped by a +**per-tenant** key-encryption key (KEK) held in a customer-controlled KMS +(AWS KMS / GCP KMS / Azure Key Vault / HashiCorp Vault — "Bring Your Own Key"). +The DEK does the bulk symmetric encryption; only the small DEK is sent to the KMS +to be wrapped/unwrapped, so the KEK never leaves the customer's control. + +The ciphertext returned by :meth:`encrypt` is a self-describing envelope +(wrapped DEK + nonces + ciphertext) — callers treat it as **opaque bytes** and +store it wherever they would have stored the plaintext (e.g. behind the +:class:`~rag_core.spi.storage.Storage` SPI via +:class:`~rag_core.encrypting_storage.EncryptingStorage`). + +Per-tenant isolation + **sealing**: the KEK is resolved from ``ctx.tenant_id``; +if a tenant's KEK is revoked or unreachable, :meth:`decrypt` raises +:class:`~rag_core.errors.KeyUnavailableError` and that tenant's data becomes +unreadable — with no cross-tenant impact. A tampered or wrong-tenant ciphertext +fails authentication and raises :class:`~rag_core.errors.EncryptionError`. + +Embedding *vectors* are intentionally **not** the target of encryption (they must +stay plaintext for similarity search); the sensitive **chunk content / blobs** at +rest are. See [ADR-0039](../../../../../../docs/adr/ADR-0039-byok-envelope-encryption.md). +""" + +from __future__ import annotations + +import abc + +from rag_core.spi._base import HealthCheckMixin +from rag_core.types import RequestContext + + +class KeyManager(HealthCheckMixin, abc.ABC): + """Abstract per-tenant envelope-encryption provider (BYOK).""" + + @abc.abstractmethod + async def encrypt(self, ctx: RequestContext, plaintext: bytes) -> bytes: + """Envelope-encrypt ``plaintext`` for ``ctx.tenant_id`` → opaque ciphertext. + + Raises: + KeyUnavailableError: the tenant's KEK is unavailable (sealed / unreachable). + """ + + @abc.abstractmethod + async def decrypt(self, ctx: RequestContext, ciphertext: bytes) -> bytes: + """Decrypt an envelope produced by :meth:`encrypt`. + + Raises: + KeyUnavailableError: the tenant's KEK is unavailable (sealing). + EncryptionError: the ciphertext is malformed or fails authentication + (tampering, or it belongs to a different tenant). + """ diff --git a/packages/core/src/rag_core/spi/noop/__init__.py b/packages/core/src/rag_core/spi/noop/__init__.py index 325b6be..ff3faad 100644 --- a/packages/core/src/rag_core/spi/noop/__init__.py +++ b/packages/core/src/rag_core/spi/noop/__init__.py @@ -12,6 +12,7 @@ from rag_core.spi.noop.enricher import NoopEnricher from rag_core.spi.noop.feedback_store import NoopFeedbackStore from rag_core.spi.noop.graph_store import NoopGraphStore +from rag_core.spi.noop.key_manager import NoopKeyManager from rag_core.spi.noop.keyword_store import NoopKeywordStore from rag_core.spi.noop.llm import NoopLLM from rag_core.spi.noop.nli import NoopNLIScorer @@ -43,6 +44,7 @@ "NoopEnricher", "NoopFeedbackStore", "NoopGraphStore", + "NoopKeyManager", "NoopKeywordStore", "NoopLLM", "NoopNLIScorer", diff --git a/packages/core/src/rag_core/spi/noop/key_manager.py b/packages/core/src/rag_core/spi/noop/key_manager.py new file mode 100644 index 0000000..d01d30a --- /dev/null +++ b/packages/core/src/rag_core/spi/noop/key_manager.py @@ -0,0 +1,26 @@ +"""Passthrough noop KeyManager — performs NO encryption. + +Returns plaintext unchanged from both :meth:`encrypt` and :meth:`decrypt`, so the +:class:`~rag_core.encrypting_storage.EncryptingStorage` wiring can be exercised in +tests / local dev without a real KMS. **Never use in production** — it provides +no confidentiality. A real provider (e.g. ``LocalKeyManager`` in ``rag-backends``, +or a cloud KMS) supplies actual envelope encryption. +""" + +from __future__ import annotations + +from rag_core.spi.key_manager import KeyManager +from rag_core.types import RequestContext + + +class NoopKeyManager(KeyManager): + """No-op (identity) key manager — passthrough, no confidentiality.""" + + async def encrypt(self, ctx: RequestContext, plaintext: bytes) -> bytes: + return plaintext + + async def decrypt(self, ctx: RequestContext, ciphertext: bytes) -> bytes: + return ciphertext + + async def health(self) -> bool: + return True diff --git a/packages/ragctl/src/ragctl/main.py b/packages/ragctl/src/ragctl/main.py index 8937bec..dd8c329 100644 --- a/packages/ragctl/src/ragctl/main.py +++ b/packages/ragctl/src/ragctl/main.py @@ -4885,6 +4885,69 @@ def audit( typer.echo(f" written: {out}") +@app.command("kms") +def kms( + text: str = typer.Option("PII: alice@example.com", "--text", help="Sample plaintext."), + tenant: str = typer.Option("acme", "--tenant", help="Tenant to encrypt for."), +) -> None: + """Demo Step 6.7 BYOK envelope encryption with an in-process LocalKeyManager. + + Builds per-tenant KEKs, encrypts → decrypts (round-trip), then shows the two + BYOK guarantees: per-tenant **isolation** (another tenant can't read the blob) + and **sealing** (a tenant with no KEK is denied with a typed error). No + external services / KMS. + + Example:: + + ragctl kms --tenant acme --text "secret" + """ + import asyncio + import os + + from rag_backends import LocalKeyManager + from rag_core.errors import EncryptionError, KeyUnavailableError + from rag_core.types import ( + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, + ) + + def _ctx(t: str) -> RequestContext: + tid = TenantId(t) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("cli"), kind=PrincipalKind.user, display_name="cli", tenant_id=tid + ), + ) + + km = LocalKeyManager(keks={tenant: os.urandom(32), "other": os.urandom(32)}) + pt = text.encode("utf-8") + + async def _run() -> None: + ct = await km.encrypt(_ctx(tenant), pt) + rt = await km.decrypt(_ctx(tenant), ct) + typer.echo(f"\nBYOK envelope encryption — tenant={tenant}") + typer.echo("─" * 64) + typer.echo(f" plaintext: {len(pt)} bytes") + typer.echo(f" ciphertext: {len(ct)} bytes (opaque envelope)") + typer.echo(f" round-trip: {'ok' if rt == pt else 'FAIL'}") + try: + await km.decrypt(_ctx("other"), ct) + typer.echo(" isolation: FAIL (cross-tenant read)") + except EncryptionError: + typer.echo(" isolation: ok (another tenant cannot read the blob)") + try: + await km.encrypt(_ctx("ghost"), pt) + typer.echo(" sealing: FAIL") + except KeyUnavailableError: + typer.echo(" sealing: ok (tenant with no KEK is denied)") + + asyncio.run(_run()) + + def main() -> None: app() diff --git a/packages/ragctl/tests/test_kms.py b/packages/ragctl/tests/test_kms.py new file mode 100644 index 0000000..fa39927 --- /dev/null +++ b/packages/ragctl/tests/test_kms.py @@ -0,0 +1,20 @@ +"""Tests for ``ragctl kms`` — Step 6.7. + +Smoke command builds an in-process LocalKeyManager, encrypts → decrypts a sample, +and demonstrates per-tenant isolation + sealing. No infrastructure. +""" + +from __future__ import annotations + +from ragctl.main import app +from typer.testing import CliRunner + +runner = CliRunner() + + +def test_kms_demo_round_trip_isolation_sealing() -> None: + result = runner.invoke(app, ["kms", "--tenant", "acme", "--text", "secret"]) + assert result.exit_code == 0, result.output + assert "round-trip: ok" in result.output + assert "isolation: ok" in result.output + assert "sealing: ok" in result.output diff --git a/tests/contract/test_key_manager.py b/tests/contract/test_key_manager.py new file mode 100644 index 0000000..3c6cb5f --- /dev/null +++ b/tests/contract/test_key_manager.py @@ -0,0 +1,56 @@ +"""Conformance tests for the KeyManager SPI (Step 6.7). + +The contract every KeyManager honours: ``decrypt(encrypt(x)) == x`` and a +truthy ``health()``. Crypto-specific behaviour (tamper / sealing / per-tenant +isolation) is exercised against the real ``LocalKeyManager`` in +``tests/kms/test_envelope.py``. +""" + +from __future__ import annotations + +import os + +import pytest +from rag_backends import LocalKeyManager +from rag_core.spi.key_manager import KeyManager +from rag_core.spi.noop import NoopKeyManager +from rag_core.types import ( + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) + +pytestmark = pytest.mark.contract + + +def _ctx(tenant: str = "acme") -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), kind=PrincipalKind.user, display_name="p", tenant_id=tid + ), + ) + + +def _managers() -> list[KeyManager]: + return [NoopKeyManager(), LocalKeyManager(default_kek=os.urandom(32))] + + +@pytest.mark.parametrize("km", _managers()) +async def test_round_trip(km: KeyManager) -> None: + ctx = _ctx() + assert await km.decrypt(ctx, await km.encrypt(ctx, b"hello world")) == b"hello world" + + +@pytest.mark.parametrize("km", _managers()) +async def test_round_trip_empty(km: KeyManager) -> None: + ctx = _ctx() + assert await km.decrypt(ctx, await km.encrypt(ctx, b"")) == b"" + + +@pytest.mark.parametrize("km", _managers()) +async def test_health(km: KeyManager) -> None: + assert await km.health() is True diff --git a/tests/kms/test_envelope.py b/tests/kms/test_envelope.py new file mode 100644 index 0000000..9bfd0db --- /dev/null +++ b/tests/kms/test_envelope.py @@ -0,0 +1,153 @@ +"""Envelope encryption tests — LocalKeyManager + EncryptingStorage (Step 6.7). + +Covers the security-critical behaviours: confidentiality, tamper detection, +per-tenant isolation, and **sealing** (a tenant with no KEK can't read its data, +with no cross-tenant impact). +""" + +from __future__ import annotations + +import os + +import pytest +from rag_backends import LocalKeyManager +from rag_core.encrypting_storage import EncryptingStorage +from rag_core.errors import EncryptionError, KeyUnavailableError +from rag_core.spi.noop import NoopStorage +from rag_core.types import ( + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) + +_PLAINTEXT = b"PII: alice@example.com SSN 123-45-6789" + + +def _ctx(tenant: str) -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), kind=PrincipalKind.user, display_name="p", tenant_id=tid + ), + ) + + +def _km() -> LocalKeyManager: + return LocalKeyManager(keks={"acme": os.urandom(32), "globex": os.urandom(32)}) + + +# --------------------------------------------------------------------------- +# confidentiality + round-trip +# --------------------------------------------------------------------------- +async def test_ciphertext_differs_and_round_trips() -> None: + km = _km() + ctx = _ctx("acme") + ct = await km.encrypt(ctx, _PLAINTEXT) + assert ct != _PLAINTEXT + assert _PLAINTEXT not in ct # no plaintext leakage + assert await km.decrypt(ctx, ct) == _PLAINTEXT + + +async def test_encrypt_is_nondeterministic() -> None: + km = _km() + ctx = _ctx("acme") + # fresh DEK + nonce per call → distinct ciphertexts for the same plaintext + assert await km.encrypt(ctx, _PLAINTEXT) != await km.encrypt(ctx, _PLAINTEXT) + + +# --------------------------------------------------------------------------- +# tamper detection +# --------------------------------------------------------------------------- +async def test_tampered_ciphertext_is_rejected() -> None: + km = _km() + ctx = _ctx("acme") + ct = bytearray(await km.encrypt(ctx, _PLAINTEXT)) + ct[-1] ^= 0x01 # flip a bit in the ciphertext body + with pytest.raises(EncryptionError): + await km.decrypt(ctx, bytes(ct)) + + +async def test_malformed_envelope_is_rejected() -> None: + km = _km() + with pytest.raises(EncryptionError): + await km.decrypt(_ctx("acme"), b"not-an-envelope") + + +# --------------------------------------------------------------------------- +# per-tenant isolation +# --------------------------------------------------------------------------- +async def test_cross_tenant_decrypt_is_blocked() -> None: + km = _km() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + # globex has its own KEK + the AAD binds the blob to acme → unreadable. + with pytest.raises(EncryptionError): + await km.decrypt(_ctx("globex"), ct) + + +async def test_same_kek_different_tenant_still_blocked() -> None: + # Even if two tenants shared a KEK, the tenant-bound AAD blocks cross-reads. + shared = os.urandom(32) + km = LocalKeyManager(keks={"acme": shared, "globex": shared}) + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + with pytest.raises(EncryptionError): + await km.decrypt(_ctx("globex"), ct) + + +# --------------------------------------------------------------------------- +# sealing — KEK unavailable +# --------------------------------------------------------------------------- +async def test_unknown_tenant_is_sealed_on_encrypt() -> None: + with pytest.raises(KeyUnavailableError): + await _km().encrypt(_ctx("ghost"), _PLAINTEXT) + + +async def test_unknown_tenant_is_sealed_on_decrypt() -> None: + km = _km() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + # Simulate KEK revocation: a manager without acme's KEK can't decrypt. + revoked = LocalKeyManager(keks={"globex": os.urandom(32)}) + with pytest.raises(KeyUnavailableError): + await revoked.decrypt(_ctx("acme"), ct) + + +async def test_wrong_size_kek_is_rejected() -> None: + km = LocalKeyManager(keks={"acme": b"too-short"}) + with pytest.raises(KeyUnavailableError): + await km.encrypt(_ctx("acme"), _PLAINTEXT) + + +async def test_sealing_one_tenant_does_not_affect_another() -> None: + km = LocalKeyManager(keks={"acme": os.urandom(32)}) # globex sealed + ctx = _ctx("acme") + assert await km.decrypt(ctx, await km.encrypt(ctx, _PLAINTEXT)) == _PLAINTEXT # acme fine + with pytest.raises(KeyUnavailableError): + await km.encrypt(_ctx("globex"), _PLAINTEXT) # globex sealed + + +# --------------------------------------------------------------------------- +# EncryptingStorage decorator +# --------------------------------------------------------------------------- +async def test_encrypting_storage_stores_ciphertext_and_round_trips() -> None: + inner = NoopStorage() + store = EncryptingStorage(inner, _km()) + ctx = _ctx("acme") + await store.put(ctx, "doc/1", _PLAINTEXT) + # the underlying store holds ciphertext, not plaintext + raw = await inner.get(ctx, "doc/1") + assert raw != _PLAINTEXT + assert _PLAINTEXT not in raw + # reading back through the decorator decrypts + assert await store.get(ctx, "doc/1") == _PLAINTEXT + + +async def test_encrypting_storage_delegates_key_ops() -> None: + inner = NoopStorage() + store = EncryptingStorage(inner, _km()) + ctx = _ctx("acme") + await store.put(ctx, "k", _PLAINTEXT) + assert await store.exists(ctx, "k") is True + await store.delete(ctx, "k") + assert await store.exists(ctx, "k") is False diff --git a/uv.lock b/uv.lock index 30d0e4f..5c9bd87 100644 --- a/uv.lock +++ b/uv.lock @@ -6944,6 +6944,7 @@ dependencies = [ { name = "aioboto3" }, { name = "aiofiles" }, { name = "asyncpg" }, + { name = "cryptography" }, { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, { name = "pgvector" }, @@ -7000,6 +7001,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.30" }, { name = "azure-identity", marker = "extra == 'azure'", specifier = ">=1.17" }, { name = "azure-storage-blob", marker = "extra == 'azure'", specifier = ">=12.22" }, + { name = "cryptography", specifier = ">=43.0" }, { name = "elasticsearch", extras = ["async"], marker = "extra == 'elasticsearch'", specifier = ">=8.15" }, { name = "gcloud-aio-storage", marker = "extra == 'gcs'", specifier = ">=9.3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=2.1" },