From 2657e54da5d7162904fec3e4da208321d5636712 Mon Sep 17 00:00:00 2001 From: Deep Kumar Singh Kushwah Date: Mon, 8 Jun 2026 09:12:28 +0530 Subject: [PATCH] feat(crypto): GCP / Azure / Vault KMS providers (Step 6.7c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the four-provider BYOK KMS matrix. Each new provider subclasses EnvelopeKeyManager (inheriting the DEK + AES-GCM + tenant-AAD + envelope framing) and only wraps/unwraps the DEK via its KMS API: - GcpKmsKeyManager — google-cloud-kms encrypt/decrypt, shared async client. - AzureKeyVaultKeyManager — azure-keyvault-keys wrap_key/unwrap_key (RSA-OAEP-256); a CryptographyClient per key via a cached factory (Azure binds a client to one key). - VaultKeyManager — HashiCorp Vault Transit encrypt_data/decrypt_data (sync hvac run in a thread, configurable mount_point). Each is behind a [kms-gcp] / [kms-azure] / [kms-vault] extra (SDKs lazy-imported, so the modules import without the SDK and selecting a provider without its extra raises a clear ImportError). An injectable client / client_factory seam makes every provider fully unit-testable with a fake KMS — no cloud creds/network. Connection uses each SDK's standard credential discovery; per-tenant key resolution + sealing (KeyUnavailableError) are uniform with AWS. KmsProvider enum + build_key_manager_from_config factory + KmsConfig.vault_mount extended; mypy overrides added for the three SDK module trees. Scope: providers only — key rotation is 6.7d; wiring EncryptingStorage into the ingest path stays deferred (tiered storage). Co-Authored-By: Claude Opus 4.7 --- TRACKER.md | 12 +- apps/gateway/src/rag_gateway/wiring.py | 35 ++- apps/gateway/tests/test_kms.py | 17 ++ dist/rag.schema.json | 12 +- dist/rag.schema.yaml | 22 +- docs/README.md | 4 +- docs/adr/ADR-0039-byok-envelope-encryption.md | 15 +- docs/architecture/byok.md | 9 +- docs/reference/encryption.md | 22 +- packages/backends/pyproject.toml | 6 + .../backends/src/rag_backends/__init__.py | 6 + .../backends/src/rag_backends/kms/__init__.py | 19 +- .../backends/src/rag_backends/kms/azure.py | 102 ++++++++ packages/backends/src/rag_backends/kms/gcp.py | 79 +++++++ .../backends/src/rag_backends/kms/vault.py | 96 ++++++++ packages/config/src/rag_config/schema.py | 12 +- pyproject.toml | 5 + tests/kms/test_cloud_providers.py | 186 +++++++++++++++ uv.lock | 218 +++++++++++++++++- 19 files changed, 837 insertions(+), 40 deletions(-) create mode 100644 packages/backends/src/rag_backends/kms/azure.py create mode 100644 packages/backends/src/rag_backends/kms/gcp.py create mode 100644 packages/backends/src/rag_backends/kms/vault.py create mode 100644 tests/kms/test_cloud_providers.py diff --git a/TRACKER.md b/TRACKER.md index e4b436a..2cbe592 100644 --- a/TRACKER.md +++ b/TRACKER.md @@ -16,10 +16,11 @@ | **Last updated** | 2026-06-08 | | **Current phase** | Phase 6 — Governance & Tenancy (**6 / 10 steps**) | | **Overall** | **70 / 84 steps** — Phases 0–5 complete | -| **Next action** | **Step 6.7c — GCP/Azure/Vault KMS + key rotation**: the remaining cloud providers (GCP KMS / Azure Key Vault / HashiCorp Vault, same `EnvelopeKeyManager` pattern, behind `[kms-*]` extras) + zero-downtime key rotation (old keys retained decrypt-only until expiry). Closes Step 6.7. 6.7a (library + local) + 6.7b (config/factory/wiring + AWS KMS) shipped. | +| **Next action** | **Step 6.7d — Key rotation**: zero-downtime KEK rotation — old keys retained decrypt-only until expiry, background re-encryption. Closes Step 6.7. 6.7a (library + local) + 6.7b (config/factory + AWS KMS) + 6.7c (GCP/Azure/Vault providers) shipped. | **Recently shipped** +- **6.7c** 🚧 GCP/Azure/Vault KMS providers — `GcpKmsKeyManager` (google-cloud-kms), `AzureKeyVaultKeyManager` (`wrap_key`/`unwrap_key`), `VaultKeyManager` (Vault Transit) — each subclasses `EnvelopeKeyManager` behind a `[kms-*]` extra (lazy SDK, injectable client, fake-client unit-tested); factory + enum extended; completes all four KMS providers — [#157](https://github.com/officialCodeWork/AgentContextOS/pull/157) - **6.7b** 🚧 Cloud KMS (AWS) + config/factory/wiring — `cfg.kms` (provider `noop`/`local`/`aws`) + `tenants[].kms_key_id`; `build_key_manager_from_config` factory + `app.state.key_manager` gateway seam; **`AwsKmsKeyManager`** (aioboto3, per-tenant CMK, KMS-error → sealing, fake-client unit-tested) — [#156](https://github.com/officialCodeWork/AgentContextOS/pull/156) - **6.7a** 🚧 BYOK envelope encryption library — `KeyManager` SPI + `NoopKeyManager` + `EncryptingStorage` (rag-core); `EnvelopeKeyManager` (AES-256-GCM DEK + `tenant_id` AAD) + `LocalKeyManager` (in-process per-tenant KEK) (rag-backends); per-tenant isolation + **sealing** (`KeyUnavailableError`) + tamper detection; `ragctl kms` — [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) - **6.6** ✅ Immutable audit log — **6.6a** shared `AuditWriter`/store on `app.state` + `GET /v1/audit` (tenant-scoped, `chain_verified`) + `GET /v1/audit/verify` (whole-log) + `cfg.audit.enabled` ([#153](https://github.com/officialCodeWork/AgentContextOS/pull/153)); **6.6b** `AuditExporter` self-verifying WORM bundle (SHA-256 `content_hash` + HMAC, offline `verify()`), `POST /v1/audit/export` (tenant-scoped) + `ragctl audit`, `cfg.audit.export_secret` ([#154](https://github.com/officialCodeWork/AgentContextOS/pull/154)) @@ -657,7 +658,7 @@ | 6.4 | ACL egress verifier | ✅ | [#151](https://github.com/officialCodeWork/AgentContextOS/pull/151) — `AclEgressVerifier` re-checks returned chunks at the gateway router boundary (defense-in-depth above the 6.3 push-down); `acl.egress_violation`; zero-violation-rate red-team gate | | 6.5 | PII policies | ✅ | [#152](https://github.com/officialCodeWork/AgentContextOS/pull/152) — `PiiPolicyEngine` egress_text decorator (allow / redact / mask / block per tenant) over answer + context; reuses Step 1.7 detector; `pii.egress_blocked` | | 6.6 | Immutable audit log | ✅ | **6.6a** [#153](https://github.com/officialCodeWork/AgentContextOS/pull/153) — read API `GET /v1/audit` + `GET /v1/audit/verify` + shared store + `cfg.audit`. **6.6b** [#154](https://github.com/officialCodeWork/AgentContextOS/pull/154) — `AuditExporter` signed WORM bundle + `POST /v1/audit/export` + `ragctl audit` | -| 6.7 | BYOK (Bring Your Own Key) | 🚧 | **6.7a** ✅ [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) — `KeyManager` SPI + envelope encryption (`LocalKeyManager`) + `EncryptingStorage` + sealing/isolation/tamper. **6.7b** ✅ [#156](https://github.com/officialCodeWork/AgentContextOS/pull/156) — `cfg.kms` + per-tenant key + `build_key_manager_from_config` factory + `AwsKmsKeyManager`. **6.7c** ⏳ — GCP/Azure/Vault + key rotation | +| 6.7 | BYOK (Bring Your Own Key) | 🚧 | **6.7a** ✅ [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) — `KeyManager` SPI + envelope encryption (`LocalKeyManager`) + `EncryptingStorage` + sealing/isolation/tamper. **6.7b** ✅ [#156](https://github.com/officialCodeWork/AgentContextOS/pull/156) — `cfg.kms` + per-tenant key + `build_key_manager_from_config` factory + `AwsKmsKeyManager`. **6.7c** ✅ [#157](https://github.com/officialCodeWork/AgentContextOS/pull/157) — `GcpKmsKeyManager` / `AzureKeyVaultKeyManager` / `VaultKeyManager` behind `[kms-*]` extras. **6.7d** ⏳ — key rotation | | 6.8 | SSO / SCIM | ⏳ | OIDC + SAML IdP federation; SCIM 2.0 user provisioning; per-tenant IdP config | | 6.9 | Air-gapped install bundle | ⏳ | Signed tarball with all images + Helm chart; offline bootstrap; cosign verification | | 6.10 | Compliance posture | ⏳ | SOC 2 Type II control mapping; GDPR data-residency config; data-retention policies | @@ -747,6 +748,12 @@ New ground — the only prior crypto was HMAC signing. The V1 plan calls for en - **`AwsKmsKeyManager`** (`rag-backends/kms/aws.py`) subclasses `EnvelopeKeyManager` and wraps/unwraps the DEK via **AWS KMS** Encrypt/Decrypt for the tenant's customer-managed key (over the already-present `aioboto3` — **no new dep**). Per-tenant CMK from the key-id map (+ `default_key_id`); any KMS failure (revoked / denied / unreachable) or a missing key id → **`KeyUnavailableError`** (sealing). An **injectable `client` seam** makes it fully unit-testable with a fake KMS — no AWS creds / network - **Scope:** AWS provider + config + factory + the `app.state.key_manager` seam. **Deferred:** GCP KMS / Azure Key Vault / HashiCorp Vault providers + key rotation (6.7c); wiring `EncryptingStorage` into the ingest path (blocked on tiered-storage plumbing — chunk content is inline today). ~14 new tests (AWS provider over a fake KMS: round-trip / KMS-key-bound cross-tenant block / sealing / default-key / KMS-error mapping; config defaults; factory noop/local/aws; gateway `app.state.key_manager` + `kms_enabled` + a local round-trip). `KmsConfig` → `rag.schema` regenerated; all gates green (ruff, mypy --strict 306 files, RAG001, policy-coverage, log-schema). [ADR-0039](docs/adr/ADR-0039-byok-envelope-encryption.md), [reference/encryption.md](docs/reference/encryption.md), [architecture/byok.md](docs/architecture/byok.md) +#### 6.7c — GCP / Azure / Vault KMS providers ✅ [#157](https://github.com/officialCodeWork/AgentContextOS/pull/157) + +- Completes the **four-provider KMS matrix** — each subclasses `EnvelopeKeyManager` (inheriting the DEK + AES-GCM + tenant-AAD + envelope framing) and only wraps/unwraps the DEK via its KMS API: **`GcpKmsKeyManager`** (google-cloud-kms `encrypt`/`decrypt`), **`AzureKeyVaultKeyManager`** (azure-keyvault-keys `wrap_key`/`unwrap_key`, RSA-OAEP-256; a `CryptographyClient` per key via a cached factory since Azure binds a client to one key), **`VaultKeyManager`** (HashiCorp Vault Transit `encrypt_data`/`decrypt_data`, sync `hvac` run in a thread, `mount_point`) +- Each is **behind a `[kms-gcp]` / `[kms-azure]` / `[kms-vault]` extra** (the SDKs are lazy-imported, so the modules import without the SDK and selecting a provider without its extra raises a clear `ImportError`); an **injectable `client` / `client_factory` seam** makes every provider fully unit-testable with a fake KMS — no cloud creds / network. Connection uses each SDK's standard credential discovery (GCP ADC, Azure `DefaultAzureCredential`, Vault `VAULT_ADDR`/`VAULT_TOKEN`); per-tenant key resolution + sealing (`KeyUnavailableError`) are uniform with AWS +- `KmsProvider` enum + `build_key_manager_from_config` factory + `KmsConfig.vault_mount` extended. mypy overrides added for the three SDK module trees (absent at lint time). **Scope:** providers only — key rotation is **6.7d**; wiring `EncryptingStorage` into the ingest path stays deferred (tiered storage). ~18 new tests (each provider over a fake client: round-trip / cross-tenant block / sealing / SDK-error mapping; factory cloud-provider-requires-extra contract). `KmsConfig` → `rag.schema` regenerated; all gates green (ruff, mypy --strict 309 files, RAG001, policy-coverage, log-schema). [ADR-0039](docs/adr/ADR-0039-byok-envelope-encryption.md), [reference/encryption.md](docs/reference/encryption.md), [architecture/byok.md](docs/architecture/byok.md) + --- ## Phase 7 — Pilot, Harden, GA (Weeks 34–40) ⏳ @@ -907,6 +914,7 @@ Complete log of every PR. Routine Dependabot bumps are grouped; everything else | [#154](https://github.com/officialCodeWork/AgentContextOS/pull/154) | 2026-06-08 | feat(audit): WORM signed export — AuditExporter + POST /v1/audit/export (Step 6.6b) | | [#155](https://github.com/officialCodeWork/AgentContextOS/pull/155) | 2026-06-08 | feat(crypto): BYOK envelope encryption library + local KMS (Step 6.7a) | | [#156](https://github.com/officialCodeWork/AgentContextOS/pull/156) | 2026-06-08 | feat(crypto): cfg.kms + key-manager factory + AWS KMS provider (Step 6.7b) | +| [#157](https://github.com/officialCodeWork/AgentContextOS/pull/157) | 2026-06-08 | feat(crypto): GCP / Azure / Vault KMS providers (Step 6.7c) | | #78–#80, #116–#118 | Open | Dependabot bumps — awaiting merge | | #81 | Closed | Dependabot bump — superseded | diff --git a/apps/gateway/src/rag_gateway/wiring.py b/apps/gateway/src/rag_gateway/wiring.py index ea7b090..da4b029 100644 --- a/apps/gateway/src/rag_gateway/wiring.py +++ b/apps/gateway/src/rag_gateway/wiring.py @@ -157,10 +157,11 @@ def build_key_manager_from_config(cfg: RagConfig) -> Any: Returns a passthrough ``NoopKeyManager`` when disabled / ``provider=noop``; a ``LocalKeyManager`` (in-process per-tenant KEKs decoded from - ``tenants[].kms_key_id`` hex) for ``local``; an ``AwsKmsKeyManager`` (per-tenant - KMS key ARNs) for ``aws``. GCP / Azure / Vault land in Step 6.7c. Each - tenant's key is ``tenants[].kms_key_id`` with ``kms.default_key_id`` as the - fallback; a tenant with neither is *sealed* at encrypt/decrypt time. + ``tenants[].kms_key_id`` hex) for ``local``; or a cloud KMS provider — + ``AwsKmsKeyManager`` / ``GcpKmsKeyManager`` / ``AzureKeyVaultKeyManager`` / + ``VaultKeyManager`` — keyed by ``tenants[].kms_key_id`` (an ARN / resource name + / key URL / transit key name). Each tenant's key has ``kms.default_key_id`` as + the fallback; a tenant with neither is *sealed* at encrypt/decrypt time. """ from rag_config.schema import KmsProvider from rag_core.spi.noop import NoopKeyManager @@ -176,12 +177,32 @@ def build_key_manager_from_config(cfg: RagConfig) -> Any: default = bytes.fromhex(kms.local_key) if kms.local_key else None return LocalKeyManager(keks=keks, default_kek=default) + # Cloud providers share the per-tenant key-id map (ARN / resource name / key + # URL / transit key) + default. Connection comes from each SDK's standard + # credential discovery; the provider modules import their SDK lazily. + key_ids = {t.id: t.kms_key_id for t in cfg.tenants if t.kms_key_id} + default_key_id = kms.default_key_id or None + if kms.provider is KmsProvider.AWS: from rag_backends import AwsKmsKeyManager - key_ids = {t.id: t.kms_key_id for t in cfg.tenants if t.kms_key_id} - return AwsKmsKeyManager( - key_ids=key_ids, default_key_id=kms.default_key_id or None, region=kms.region + return AwsKmsKeyManager(key_ids=key_ids, default_key_id=default_key_id, region=kms.region) + + if kms.provider is KmsProvider.GCP: + from rag_backends import GcpKmsKeyManager + + return GcpKmsKeyManager(key_ids=key_ids, default_key_id=default_key_id) + + if kms.provider is KmsProvider.AZURE: + from rag_backends import AzureKeyVaultKeyManager + + return AzureKeyVaultKeyManager(key_ids=key_ids, default_key_id=default_key_id) + + if kms.provider is KmsProvider.VAULT: + from rag_backends import VaultKeyManager + + return VaultKeyManager( + key_ids=key_ids, default_key_id=default_key_id, mount_point=kms.vault_mount ) return NoopKeyManager() # pragma: no cover - providers above are exhaustive diff --git a/apps/gateway/tests/test_kms.py b/apps/gateway/tests/test_kms.py index b9ea51a..79db650 100644 --- a/apps/gateway/tests/test_kms.py +++ b/apps/gateway/tests/test_kms.py @@ -45,6 +45,23 @@ def test_factory_aws() -> None: assert isinstance(build_key_manager_from_config(cfg), AwsKmsKeyManager) +def test_factory_gcp_requires_extra() -> None: + # Selecting a cloud provider whose SDK extra isn't installed raises a clear + # ImportError (in CI the [kms-gcp] extra is absent). + try: + import pytest + from google.cloud import kms_v1 # noqa: F401 + + pytest.skip("kms-gcp extra installed") + except ImportError: + pass + cfg = RagConfig(kms=KmsConfig(enabled=True, provider=KmsProvider.GCP, default_key_id="k")) + import pytest + + with pytest.raises(ImportError): + build_key_manager_from_config(cfg) + + # --------------------------------------------------------------------------- # gateway wiring # --------------------------------------------------------------------------- diff --git a/dist/rag.schema.json b/dist/rag.schema.json index 803988e..3b328cb 100644 --- a/dist/rag.schema.json +++ b/dist/rag.schema.json @@ -892,7 +892,7 @@ }, "KmsConfig": { "additionalProperties": false, - "description": "BYOK envelope encryption (Step 6.7).\n\nWhen ``enabled`` the gateway builds a ``KeyManager`` (selected by ``provider``)\nand exposes it on ``app.state.key_manager``; wrap a ``Storage`` backend with\n``EncryptingStorage`` to encrypt chunk content / blobs **at rest** with\nper-tenant, customer-controlled keys (embedding vectors stay plaintext for\nsearch). Each tenant's key reference is ``tenants[].kms_key_id`` (a KMS key\nARN for ``aws``, a hex 32-byte KEK for ``local``); a tenant with no key \u2014\nand no ``default_key_id`` \u2014 is **sealed** (``KeyUnavailableError``).\n\n**Disabled by default.** ``provider``: ``local`` (in-process KEK, dev /\nair-gapped), ``aws`` (AWS KMS), or ``noop`` (passthrough). GCP / Azure /\nVault providers + key rotation land in Step 6.7c.\n\n* ``default_key_id`` \u2014 fallback key reference for tenants without their own.\n* ``region`` \u2014 AWS region for the ``aws`` provider.\n* ``local_key`` \u2014 hex 32-byte default KEK for the ``local`` provider\n (``${ENV_VAR}``-interpolated; keep it out of the file).", + "description": "BYOK envelope encryption (Step 6.7).\n\nWhen ``enabled`` the gateway builds a ``KeyManager`` (selected by ``provider``)\nand exposes it on ``app.state.key_manager``; wrap a ``Storage`` backend with\n``EncryptingStorage`` to encrypt chunk content / blobs **at rest** with\nper-tenant, customer-controlled keys (embedding vectors stay plaintext for\nsearch). Each tenant's key reference is ``tenants[].kms_key_id`` (a KMS key\nARN for ``aws``, a hex 32-byte KEK for ``local``); a tenant with no key \u2014\nand no ``default_key_id`` \u2014 is **sealed** (``KeyUnavailableError``).\n\n**Disabled by default.** ``provider``: ``local`` (in-process KEK, dev /\nair-gapped), ``aws`` / ``gcp`` / ``azure`` / ``vault`` (customer-controlled\nKMS), or ``noop`` (passthrough). Cloud-provider connection uses standard\ncredential discovery (AWS chain, GCP ADC, Azure ``DefaultAzureCredential``,\nVault ``VAULT_ADDR`` / ``VAULT_TOKEN``); the per-tenant key reference is\n``tenants[].kms_key_id``. Key rotation lands in Step 6.7d.\n\n* ``default_key_id`` \u2014 fallback key reference for tenants without their own.\n* ``region`` \u2014 AWS region for the ``aws`` provider.\n* ``local_key`` \u2014 hex 32-byte default KEK for the ``local`` provider\n (``${ENV_VAR}``-interpolated; keep it out of the file).\n* ``vault_mount`` \u2014 Transit secrets-engine mount path for the ``vault`` provider.", "properties": { "enabled": { "default": false, @@ -917,6 +917,11 @@ "default": "", "title": "Local Key", "type": "string" + }, + "vault_mount": { + "default": "transit", + "title": "Vault Mount", + "type": "string" } }, "title": "KmsConfig", @@ -927,7 +932,10 @@ "enum": [ "noop", "local", - "aws" + "aws", + "gcp", + "azure", + "vault" ], "title": "KmsProvider", "type": "string" diff --git a/dist/rag.schema.yaml b/dist/rag.schema.yaml index 81e35e1..15dcf73 100644 --- a/dist/rag.schema.yaml +++ b/dist/rag.schema.yaml @@ -931,12 +931,15 @@ $defs: \ (a KMS key\nARN for ``aws``, a hex 32-byte KEK for ``local``); a tenant with\ \ no key —\nand no ``default_key_id`` — is **sealed** (``KeyUnavailableError``).\n\ \n**Disabled by default.** ``provider``: ``local`` (in-process KEK, dev /\n\ - air-gapped), ``aws`` (AWS KMS), or ``noop`` (passthrough). GCP / Azure /\n\ - Vault providers + key rotation land in Step 6.7c.\n\n* ``default_key_id`` —\ - \ fallback key reference for tenants without their own.\n* ``region`` — AWS\ - \ region for the ``aws`` provider.\n* ``local_key`` — hex 32-byte default KEK\ - \ for the ``local`` provider\n (``${ENV_VAR}``-interpolated; keep it out of\ - \ the file)." + air-gapped), ``aws`` / ``gcp`` / ``azure`` / ``vault`` (customer-controlled\n\ + KMS), or ``noop`` (passthrough). Cloud-provider connection uses standard\n\ + credential discovery (AWS chain, GCP ADC, Azure ``DefaultAzureCredential``,\n\ + Vault ``VAULT_ADDR`` / ``VAULT_TOKEN``); the per-tenant key reference is\n``tenants[].kms_key_id``.\ + \ Key rotation lands in Step 6.7d.\n\n* ``default_key_id`` — fallback key reference\ + \ for tenants without their own.\n* ``region`` — AWS region for the ``aws``\ + \ provider.\n* ``local_key`` — hex 32-byte default KEK for the ``local`` provider\n\ + \ (``${ENV_VAR}``-interpolated; keep it out of the file).\n* ``vault_mount``\ + \ — Transit secrets-engine mount path for the ``vault`` provider." properties: enabled: default: false @@ -957,6 +960,10 @@ $defs: default: '' title: Local Key type: string + vault_mount: + default: transit + title: Vault Mount + type: string title: KmsConfig type: object KmsProvider: @@ -965,6 +972,9 @@ $defs: - noop - local - aws + - gcp + - azure + - vault title: KmsProvider type: string LLMConfig: diff --git a/docs/README.md b/docs/README.md index 54e50ac..9bbcdd1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,7 +10,7 @@ | [multi-tenancy.md](architecture/multi-tenancy.md) | Logical multi-tenancy (Step 6.1): make per-tenant `rag.yaml` config drive requests. `TenantResolver` (`rag_config.tenancy`) maps a tenant id → frozen `TenantSettings` (namespace / pii_policy / acl_labels), applied **once** at the gateway boundary (namespace + pii_policy onto the `RequestContext`, acl_labels unioned into the principal); unknown tenants resolve to safe defaults (namespace = id, default PII, no labels — isolated not privileged); `RequestContext.namespace` defaults to `tenant_id` (Pinecone partitions on it; `filter_pushdown` unchanged); resolves+threads only — ACL push-down is 6.3, PII egress 6.5, physical tenancy 6.2; `GET /v1/status/tenant`; inert in `build_app` | | [policy-engine.md](architecture/policy-engine.md) | `PolicyEngine` (PDP) — single decision point for ACL, PII, quotas, redaction; replaces scattered governance checks | | [audit-log.md](architecture/audit-log.md) | Immutable audit log (Step 6.6): tamper-evidence (SHA-256 hash chain) vs immutability-at-rest (WORM export, 6.6b); one shared `AuditWriter`/store on `app.state`; read-path tenant scoping (tenant-scoped list vs whole-log verify); why the read API defaults on | -| [byok.md](architecture/byok.md) | BYOK / envelope encryption (Step 6.7): what's encrypted (chunk content at rest) vs not (embedding vectors — search needs plaintext); DEK+KEK envelope (client-side AES-GCM DEK, provider wraps the DEK); per-tenant isolation via KEK + `tenant_id` AAD; sealing as a typed error; rag-core/rag-backends split; slicing (6.7a library, 6.7b config + factory + AWS KMS, 6.7c GCP/Azure/Vault + rotation) | +| [byok.md](architecture/byok.md) | BYOK / envelope encryption (Step 6.7): what's encrypted (chunk content at rest) vs not (embedding vectors — search needs plaintext); DEK+KEK envelope (client-side AES-GCM DEK, provider wraps the DEK); per-tenant isolation via KEK + `tenant_id` AAD; sealing as a typed error; rag-core/rag-backends split; slicing (6.7a library, 6.7b config + factory + AWS KMS, 6.7c GCP/Azure/Vault, 6.7d rotation) | | [caching.md](architecture/caching.md) | Three-cache split: `EmbeddingCache`, `RetrievalCache`, `AnswerCache` — distinct invalidation rules | | [performance.md](architecture/performance.md) | Hot-path discipline, per-SPI p99 budgets, async telemetry, reviewer checklist | | [pipeline-batcher.md](architecture/pipeline-batcher.md) | `Pipeline` (async DAG, bounded queues, per-stage workers) + `Batcher` (DataLoader-pattern coalescing) primitives — Step 1.1d | @@ -107,7 +107,7 @@ | [admin-ui.md](reference/admin-ui.md) | Admin console (Step 3.10) — Next.js 14 operator GUI (`apps/admin-ui`); 9 pages (dashboard, corpora, connectors, glossary, webhooks, audit, API keys, tenants, config), live-vs-seed hybrid + `NEXT_PUBLIC_GATEWAY_URL`, header identity, running it, internals (shell/primitives/data layer), extension points | | [tenancy.md](reference/tenancy.md) | Logical multi-tenancy (Step 6.1) — per-tenant `rag.yaml` config (`namespace` / `acl_labels` / `pii_policy` / `quota`); `TenantResolver.resolve(id) → TenantSettings`; `RequestContext.namespace`; `GET /v1/status/tenant`; `ragctl tenant list` / `resolve`; config table + scope/boundaries (6.2/6.3/6.5) + extension points; physical tenancy (6.2), ACL push-down (6.3) + egress verifier (6.4 — `cfg.acl.verify_egress`) sections | | [audit.md](reference/audit.md) | Audit log (Step 6.6) — `AuditEvent` / `AuditStore` (append / events / verify_chain) / `NoopAuditStore` SHA-256 hash chain / `AuditWriter` (+ `.store`); read API `GET /v1/audit` (tenant-scoped, `chain_verified`) + `GET /v1/audit/verify` (whole-log); WORM signed export (6.6b) — `AuditExporter` (content_hash + HMAC), `POST /v1/audit/export`, offline `verify()`, `ragctl audit`; `cfg.audit.enabled` / `export_secret`; durable-store extension points | -| [encryption.md](reference/encryption.md) | BYOK envelope encryption (Step 6.7) — `KeyManager` SPI + `NoopKeyManager`; `EnvelopeKeyManager` (AES-256-GCM DEK + `tenant_id` AAD) + `LocalKeyManager` + **`AwsKmsKeyManager`** (6.7b); `EncryptingStorage` decorator; `EncryptionError` / `KeyUnavailableError` (sealing); guarantees table; `cfg.kms` + `tenants[].kms_key_id` + `build_key_manager_from_config` factory; `ragctl kms`; GCP/Azure/Vault extension points (6.7c) | +| [encryption.md](reference/encryption.md) | BYOK envelope encryption (Step 6.7) — `KeyManager` SPI + `NoopKeyManager`; `EnvelopeKeyManager` (AES-256-GCM DEK + `tenant_id` AAD) + `LocalKeyManager` + cloud providers **`AwsKmsKeyManager`** / **`GcpKmsKeyManager`** / **`AzureKeyVaultKeyManager`** / **`VaultKeyManager`** (behind `[kms-*]` extras); `EncryptingStorage` decorator; `EncryptionError` / `KeyUnavailableError` (sealing); provider table; `cfg.kms` + `tenants[].kms_key_id` + `build_key_manager_from_config` factory; `ragctl kms`; rotation deferred (6.7d) | | [webhooks.md](reference/webhooks.md) | Outbound webhooks (Step 3.9) — event catalogue (`ingest.completed` / `audit.policy_violation` / `drift.detected` / `eval.regression`), event envelope, HMAC signing + `verify()`, at-least-once delivery, `/v1/webhooks/subscriptions` CRUD + test, `rag.yaml` block, `ragctl webhooks demo`, internals + extension points | | [integrations.md](reference/integrations.md) | Framework adapters (Step 3.8) — `agentcontextos.integrations.*` for LangChain / LlamaIndex / Haystack / DSPy / LangGraph / CrewAI / AutoGen / Semantic Kernel; per-framework extras, shared config + chunk metadata, usage per framework, internals + extension points | | [status-api.md](reference/status-api.md) | Status & Metrics API (Step 3.11) — `/v1/status/health` / `metrics` / `logs` (+ SSE `logs/stream`), `WS /v1/status/ws`, `/v1/connectors/status`; metric catalogue + request-timing middleware, the `MetricsCollector` / `LogTail` read-side, CORS + query-param identity for browser streams, extension points | diff --git a/docs/adr/ADR-0039-byok-envelope-encryption.md b/docs/adr/ADR-0039-byok-envelope-encryption.md index 4a2957b..bbeaf6a 100644 --- a/docs/adr/ADR-0039-byok-envelope-encryption.md +++ b/docs/adr/ADR-0039-byok-envelope-encryption.md @@ -75,12 +75,15 @@ where the real provider impls live. rag-core stays crypto-free (SPI + decorator request — BYOK protects data **at rest** + gates access via the KEK, not against a compromised running server. - **6.7b** added the per-tenant key config (`cfg.kms` + `tenants[].kms_key_id`), a - `build_key_manager_from_config` factory (`noop` / `local` / `aws`), the - `app.state.key_manager` gateway seam, and the **AWS KMS** provider - (`AwsKmsKeyManager` over aioboto3, per-tenant CMK, KMS-error → sealing, - fake-client unit-tested). **GCP KMS / Azure Key Vault / HashiCorp Vault** - providers, wiring `EncryptingStorage` into the ingest/storage path (blocked on - tiered-storage plumbing), and **6.7c** zero-downtime rotation remain deferred. + `build_key_manager_from_config` factory, the `app.state.key_manager` gateway + seam, and the **AWS KMS** provider. **6.7c** added the remaining cloud providers + — **GCP KMS** (`GcpKmsKeyManager`), **Azure Key Vault** + (`AzureKeyVaultKeyManager`, `wrap_key`/`unwrap_key`), **HashiCorp Vault Transit** + (`VaultKeyManager`) — each behind a `[kms-*]` extra (lazy SDK, injectable client, + fake-client unit-tested), completing all four KMS providers. +- Still deferred: wiring `EncryptingStorage` into the ingest/storage path (blocked + on tiered-storage plumbing — chunk content is inline today) and **6.7d** + zero-downtime key rotation (old keys retained decrypt-only until expiry). ## See also - [architecture/byok.md](../architecture/byok.md) — envelope design, AAD, sealing, slicing diff --git a/docs/architecture/byok.md b/docs/architecture/byok.md index f0b4055..c7f499a 100644 --- a/docs/architecture/byok.md +++ b/docs/architecture/byok.md @@ -83,9 +83,12 @@ asserts its reads fail while another tenant's succeed. type → a rag-backends provider via lazy imports, keeping rag-backends config-free). Wiring `EncryptingStorage` into the ingest path is deferred — chunk content is inline today; it lands with tiered storage (ADR-0007). -- **6.7c** — GCP KMS / Azure Key Vault / HashiCorp Vault providers (behind - `[kms-*]` extras, same `EnvelopeKeyManager` pattern) + zero-downtime key - rotation (old keys retained decrypt-only until expiry; background re-encryption). +- **6.7c** — the remaining cloud providers: **GCP KMS** (`GcpKmsKeyManager`), + **Azure Key Vault** (`AzureKeyVaultKeyManager`, via `wrap_key`/`unwrap_key`), + **HashiCorp Vault Transit** (`VaultKeyManager`), each behind a `[kms-*]` extra + (lazy SDK, injectable client). Completes all four KMS providers. +- **6.7d** — zero-downtime key rotation (old keys retained decrypt-only until + expiry; background re-encryption). ## Boundary note diff --git a/docs/reference/encryption.md b/docs/reference/encryption.md index f92ec66..6be1fbe 100644 --- a/docs/reference/encryption.md +++ b/docs/reference/encryption.md @@ -76,11 +76,23 @@ tenants: kms_key_id: ${BETA_KEK_HEX} # provider=local → hex 32-byte KEK ``` -- **`AwsKmsKeyManager`** (`rag-backends`, over the already-present `aioboto3`) - wraps/unwraps the DEK via AWS KMS Encrypt/Decrypt for the tenant's CMK; any KMS - failure (revoked / denied / unreachable) → `KeyUnavailableError` (sealing). -- `tenants[].kms_key_id` is the per-tenant key (a KMS ARN for `aws`, a hex KEK for - `local`); a tenant with neither its own key nor `default_key_id` is sealed. +| `provider` | Class | Extra | Per-tenant `kms_key_id` | +|------------|-------|-------|--------------------------| +| `local` | `LocalKeyManager` | — | hex 32-byte KEK | +| `aws` | `AwsKmsKeyManager` | — (aioboto3 base dep) | KMS key ARN | +| `gcp` | `GcpKmsKeyManager` | `[kms-gcp]` | CryptoKey resource name | +| `azure` | `AzureKeyVaultKeyManager` | `[kms-azure]` | Key Vault key URL | +| `vault` | `VaultKeyManager` | `[kms-vault]` | Transit key name | + +Each cloud provider subclasses `EnvelopeKeyManager` and only wraps/unwraps the +DEK via its KMS API (AWS Encrypt/Decrypt; GCP encrypt/decrypt; Azure +`wrap_key`/`unwrap_key`; Vault Transit `encrypt_data`/`decrypt_data`). Connection +uses each SDK's standard credential discovery (AWS chain, GCP ADC, Azure +`DefaultAzureCredential`, Vault `VAULT_ADDR`/`VAULT_TOKEN`); `kms.vault_mount` sets +the Vault Transit mount. Any KMS failure (revoked / denied / unreachable) or a +missing key → `KeyUnavailableError` (sealing). A tenant with neither its own +`kms_key_id` nor `default_key_id` is sealed. The SDKs are lazy-imported, so +selecting a provider without its extra raises a clear `ImportError`. ## Internals diff --git a/packages/backends/pyproject.toml b/packages/backends/pyproject.toml index 3e16a16..971b3ca 100644 --- a/packages/backends/pyproject.toml +++ b/packages/backends/pyproject.toml @@ -51,6 +51,12 @@ tantivy = ["tantivy>=0.22"] neo4j = ["neo4j>=5.20"] memgraph = ["neo4j>=5.20"] networkx = ["networkx>=3.2"] +# Step 6.7c — BYOK cloud KMS providers (each opt-in; the AWS provider needs no +# extra since aioboto3 is already a base dependency). Lazy-imported so the +# modules import without the SDK; tests inject a fake client. +kms-gcp = ["google-cloud-kms>=2.21"] +kms-azure = ["azure-keyvault-keys>=4.9", "azure-identity>=1.17"] +kms-vault = ["hvac>=2.1"] dev = [ "pytest>=9.0", "pytest-asyncio>=1.3", diff --git a/packages/backends/src/rag_backends/__init__.py b/packages/backends/src/rag_backends/__init__.py index b815e14..85f06a7 100644 --- a/packages/backends/src/rag_backends/__init__.py +++ b/packages/backends/src/rag_backends/__init__.py @@ -12,7 +12,10 @@ from rag_backends.cache.redis_typed import RedisAnswerCache, RedisRetrievalCache from rag_backends.corpus.pg_corpus_store import PgCorpusStore from rag_backends.kms.aws import AwsKmsKeyManager +from rag_backends.kms.azure import AzureKeyVaultKeyManager +from rag_backends.kms.gcp import GcpKmsKeyManager from rag_backends.kms.local import LocalKeyManager +from rag_backends.kms.vault import VaultKeyManager from rag_backends.quota.redis import RedisQuotaStore from rag_backends.storage.local import LocalFileStorage from rag_backends.storage.s3 import S3Storage @@ -21,7 +24,10 @@ __all__ = [ "AwsKmsKeyManager", + "AzureKeyVaultKeyManager", + "GcpKmsKeyManager", "LocalKeyManager", + "VaultKeyManager", "PgCorpusStore", "PgVectorStore", "QdrantVectorStore", diff --git a/packages/backends/src/rag_backends/kms/__init__.py b/packages/backends/src/rag_backends/kms/__init__.py index 37817c0..14f32a4 100644 --- a/packages/backends/src/rag_backends/kms/__init__.py +++ b/packages/backends/src/rag_backends/kms/__init__.py @@ -2,13 +2,24 @@ ``EnvelopeKeyManager`` is the client-side envelope-encryption base; ``LocalKeyManager`` wraps the data key with an in-process per-tenant KEK (dev / -tests / air-gapped); ``AwsKmsKeyManager`` (Step 6.7b) wraps it via AWS KMS. -GCP KMS / Azure Key Vault / HashiCorp Vault providers land in Step 6.7c behind -``[kms-*]`` extras. +tests / air-gapped). Cloud providers wrap the DEK via a customer-controlled KMS: +``AwsKmsKeyManager`` (Step 6.7b), and ``GcpKmsKeyManager`` / +``AzureKeyVaultKeyManager`` / ``VaultKeyManager`` (Step 6.7c) behind the +``[kms-gcp]`` / ``[kms-azure]`` / ``[kms-vault]`` extras. """ from rag_backends.kms.aws import AwsKmsKeyManager +from rag_backends.kms.azure import AzureKeyVaultKeyManager from rag_backends.kms.envelope import EnvelopeKeyManager +from rag_backends.kms.gcp import GcpKmsKeyManager from rag_backends.kms.local import LocalKeyManager +from rag_backends.kms.vault import VaultKeyManager -__all__ = ["AwsKmsKeyManager", "EnvelopeKeyManager", "LocalKeyManager"] +__all__ = [ + "AwsKmsKeyManager", + "AzureKeyVaultKeyManager", + "EnvelopeKeyManager", + "GcpKmsKeyManager", + "LocalKeyManager", + "VaultKeyManager", +] diff --git a/packages/backends/src/rag_backends/kms/azure.py b/packages/backends/src/rag_backends/kms/azure.py new file mode 100644 index 0000000..5c785c6 --- /dev/null +++ b/packages/backends/src/rag_backends/kms/azure.py @@ -0,0 +1,102 @@ +"""AzureKeyVaultKeyManager — Azure Key Vault as the BYOK KEK (Step 6.7c). + +Subclasses :class:`~rag_backends.kms.envelope.EnvelopeKeyManager`; wraps/unwraps +the DEK via an Azure Key Vault key's ``wrap_key`` / ``unwrap_key`` (RSA-OAEP-256 +by default). Unlike the AWS / GCP / Vault providers (one client, many keys), +Azure's ``CryptographyClient`` is **bound to a single key**, so this manager +builds one per tenant key via a ``client_factory`` (cached). + +The ``azure-keyvault-keys`` + ``azure-identity`` SDKs are the +``rag-backends[kms-azure]`` extra; connection uses ``DefaultAzureCredential`` +(env-based). Tests inject a ``client_factory`` so no Azure credentials / network +are needed (the wrap algorithm is then left to the fake). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from rag_core.errors import KeyUnavailableError +from rag_core.types import RequestContext + +from rag_backends.kms.envelope import EnvelopeKeyManager + +__all__ = ["AzureKeyVaultKeyManager"] + + +class AzureKeyVaultKeyManager(EnvelopeKeyManager): + """Envelope encryption with per-tenant Azure Key Vault keys.""" + + def __init__( + self, + *, + key_ids: Mapping[str, str] | None = None, + default_key_id: str | None = None, + client_factory: Callable[[str], Any] | None = None, + wrap_algorithm: Any | None = None, + ) -> None: + self._key_ids = dict(key_ids or {}) + self._default = default_key_id + self._algorithm = wrap_algorithm + if client_factory is None: + try: + from azure.identity.aio import DefaultAzureCredential + from azure.keyvault.keys.crypto import KeyWrapAlgorithm + from azure.keyvault.keys.crypto.aio import CryptographyClient + except ImportError as exc: # pragma: no cover - extras-not-installed + raise ImportError( + "Azure Key Vault support requires rag-backends[kms-azure]" + ) from exc + self._algorithm = self._algorithm or KeyWrapAlgorithm.rsa_oaep_256 + credential = DefaultAzureCredential() + + def _make(key_id: str) -> Any: + return CryptographyClient(key_id, credential) + + client_factory = _make + self._factory = client_factory + self._cache: dict[str, Any] = {} + + def _key_for(self, ctx: RequestContext) -> str: + key = self._key_ids.get(str(ctx.tenant_id), self._default) + if not key: + raise KeyUnavailableError( + f"no Azure Key Vault key configured for tenant {ctx.tenant_id!r} (sealed)", + tenant_id=str(ctx.tenant_id), + ) + return key + + def _client_for(self, key_id: str) -> Any: + if key_id not in self._cache: + self._cache[key_id] = self._factory(key_id) + return self._cache[key_id] + + async def _wrap_dek(self, ctx: RequestContext, dek: bytes) -> bytes: + key_id = self._key_for(ctx) + try: + result = await self._client_for(key_id).wrap_key(self._algorithm, dek) + return bytes(result.encrypted_key) + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — any Key Vault failure seals this tenant + raise KeyUnavailableError( + f"Azure Key Vault wrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def _unwrap_dek(self, ctx: RequestContext, wrapped: bytes) -> bytes: + key_id = self._key_for(ctx) + try: + result = await self._client_for(key_id).unwrap_key(self._algorithm, wrapped) + return bytes(result.key) + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — revoked / denied / unreachable → sealed + raise KeyUnavailableError( + f"Azure Key Vault unwrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def health(self) -> bool: + return True diff --git a/packages/backends/src/rag_backends/kms/gcp.py b/packages/backends/src/rag_backends/kms/gcp.py new file mode 100644 index 0000000..b9771ac --- /dev/null +++ b/packages/backends/src/rag_backends/kms/gcp.py @@ -0,0 +1,79 @@ +"""GcpKmsKeyManager — Google Cloud KMS as the BYOK KEK (Step 6.7c). + +Subclasses :class:`~rag_backends.kms.envelope.EnvelopeKeyManager`; only the DEK +wrap/unwrap differs — it calls Cloud KMS ``encrypt`` / ``decrypt`` for the +tenant's CryptoKey (resource name ``projects/.../cryptoKeys/K``). The +``google-cloud-kms`` SDK is the ``rag-backends[kms-gcp]`` extra, imported lazily; +tests inject a fake ``client`` so no GCP credentials / network are needed. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from rag_core.errors import KeyUnavailableError +from rag_core.types import RequestContext + +from rag_backends.kms.envelope import EnvelopeKeyManager + +__all__ = ["GcpKmsKeyManager"] + + +class GcpKmsKeyManager(EnvelopeKeyManager): + """Envelope encryption with per-tenant Google Cloud KMS keys.""" + + def __init__( + self, + *, + key_ids: Mapping[str, str] | None = None, + default_key_id: str | None = None, + client: Any | None = None, + ) -> None: + self._key_ids = dict(key_ids or {}) + self._default = default_key_id + self._client: Any = client + if client is None: + try: + from google.cloud import kms_v1 + except ImportError as exc: # pragma: no cover - extras-not-installed + raise ImportError("GCP KMS support requires rag-backends[kms-gcp]") from exc + self._client = kms_v1.KeyManagementServiceAsyncClient() + + def _key_for(self, ctx: RequestContext) -> str: + key = self._key_ids.get(str(ctx.tenant_id), self._default) + if not key: + raise KeyUnavailableError( + f"no GCP KMS key configured for tenant {ctx.tenant_id!r} (sealed)", + tenant_id=str(ctx.tenant_id), + ) + return key + + async def _wrap_dek(self, ctx: RequestContext, dek: bytes) -> bytes: + key = self._key_for(ctx) + try: + resp = await self._client.encrypt(request={"name": key, "plaintext": dek}) + return bytes(resp.ciphertext) + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — any KMS failure seals this tenant + raise KeyUnavailableError( + f"GCP KMS wrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def _unwrap_dek(self, ctx: RequestContext, wrapped: bytes) -> bytes: + key = self._key_for(ctx) + try: + resp = await self._client.decrypt(request={"name": key, "ciphertext": wrapped}) + return bytes(resp.plaintext) + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — revoked / denied / unreachable → sealed + raise KeyUnavailableError( + f"GCP KMS unwrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def health(self) -> bool: + return True diff --git a/packages/backends/src/rag_backends/kms/vault.py b/packages/backends/src/rag_backends/kms/vault.py new file mode 100644 index 0000000..4685ca2 --- /dev/null +++ b/packages/backends/src/rag_backends/kms/vault.py @@ -0,0 +1,96 @@ +"""VaultKeyManager — HashiCorp Vault Transit as the BYOK KEK (Step 6.7c). + +Subclasses :class:`~rag_backends.kms.envelope.EnvelopeKeyManager`; wraps/unwraps +the DEK via Vault's Transit secrets engine (``encrypt_data`` / ``decrypt_data``) +for the tenant's named transit key. The ``hvac`` SDK is the +``rag-backends[kms-vault]`` extra (synchronous, so calls run in a thread); +connection comes from the standard ``VAULT_ADDR`` / ``VAULT_TOKEN`` env. Tests +inject a fake ``client`` so no Vault server is needed. +""" + +from __future__ import annotations + +import asyncio +import base64 +from collections.abc import Mapping +from typing import Any + +from rag_core.errors import KeyUnavailableError +from rag_core.types import RequestContext + +from rag_backends.kms.envelope import EnvelopeKeyManager + +__all__ = ["VaultKeyManager"] + + +class VaultKeyManager(EnvelopeKeyManager): + """Envelope encryption with per-tenant HashiCorp Vault Transit keys.""" + + def __init__( + self, + *, + key_ids: Mapping[str, str] | None = None, + default_key_id: str | None = None, + mount_point: str = "transit", + client: Any | None = None, + ) -> None: + self._key_ids = dict(key_ids or {}) + self._default = default_key_id + self._mount = mount_point + self._client: Any = client + if client is None: + try: + import hvac + except ImportError as exc: # pragma: no cover - extras-not-installed + raise ImportError( + "HashiCorp Vault support requires rag-backends[kms-vault]" + ) from exc + self._client = hvac.Client() # VAULT_ADDR / VAULT_TOKEN from env + + def _key_for(self, ctx: RequestContext) -> str: + key = self._key_ids.get(str(ctx.tenant_id), self._default) + if not key: + raise KeyUnavailableError( + f"no Vault transit key configured for tenant {ctx.tenant_id!r} (sealed)", + tenant_id=str(ctx.tenant_id), + ) + return key + + async def _wrap_dek(self, ctx: RequestContext, dek: bytes) -> bytes: + key = self._key_for(ctx) + try: + resp = await asyncio.to_thread( + self._client.secrets.transit.encrypt_data, + name=key, + plaintext=base64.b64encode(dek).decode("ascii"), + mount_point=self._mount, + ) + return str(resp["data"]["ciphertext"]).encode("utf-8") # "vault:v1:…" + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — any Vault failure seals this tenant + raise KeyUnavailableError( + f"Vault wrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def _unwrap_dek(self, ctx: RequestContext, wrapped: bytes) -> bytes: + key = self._key_for(ctx) + try: + resp = await asyncio.to_thread( + self._client.secrets.transit.decrypt_data, + name=key, + ciphertext=wrapped.decode("utf-8"), + mount_point=self._mount, + ) + return base64.b64decode(resp["data"]["plaintext"]) + except KeyUnavailableError: + raise + except Exception as exc: # noqa: BLE001 — revoked / denied / unreachable → sealed + raise KeyUnavailableError( + f"Vault unwrap failed for tenant {ctx.tenant_id!r}: {exc}", + tenant_id=str(ctx.tenant_id), + ) from exc + + async def health(self) -> bool: + return True diff --git a/packages/config/src/rag_config/schema.py b/packages/config/src/rag_config/schema.py index 9332316..ee9a780 100644 --- a/packages/config/src/rag_config/schema.py +++ b/packages/config/src/rag_config/schema.py @@ -621,6 +621,9 @@ class KmsProvider(StrEnum): NOOP = "noop" # passthrough, no encryption (dev) LOCAL = "local" # in-process per-tenant KEK (LocalKeyManager) AWS = "aws" # AWS KMS (Step 6.7b) + GCP = "gcp" # Google Cloud KMS (Step 6.7c) + AZURE = "azure" # Azure Key Vault (Step 6.7c) + VAULT = "vault" # HashiCorp Vault Transit (Step 6.7c) class KmsConfig(_StrictBase): @@ -635,13 +638,17 @@ class KmsConfig(_StrictBase): and no ``default_key_id`` — is **sealed** (``KeyUnavailableError``). **Disabled by default.** ``provider``: ``local`` (in-process KEK, dev / - air-gapped), ``aws`` (AWS KMS), or ``noop`` (passthrough). GCP / Azure / - Vault providers + key rotation land in Step 6.7c. + air-gapped), ``aws`` / ``gcp`` / ``azure`` / ``vault`` (customer-controlled + KMS), or ``noop`` (passthrough). Cloud-provider connection uses standard + credential discovery (AWS chain, GCP ADC, Azure ``DefaultAzureCredential``, + Vault ``VAULT_ADDR`` / ``VAULT_TOKEN``); the per-tenant key reference is + ``tenants[].kms_key_id``. Key rotation lands in Step 6.7d. * ``default_key_id`` — fallback key reference for tenants without their own. * ``region`` — AWS region for the ``aws`` provider. * ``local_key`` — hex 32-byte default KEK for the ``local`` provider (``${ENV_VAR}``-interpolated; keep it out of the file). + * ``vault_mount`` — Transit secrets-engine mount path for the ``vault`` provider. """ enabled: bool = False @@ -649,6 +656,7 @@ class KmsConfig(_StrictBase): default_key_id: str = "" region: str = "us-east-1" local_key: str = "" + vault_mount: str = "transit" class QuotaConfig(_StrictBase): diff --git a/pyproject.toml b/pyproject.toml index fbaf223..0018d24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,6 +165,11 @@ module = [ # [networkx] extras. "neo4j.*", "networkx.*", + # Step 6.7c — BYOK cloud KMS providers behind [kms-gcp] / [kms-azure] / + # [kms-vault] extras; absent at lint time on the default install. + "google.cloud.*", + "azure.*", + "hvac.*", # Step 2.9 — optional Leiden detector behind the [leiden] extra. "leidenalg.*", "igraph.*", diff --git a/tests/kms/test_cloud_providers.py b/tests/kms/test_cloud_providers.py new file mode 100644 index 0000000..6962eea --- /dev/null +++ b/tests/kms/test_cloud_providers.py @@ -0,0 +1,186 @@ +"""GCP / Azure / Vault KMS provider tests with injected fake clients (Step 6.7c). + +The envelope crypto is covered by ``test_envelope.py``; here we verify each cloud +provider's wrap/unwrap glue against a fake KMS — round-trip, per-tenant key +binding, sealing on a missing key, and SDK-error → KeyUnavailableError. No real +GCP / Azure / Vault SDKs, credentials, or network (the SDKs are opt-in extras). +""" + +from __future__ import annotations + +import os +from types import SimpleNamespace +from typing import Any + +import pytest +from rag_backends import ( + AzureKeyVaultKeyManager, + GcpKmsKeyManager, + VaultKeyManager, +) +from rag_core.errors import KeyUnavailableError +from rag_core.types import ( + Principal, + PrincipalId, + PrincipalKind, + RequestContext, + TenantId, +) + +_PLAINTEXT = b"secret chunk text" +_KEYS = {"acme": "key/acme", "globex": "key/globex"} + + +def _ctx(tenant: str) -> RequestContext: + tid = TenantId(tenant) + return RequestContext( + tenant_id=tid, + principal=Principal( + id=PrincipalId("p"), kind=PrincipalKind.user, display_name="p", tenant_id=tid + ), + ) + + +# --------------------------------------------------------------------------- +# GCP KMS — shared async client, key-bound ciphertext +# --------------------------------------------------------------------------- +class _FakeGcp: + def __init__(self) -> None: + self._store: dict[bytes, tuple[str, bytes]] = {} + self.fail = False + + async def encrypt(self, *, request: dict[str, Any]) -> Any: + if self.fail: + raise RuntimeError("kms down") + token = os.urandom(16) + self._store[token] = (request["name"], request["plaintext"]) + return SimpleNamespace(ciphertext=token) + + async def decrypt(self, *, request: dict[str, Any]) -> Any: + name, plaintext = self._store[request["ciphertext"]] + if name != request["name"]: + raise RuntimeError("key mismatch") + return SimpleNamespace(plaintext=plaintext) + + +def _gcp(client: _FakeGcp | None = None) -> GcpKmsKeyManager: + return GcpKmsKeyManager(key_ids=_KEYS, client=client or _FakeGcp()) + + +async def test_gcp_round_trip() -> None: + km = _gcp() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + assert ct != _PLAINTEXT + assert await km.decrypt(_ctx("acme"), ct) == _PLAINTEXT + + +async def test_gcp_cross_tenant_blocked() -> None: + km = _gcp() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + with pytest.raises(KeyUnavailableError): + await km.decrypt(_ctx("globex"), ct) + + +async def test_gcp_sealing_and_errors() -> None: + with pytest.raises(KeyUnavailableError): + await _gcp().encrypt(_ctx("ghost"), _PLAINTEXT) + fake = _FakeGcp() + fake.fail = True + with pytest.raises(KeyUnavailableError): + await _gcp(fake).encrypt(_ctx("acme"), _PLAINTEXT) + + +# --------------------------------------------------------------------------- +# HashiCorp Vault Transit — sync client (run in a thread), key-bound ciphertext +# --------------------------------------------------------------------------- +class _FakeTransit: + def __init__(self) -> None: + self._store: dict[str, tuple[str, str]] = {} + self.fail = False + + def encrypt_data(self, *, name: str, plaintext: str, mount_point: str) -> dict[str, Any]: + if self.fail: + raise RuntimeError("vault down") + token = f"vault:v1:{os.urandom(8).hex()}" + self._store[token] = (name, plaintext) + return {"data": {"ciphertext": token}} + + def decrypt_data(self, *, name: str, ciphertext: str, mount_point: str) -> dict[str, Any]: + n, plaintext = self._store[ciphertext] + if n != name: + raise RuntimeError("key mismatch") + return {"data": {"plaintext": plaintext}} + + +def _vault(transit: _FakeTransit | None = None) -> VaultKeyManager: + client = SimpleNamespace(secrets=SimpleNamespace(transit=transit or _FakeTransit())) + return VaultKeyManager(key_ids=_KEYS, client=client) + + +async def test_vault_round_trip() -> None: + km = _vault() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + assert ct != _PLAINTEXT + assert await km.decrypt(_ctx("acme"), ct) == _PLAINTEXT + + +async def test_vault_cross_tenant_blocked() -> None: + km = _vault() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + with pytest.raises(KeyUnavailableError): + await km.decrypt(_ctx("globex"), ct) + + +async def test_vault_sealing_and_errors() -> None: + with pytest.raises(KeyUnavailableError): + await _vault().encrypt(_ctx("ghost"), _PLAINTEXT) + transit = _FakeTransit() + transit.fail = True + with pytest.raises(KeyUnavailableError): + await _vault(transit).encrypt(_ctx("acme"), _PLAINTEXT) + + +# --------------------------------------------------------------------------- +# Azure Key Vault — per-key CryptographyClient via a factory, key-bound blob +# --------------------------------------------------------------------------- +class _FakeAzClient: + def __init__(self, key_id: str, store: dict[bytes, tuple[str, bytes]]) -> None: + self._key_id = key_id + self._store = store + + async def wrap_key(self, algorithm: Any, key: bytes) -> Any: + token = os.urandom(16) + self._store[token] = (self._key_id, key) + return SimpleNamespace(encrypted_key=token) + + async def unwrap_key(self, algorithm: Any, encrypted_key: bytes) -> Any: + kid, key = self._store[encrypted_key] + if kid != self._key_id: + raise RuntimeError("key mismatch") + return SimpleNamespace(key=key) + + +def _azure() -> AzureKeyVaultKeyManager: + store: dict[bytes, tuple[str, bytes]] = {} + return AzureKeyVaultKeyManager( + key_ids=_KEYS, client_factory=lambda key_id: _FakeAzClient(key_id, store) + ) + + +async def test_azure_round_trip() -> None: + km = _azure() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + assert ct != _PLAINTEXT + assert await km.decrypt(_ctx("acme"), ct) == _PLAINTEXT + + +async def test_azure_cross_tenant_blocked() -> None: + km = _azure() + ct = await km.encrypt(_ctx("acme"), _PLAINTEXT) + with pytest.raises(KeyUnavailableError): + await km.decrypt(_ctx("globex"), ct) + + +async def test_azure_sealing() -> None: + with pytest.raises(KeyUnavailableError): + await _azure().encrypt(_ctx("ghost"), _PLAINTEXT) diff --git a/uv.lock b/uv.lock index 5c9bd87..1e143cc 100644 --- a/uv.lock +++ b/uv.lock @@ -715,6 +715,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/9a/417b3a533e01953a7c618884df2cb05a71e7b68bdbce4fbdb62349d2a2e8/azure_identity-1.25.3-py3-none-any.whl", hash = "sha256:f4d0b956a8146f30333e071374171f3cfa7bdb8073adb8c3814b65567aa7447c", size = 192138, upload-time = "2026-03-13T01:12:22.951Z" }, ] +[[package]] +name = "azure-keyvault-keys" +version = "4.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "azure-core" }, + { name = "cryptography" }, + { name = "isodate" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/03/5ce6db28b545427d4ab572f6a4ef2a727b6b4e7bf6941cedddf98822535b/azure_keyvault_keys-4.11.1.tar.gz", hash = "sha256:90caa3a7b2c8f6b53c247ec115cf1c1dad7f107cc3aa9f35aff4838bbce7e562", size = 260915, upload-time = "2026-05-19T20:01:08.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/3d/7bed91ae9268cf48124cf6990d8cd2c3daff7a2bb1e91a439d26ee90d705/azure_keyvault_keys-4.11.1-py3-none-any.whl", hash = "sha256:f46cdf6ee7a9baf27f70e6838327032886c8a087041dd56397773b1639da8fe2", size = 200651, upload-time = "2026-05-19T20:01:09.809Z" }, +] + [[package]] name = "azure-storage-blob" version = "12.29.0" @@ -2149,6 +2164,102 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/70/b40dd004721f450995b368453e673b6ee23743f9de56fe5570c514622433/gcloud_aio_storage-9.6.4-py3-none-any.whl", hash = "sha256:a3b8af75e98485325cee80443a6f3e74f8efa7ed6ed10697db163682390cc1e4", size = 17383, upload-time = "2026-02-26T17:47:16.006Z" }, ] +[[package]] +name = "google-api-core" +version = "2.25.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "google-auth", marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "googleapis-common-protos", marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "proto-plus", marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "requests", marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/cd/63f1557235c2440fe0577acdbc32577c5c002684c58c7f4d770a92366a24/google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300", size = 166266, upload-time = "2025-10-03T00:07:34.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/d8/894716a5423933f5c8d2d5f04b16f052a515f78e815dab0c2c6f1fd105dc/google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7", size = 162489, upload-time = "2025-10-03T00:07:32.924Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio", marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "grpcio-status", version = "1.71.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, +] + +[[package]] +name = "google-api-core" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", +] +dependencies = [ + { name = "google-auth", marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, + { name = "googleapis-common-protos", marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, + { name = "proto-plus", marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.13.*'" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, + { name = "requests", marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/22/155cadf1d49272a9cf48f3168c0f3874fa13397297e611a5ea00cd093880/google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2", size = 176492, upload-time = "2026-06-03T14:52:17.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio", marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, + { name = "grpcio-status", version = "1.71.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.13.*'" }, + { name = "grpcio-status", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, +] + +[[package]] +name = "google-auth" +version = "2.53.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", size = 335844, upload-time = "2026-05-15T20:53:07.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", size = 246071, upload-time = "2026-05-15T20:53:05.609Z" }, +] + +[[package]] +name = "google-cloud-kms" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", version = "2.25.2", source = { registry = "https://pypi.org/simple" }, extra = ["grpc"], marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, + { name = "google-api-core", version = "2.31.0", source = { registry = "https://pypi.org/simple" }, extra = ["grpc"], marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version != '3.14.*' and sys_platform == 'win32')" }, + { name = "google-auth" }, + { name = "grpc-google-iam-v1" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/a4/9125701c7120469a4381c3a576cebad191abbfa5bd3a0e1effd5c1ef0f52/google_cloud_kms-3.13.0.tar.gz", hash = "sha256:672fdc594b928b0415c22e41f3d67c854e940a8a5917e8ff16a6566096b12407", size = 440145, upload-time = "2026-05-07T08:04:18.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/ff/ba421b035eba2d139a58aa5913e6daae4a969a96c555eff6bf02242cd8a5/google_cloud_kms-3.13.0-py3-none-any.whl", hash = "sha256:a58a40d8d37129805db5be1353a397be09614d691433ea246b9eed63efdc5df3", size = 353997, upload-time = "2026-05-07T08:02:49.849Z" }, +] + [[package]] name = "google-crc32c" version = "1.8.0" @@ -2185,6 +2296,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, ] +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, +] + [[package]] name = "greenlet" version = "3.5.1" @@ -2287,6 +2403,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" }, ] +[[package]] +name = "grpc-google-iam-v1" +version = "0.14.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", extra = ["grpc"] }, + { name = "grpcio" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/4f/d098419ad0bfc06c9ce440575f05aa22d8973b6c276e86ac7890093d3c37/grpc_google_iam_v1-0.14.4.tar.gz", hash = "sha256:392b3796947ed6334e61171d9ab06bf7eb357f554e5fc7556ad7aab6d0e17038", size = 23706, upload-time = "2026-04-01T01:57:49.813Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/22/c2dd50c09bf679bd38173656cd4402d2511e563b33bc88f90009cf50613c/grpc_google_iam_v1-0.14.4-py3-none-any.whl", hash = "sha256:412facc320fcbd94034b4df3d557662051d4d8adfa86e0ddb4dca70a3f739964", size = 32675, upload-time = "2026-04-01T01:57:47.69Z" }, +] + [[package]] name = "grpcio" version = "1.80.0" @@ -2416,6 +2547,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/29/49fbd2593a29dab9cd5837f67668157ef7a24c16eac232852379e8e43266/grpcio_reflection-1.80.0-py3-none-any.whl", hash = "sha256:a7d0b77961b1c722400b1509968f1ad3a64e9d78280d4cf5b88b6cfe5b41eb61", size = 22917, upload-time = "2026-03-30T08:54:00.008Z" }, ] +[[package]] +name = "grpcio-status" +version = "1.71.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "googleapis-common-protos", marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, + { name = "grpcio", marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/d1/b6e9877fedae3add1afdeae1f89d1927d296da9cf977eca0eb08fb8a460e/grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50", size = 13677, upload-time = "2025-06-28T04:24:05.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/58/317b0134129b556a93a3b0afe00ee675b5657f0155509e22fcb853bafe2d/grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3", size = 14424, upload-time = "2025-06-28T04:23:42.136Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.80.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'darwin'", + "python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')", +] +dependencies = [ + { name = "googleapis-common-protos", marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, + { name = "grpcio", marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" }, +] + [[package]] name = "grpcio-tools" version = "1.71.2" @@ -2708,6 +2885,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/79/621a7dbb80c70974f73a597275351ebe03ce5bc65cb5f8f4acb5859252bc/huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22", size = 668176, upload-time = "2026-05-21T18:39:58.596Z" }, ] +[[package]] +name = "hvac" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/57/b46c397fb3842cfb02a44609aa834c887f38dd75f290c2fc5a34da4b2fee/hvac-2.4.0.tar.gz", hash = "sha256:e0056ad9064e7923e874e6769015b032580b639e29246f5ab1044f7959c1c7e0", size = 332543, upload-time = "2025-10-30T12:57:47.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/33/71e45a6bd6875f44a26f99da31c63b6840123e88bedf2c0b1ce429b8be12/hvac-2.4.0-py3-none-any.whl", hash = "sha256:008db5efd8c2f77bd37d2368ea5f713edceae1c65f11fd608393179478649e0f", size = 155921, upload-time = "2025-10-30T12:57:46.253Z" }, +] + [[package]] name = "hyperframe" version = "6.1.0" @@ -6067,6 +6256,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "proto-plus" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and python_full_version < '3.15') or (python_full_version >= '3.13' and sys_platform != 'win32')" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13' or (python_full_version >= '3.15' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, +] + [[package]] name = "protobuf" version = "5.29.6" @@ -6972,6 +7174,16 @@ elasticsearch = [ gcs = [ { name = "gcloud-aio-storage" }, ] +kms-azure = [ + { name = "azure-identity" }, + { name = "azure-keyvault-keys" }, +] +kms-gcp = [ + { name = "google-cloud-kms" }, +] +kms-vault = [ + { name = "hvac" }, +] memgraph = [ { name = "neo4j" }, ] @@ -7000,10 +7212,14 @@ requires-dist = [ { name = "aiofiles", specifier = ">=24.0" }, { name = "asyncpg", specifier = ">=0.30" }, { name = "azure-identity", marker = "extra == 'azure'", specifier = ">=1.17" }, + { name = "azure-identity", marker = "extra == 'kms-azure'", specifier = ">=1.17" }, + { name = "azure-keyvault-keys", marker = "extra == 'kms-azure'", specifier = ">=4.9" }, { name = "azure-storage-blob", marker = "extra == 'azure'", specifier = ">=12.22" }, { name = "cryptography", specifier = ">=43.0" }, { name = "elasticsearch", extras = ["async"], marker = "extra == 'elasticsearch'", specifier = ">=8.15" }, { name = "gcloud-aio-storage", marker = "extra == 'gcs'", specifier = ">=9.3" }, + { name = "google-cloud-kms", marker = "extra == 'kms-gcp'", specifier = ">=2.21" }, + { name = "hvac", marker = "extra == 'kms-vault'", specifier = ">=2.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=2.1" }, { name = "neo4j", marker = "extra == 'memgraph'", specifier = ">=5.20" }, { name = "neo4j", marker = "extra == 'neo4j'", specifier = ">=5.20" }, @@ -7022,7 +7238,7 @@ requires-dist = [ { name = "types-aiobotocore", extras = ["s3"], specifier = ">=2.13" }, { name = "weaviate-client", marker = "extra == 'weaviate'", specifier = ">=4.9" }, ] -provides-extras = ["gcs", "azure", "postgres-cdc", "weaviate", "pinecone", "elasticsearch", "tantivy", "neo4j", "memgraph", "networkx", "dev"] +provides-extras = ["gcs", "azure", "postgres-cdc", "weaviate", "pinecone", "elasticsearch", "tantivy", "neo4j", "memgraph", "networkx", "kms-gcp", "kms-azure", "kms-vault", "dev"] [[package]] name = "rag-breaker"