diff --git a/.env.example b/.env.example index 6a1a008..598fccb 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,46 @@ ATOM_DB_CONNECT_TIMEOUT_SECS=10 ATOM_DB_IDLE_TIMEOUT_SECS=600 ATOM_DB_MAX_LIFETIME_SECS=1800 +# --- Cache (Redis) -------------------------------------------------------- +# Redis-backed cache for AuthN/AuthZ decision *inputs* (session validity, +# entity/tenant status, credential verification, the effective-grants +# expansion) — never the allow/deny decision itself, which is always +# recomputed. Postgres remains authoritative: this is a pure performance +# optimization, off by default, and every check works correctly with it +# disabled. See src/cache/mod.rs for the full consistency model. +# +# Invalidation is precise and synchronous on every relevant mutation (not +# TTL-based) — the TTLs below are a defense-in-depth safety net only, for the +# rare case an invalidation call itself is lost (e.g. a crash mid-mutation). +# While enabled, an unreachable Redis refuses security-sensitive mutations +# (grants/session/credential changes) rather than committing a change the +# cache can't be told about; reads always fall back to Postgres regardless. +# +# Off by default. Uncomment and point at a reachable Redis (e.g. the `redis` +# service in docker-compose.yml) to enable: +ATOM_CACHE_ENABLED=false +# ATOM_CACHE_REDIS_URL=redis://redis:6379/0 +# ATOM_CACHE_POOL_MAX_SIZE=20 +# ATOM_CACHE_CONNECT_TIMEOUT_MS=2000 +# A slow Redis must never make auth slower than a Postgres-only path — keep +# this small; a request that misses this timeout just falls through to Postgres. +# ATOM_CACHE_OP_TIMEOUT_MS=50 +# If true, an unreachable Redis at startup aborts boot (like an unreachable +# Postgres does). If false (default), Atom logs an error and starts anyway +# with caching disabled — recommended, since Redis is a performance +# optimization for reads, not a correctness dependency. +# ATOM_CACHE_FAIL_FAST_ON_STARTUP=false +# Per-category entry TTLs (seconds) — defense-in-depth only, see above. +# ATOM_CACHE_TTL_SESSION_SECS=60 +# ATOM_CACHE_TTL_ENTITY_STATUS_SECS=60 +# ATOM_CACHE_TTL_TENANT_STATUS_SECS=60 +# ATOM_CACHE_TTL_CREDENTIAL_SECS=60 +# ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS=60 +# ATOM_CACHE_TTL_GRANTS_SECS=60 +# Redis should be network-private (not publicly reachable) and, in any real +# deployment, protected with auth/TLS appropriate to your network — the +# docker-compose `redis` service here is dev-only (loopback-bound, no auth). + # --- Audit retention and abuse controls -------------------------------- ATOM_AUDIT_RETENTION_DAYS=365 ATOM_AUDIT_RETENTION_ENABLED=true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cc89e8c..d4f998d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -34,6 +34,25 @@ jobs: --health-timeout 5s --health-retries 5 + # Cache-gated tests (src/cache/mod.rs's unit tests, tests/m25_cache_invalidation.rs) + # are #[ignore]d specifically so they're skippable without Redis, but + # `--include-ignored` below runs them anyway — they need a reachable + # Redis. One instance serves the whole job, but it is flushed between + # binaries alongside the database recreate: m25 caches under keys derived + # from the *fixed* seeded admin id, so without a flush the admin's grant + # expansion outlives the database it was derived from (and outlives a job + # re-run), and a later binary authorizes against a tenant/role graph that + # no longer exists. + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v4 @@ -66,15 +85,21 @@ jobs: env: PGPASSWORD: atom MAINT_URL: postgres://atom:atom@localhost:5432/atom + ATOM_TEST_REDIS_URL: redis://localhost:6379/0 run: | set -euo pipefail - command -v psql >/dev/null || { - sudo apt-get update && sudo apt-get install -y postgresql-client + command -v psql >/dev/null && command -v redis-cli >/dev/null || { + sudo apt-get update && sudo apt-get install -y postgresql-client redis-tools } psql "$MAINT_URL" -c "SELECT 1" >/dev/null + redis-cli -h localhost -p 6379 PING >/dev/null run_one() { psql "$MAINT_URL" -c "DROP DATABASE IF EXISTS atom_test;" >/dev/null psql "$MAINT_URL" -c "CREATE DATABASE atom_test;" >/dev/null + # Redis must be reset with the database, not just alongside it: + # cached entries keyed off fixed ids (the seeded admin) would + # otherwise describe the database this just dropped. + redis-cli -h localhost -p 6379 FLUSHALL >/dev/null DATABASE_URL="postgres://atom:atom@localhost:5432/atom_test" \ cargo test "$@" -- --include-ignored --test-threads=1 } diff --git a/Cargo.lock b/Cargo.lock index d22a1a4..630a371 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "argon2" version = "0.5.3" @@ -426,6 +432,7 @@ dependencies = [ "axum", "base64 0.22.1", "chrono", + "deadpool-redis", "dotenvy", "hex", "insta", @@ -443,7 +450,9 @@ dependencies = [ "prost", "rand 0.8.6", "rcgen", + "redis", "ring", + "rmp-serde", "serde", "serde_json", "sqlx", @@ -806,7 +815,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -1018,6 +1031,36 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafa30c49dafe086d10116074e422ad7fc1c3cf554697e744a3ab112599ebd09" +dependencies = [ + "deadpool", + "redis", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + [[package]] name = "der" version = "0.7.10" @@ -2594,6 +2637,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oauth2" version = "5.0.0" @@ -3298,6 +3351,30 @@ dependencies = [ "yasna", ] +[[package]] +name = "redis" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" +dependencies = [ + "arcstr", + "async-lock", + "bytes", + "cfg-if", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.6.3", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3433,6 +3510,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rsa" version = "0.9.10" @@ -3769,6 +3865,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5483,6 +5585,12 @@ dependencies = [ "time", ] +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yasna" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index f3115ba..4553abc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,9 @@ lapin = { version = "4.10.0", default-features = false, features = [ "rustls--ring", "rustls-webpki-roots-certs", ] } +redis = { version = "1", default-features = false, features = ["script"] } +deadpool-redis = { version = "0.23", default-features = false, features = ["rt_tokio_1"] } +rmp-serde = "1" [features] # Metrics are on by default. Disable at compile time for maximum-performance diff --git a/docker-compose.yml b/docker-compose.yml index e4eb3ae..c47e5a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,20 @@ services: timeout: 5s retries: 5 + # Redis-backed AuthN/AuthZ cache (see src/cache/mod.rs). Off by default + # (ATOM_CACHE_ENABLED=false) — a pure performance optimization, Postgres + # remains authoritative. Pure cache, no persistence needed. + redis: + image: redis:7-alpine + command: ["redis-server", "--save", "", "--appendonly", "no"] + ports: + - "127.0.0.1:${ATOM_CACHE_REDIS_PORT:-6379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + atom: image: ${ATOM_IMAGE:-ghcr.io/absmach/atom:latest} build: @@ -88,6 +102,20 @@ services: ATOM_LOG_LEVEL: ${ATOM_LOG_LEVEL:-${RUST_LOG:-info}} ATOM_LOG_FORMAT: ${ATOM_LOG_FORMAT:-text} ATOM_EMAIL_TEMPLATES_DIR: ${ATOM_EMAIL_TEMPLATES_DIR:-/email-templates} + # Off by default. Set ATOM_CACHE_ENABLED=true (and add `redis` to + # depends_on below) once Redis is provisioned — see .env.example. + ATOM_CACHE_ENABLED: ${ATOM_CACHE_ENABLED:-false} + ATOM_CACHE_REDIS_URL: ${ATOM_CACHE_REDIS_URL:-redis://redis:6379/0} + ATOM_CACHE_POOL_MAX_SIZE: ${ATOM_CACHE_POOL_MAX_SIZE:-20} + ATOM_CACHE_CONNECT_TIMEOUT_MS: ${ATOM_CACHE_CONNECT_TIMEOUT_MS:-2000} + ATOM_CACHE_OP_TIMEOUT_MS: ${ATOM_CACHE_OP_TIMEOUT_MS:-50} + ATOM_CACHE_FAIL_FAST_ON_STARTUP: ${ATOM_CACHE_FAIL_FAST_ON_STARTUP:-false} + ATOM_CACHE_TTL_SESSION_SECS: ${ATOM_CACHE_TTL_SESSION_SECS:-60} + ATOM_CACHE_TTL_ENTITY_STATUS_SECS: ${ATOM_CACHE_TTL_ENTITY_STATUS_SECS:-60} + ATOM_CACHE_TTL_TENANT_STATUS_SECS: ${ATOM_CACHE_TTL_TENANT_STATUS_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS:-60} + ATOM_CACHE_TTL_GRANTS_SECS: ${ATOM_CACHE_TTL_GRANTS_SECS:-60} volumes: - ${ATOM_CERTS_CA_DIR:-./certs}:/certs:ro - ${ATOM_EMAIL_TEMPLATES_HOST_DIR:-./email-templates}:/email-templates:ro @@ -175,6 +203,18 @@ services: ATOM_LOG_LEVEL: ${ATOM_LOG_LEVEL:-${RUST_LOG:-info}} ATOM_LOG_FORMAT: ${ATOM_LOG_FORMAT:-text} ATOM_EMAIL_TEMPLATES_DIR: ${ATOM_EMAIL_TEMPLATES_DIR:-/email-templates} + ATOM_CACHE_ENABLED: ${ATOM_CACHE_ENABLED:-false} + ATOM_CACHE_REDIS_URL: ${ATOM_CACHE_REDIS_URL:-redis://redis:6379/0} + ATOM_CACHE_POOL_MAX_SIZE: ${ATOM_CACHE_POOL_MAX_SIZE:-20} + ATOM_CACHE_CONNECT_TIMEOUT_MS: ${ATOM_CACHE_CONNECT_TIMEOUT_MS:-2000} + ATOM_CACHE_OP_TIMEOUT_MS: ${ATOM_CACHE_OP_TIMEOUT_MS:-50} + ATOM_CACHE_FAIL_FAST_ON_STARTUP: ${ATOM_CACHE_FAIL_FAST_ON_STARTUP:-false} + ATOM_CACHE_TTL_SESSION_SECS: ${ATOM_CACHE_TTL_SESSION_SECS:-60} + ATOM_CACHE_TTL_ENTITY_STATUS_SECS: ${ATOM_CACHE_TTL_ENTITY_STATUS_SECS:-60} + ATOM_CACHE_TTL_TENANT_STATUS_SECS: ${ATOM_CACHE_TTL_TENANT_STATUS_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS:-60} + ATOM_CACHE_TTL_GRANTS_SECS: ${ATOM_CACHE_TTL_GRANTS_SECS:-60} volumes: - ${ATOM_CERTS_CA_DIR:-./certs}:/certs:ro - ${ATOM_EMAIL_TEMPLATES_HOST_DIR:-./email-templates}:/email-templates:ro diff --git a/docs/content/docs/architecture/caching.mdx b/docs/content/docs/architecture/caching.mdx new file mode 100644 index 0000000..3d840c7 --- /dev/null +++ b/docs/content/docs/architecture/caching.mdx @@ -0,0 +1,217 @@ +--- +title: Caching +description: Atom's Redis-backed cache for authentication and authorization inputs — what is cached, how invalidation stays correct, and how it fails safe. +--- + +Atom is designed to sit on the auth path of every downstream service. Without a cache, each request costs three or four Postgres round trips, one of which is a recursive CTE. This page explains the Redis cache that sits in front of those reads: what it stores, when it invalidates, and why it can be trusted with security-sensitive decisions. + +## Goal + +For every request arriving at Atom: + +- **JWT auth** needs a three-way join across `sessions`, `entities`, and `tenants`. +- **API-key auth** needs the same shape across `credentials`, `entities`, and `tenants`, plus a KDF verification. +- **Every authorization decision** needs `subject_effective_grants(uuid)` — a recursive CTE that flattens direct policies, role assignments, and group membership. + +The cache targets these hot reads. Postgres remains the single source of truth; the cache is optional at every call site. + +## Design Principles + +1. **Cache inputs, never decisions.** The permit/deny outcome is never cached, so a policy change takes effect on the next request without touching a combinatorial `(subject, action, object)` space. +2. **Correctness over hit rate.** A revoked token, disabled entity, or removed grant must stop working immediately. Every mutation that could affect a cached value invalidates the affected keys through a race-safe barrier. +3. **Fail-safe reads.** If Redis is unreachable, reads fall through to Postgres. Auth still works — just slower. +4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate would risk serving stale grants after a revoke. +5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. + +`cache: None` means *caching is not configured*, and it is the one state in which every mutation guard degrades to a pass-through. An **enabled** cache that merely cannot reach Redis is a different state and never collapses into `None` — otherwise a replica that booted during an outage would mutate grants, sessions, and credentials without invalidating entries its peers were still serving, and a revoke on one replica would stay authorized on another. Unreachable Redis is a runtime condition: the client is retained, reads degrade to misses, and `begin` fails so security-sensitive mutations are refused until Redis returns. + +## What Is Cached + +Six categories, each keyed by a UUID under the `atom:v1:` namespace. + +| Category | Key format | Answers | +|---|---|---| +| `Session` | `atom:v1:session:` | Is this JWT's session alive? | +| `EntityStatus` | `atom:v1:entity_status:` | Is the user active? Which tenant? | +| `TenantStatus` | `atom:v1:tenant_status:` | Is the tenant active? | +| `Credential` | `atom:v1:credential:` | API-key hash, status, expiry (never plaintext). | +| `CredentialCeiling` | `atom:v1:cred_ceiling:` | Scoped-token permission cap. | +| `Grants` | `atom:v1:grants:` | The user's full flattened permission list. | + +### What is *not* cached + +- **Passwords** — never used on the request path. Used only during login, which mints a JWT; the JWT then uses the `Session` cache. +- **Plaintext API-key secrets** — only the hash used to verify them. +- **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. +- **Audit writes** — always go to Postgres. +- **Tombstones.** The entity and tenant entries carry no `deleted_at`. Both miss loaders already filter `deleted_at IS NULL`, so a cached copy would be empty by construction and any check against it a no-op that merely *looked* like a tombstone check. Denying a subsequently soft-deleted entity or tenant is the delete path's invalidation duty. + +## Request Flow + +Authorization Bearer ...] + Request --> Which{Token type?} + Which -->|JWT| JWT[JWT auth] + Which -->|API key| API[API-key auth] + JWT --> J1[Session cache] + JWT --> J2[EntityStatus cache] + JWT --> J3[TenantStatus cache] + API --> A1[Credential cache] + API --> A2[EntityStatus cache] + API --> A3[TenantStatus cache] + API --> A4[CredentialCeiling cache
only if scoped] + J1 --> Authn[AuthContext built] + J2 --> Authn + J3 --> Authn + A1 --> Authn + A2 --> Authn + A3 --> Authn + A4 --> Authn + Authn --> Authz{Authorization check?} + Authz -->|Yes| G[Grants cache] + G --> PDP[PDP evaluates
allow / deny] + Authz -->|No| Done[Handler runs] + PDP --> Done +`} /> + +The auth hot path batches the keys it can into **one pipelined round trip on a single pooled connection**. Issued one at a time, three lookups would mean three pool acquisitions and three serial round trips, each bounded by the operation timeout, before any request work started. + +Only keys with no data dependency on each other can share a round trip. For JWT auth, session, entity, and tenant are all known up front — the tenant key comes from the token's `tid` claim — so all three go together. For API-key auth the credential key is read first and alone because it gates everything after it, and the tenant key is derived from the entity's current tenant rather than from the credential, so it cannot be batched with it. + +## Consistency Model + +Every cached entry is a Redis hash with three fields: + +- `v` — an integer version, bumped on every mutation that could affect the entry. +- `dirty` — `"1"` while a mutation is in flight, absent otherwise. +- `p` — the serialized payload, present only when the entry holds a valid value. + +Four atomic Lua scripts implement a per-key **mutation barrier**. + +| Primitive | When it runs | What it does | +|---|---|---| +| `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, clears `p`, and re-applies the entry's expiry. Best-effort. | +| `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | +| `discard` | After a reader fails to deserialize a payload | Clears **only** `p`, and only if `dirty=0` and `v` still equals what that reader observed. Never touches the barrier fields. | + +### Read path + +>C: lookup(key) + alt Hit and not dirty + C-->>R: value + else Miss / dirty / Redis down + C-->>R: (Miss, version=N) + R->>P: load(key) + P-->>R: value + R->>C: try_populate(key, version=N, value) + Note over C: Writes only if version still N
and not dirty + end +`} /> + +### Write path + +>C: begin(keys) + Note over C: bump v, set dirty=1 + alt Redis unreachable + C-->>W: Err service_unavailable + Note over W: Mutation refused + else OK + C-->>W: Ok + W->>P: UPDATE / DELETE ... + P-->>W: committed + W->>C: end(keys) + Note over C: bump v again, clear dirty,
clear p, re-apply expiry + end +`} /> + +### The races this prevents + +1. **Read-before-mutation.** A reader observes version `N`, starts loading from Postgres; a mutation runs to completion, bumping to `N+2`. The reader's `try_populate` presents `N` — rejected on version mismatch. +2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). +3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. +4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — populates only apply when the observed version's key matches the key the payload describes. +5. **Cleanup destroying a live barrier.** Discarding a corrupt payload must not take `v` and `dirty` with it. Deleting the whole hash would erase a concurrent mutation's barrier: the next reader would find an absent key, observe version `0`, load pre-commit state, and populate it successfully. `discard` is version-guarded and clears only `p`; `end` also clears `p` defensively. + +## Invalidation Map + +| Mutation | Invalidates | +|---|---| +| Logout | `Session` | +| Password reset | `Session` (bulk, per active session) | +| Entity update / activate / deactivate / restore | `EntityStatus` | +| Entity delete (REST or GraphQL) | `EntityStatus` + `Session` + `Credential` | +| Tenant update | `TenantStatus` | +| Tenant delete | `TenantStatus` + child `Session`s | +| Tenant restore | `TenantStatus` + reactivated `Credential`s | +| Tenant create | `Grants` (of the creator) | +| Credential revoke / rotate | `Credential` | +| Credential scope change | `CredentialCeiling` | +| Role assignment (create / delete) | `Grants` for each affected subject | +| Direct policy (create / delete) | `Grants` for the subject | +| Role permission-block change | `Grants` for every assignee of the role | +| Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | + +Tenant creation is on this list because `create_tenant` bootstraps a tenant-admin role, role assignment, and membership for the creator in the same transaction — it grows the creator's own grant set, and the capability gate immediately above it has just warmed that exact key. `purgeTenant` performs no invalidation: it is reachable only for an already-soft-deleted tenant, whose soft delete invalidated `TenantStatus` and the members' sessions, so the tenant is already denied at the lifecycle check that runs before grant matching. + +### Race-safe enumeration + +Group and role invalidations must enumerate *who is affected* — every entity in a group closure, every assignee of a role. If enumeration happens outside a transaction, a concurrent `add_group_member` can slip a new member in between enumeration and mutation. That new member's `grants` key is never invalidated, and a stale entry survives until TTL. + +The enumeration functions therefore lock the relevant rows `FOR UPDATE` inside the caller's transaction, in id-sorted order so they cannot deadlock against themselves. The `guarded_tx_mutation` helper wires that ordering — open transaction, lock and enumerate, `begin` barrier, mutate, commit, `end` barrier — in one place so every call site does it identically. + +## Failure Modes + +| Failure | Behaviour | Impact | +|---|---|---| +| Redis unreachable at startup | The client is retained, never downgraded to `cache: None`. `ATOM_CACHE_FAIL_FAST_ON_STARTUP` decides whether to abort instead. | Reads fall through to Postgres; security-sensitive mutations are refused until Redis recovers. | +| Cache config invalid at startup | Fatal regardless of the fail-fast flag — an unparseable URL cannot recover by retrying. | Startup fails. | +| Redis unreachable during read | Treated as unavailable; falls through to Postgres. | Auth works, slower. | +| Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | Mutation does not commit. | +| Redis unreachable during `end` | Best-effort; entry stays dirty until barrier TTL. | Entry reloads on next reader. | +| Redis unreachable during `try_populate` | Best-effort; dropped silently. | Next reader still gets a miss and retries. | +| Corrupt payload | Treated as a miss; only the payload field is cleared, version-guarded, so a concurrent mutation's barrier survives. | One extra Postgres round trip. | + +## Configuration + +All knobs live under the `ATOM_CACHE_*` prefix: + +- `ATOM_CACHE_ENABLED` — master switch. +- `ATOM_CACHE_REDIS_URL` — connection URL. +- `ATOM_CACHE_POOL_MAX_SIZE` — max Redis connections. +- `ATOM_CACHE_FAIL_FAST_ON_STARTUP` — `true` aborts startup when Redis is unreachable. Default `false`. +- `ATOM_CACHE_CONNECT_TIMEOUT_MS` — startup PING timeout. +- `ATOM_CACHE_OP_TIMEOUT_MS` — per-operation timeout. +- `ATOM_CACHE_TTL__SECS` — one per category (session, entity_status, tenant_status, credential, credential_ceiling, grants). Each must be greater than zero and no more than 86400 seconds (24h); the barrier expiry is derived as `entry_ttl * 5` and has to stay representable. + +TTLs are the residual staleness bound if invalidation is missed entirely (e.g. `end` never completed and the barrier TTL elapsed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. + +## Metrics + +| Metric | Labels | Values | +|---|---|---| +| `atom_cache_lookup_total` | `category`, `outcome` | `hit`, `miss`, `error` | +| `atom_cache_invalidation_total` | `category`, `outcome` | `ok`, `error` | +| `atom_cache_populate_total` | `category`, `outcome` | `applied`, `stale`, `error` | + +Category labels are fixed enum variants, so cardinality is bounded. + +The populate metric splits `applied` (write took) from `stale` (rejected by the barrier: dirty entry, or the version moved since the caller's `lookup`). A hit rate stuck at zero can then be told apart from every write being rejected by a stuck barrier — otherwise indistinguishable. + +## Extending + +The cache client (Redis pool, Lua scripts, barrier, timeouts, metrics) is fully generic over any serde type. Only the **registry** of categories is centralised in the codebase — one enum, one TTL config struct, one file of key builders — so that: + +- Metrics label cardinality stays bounded. +- All TTLs are auditable in one file. +- No handler can silently invent a new cache category that collides with an existing one. + +Adding a new cached category is a five-step, one-file-at-a-time change: extend the category enum, add a TTL field, add a key builder, wrap read sites in the cache-aside helper, and wrap write sites in the invalidation helpers. diff --git a/docs/content/docs/architecture/meta.json b/docs/content/docs/architecture/meta.json index 6b4b02f..7450eab 100644 --- a/docs/content/docs/architecture/meta.json +++ b/docs/content/docs/architecture/meta.json @@ -1,4 +1,4 @@ { "title": "Architecture", - "pages": ["index", "data-model"] + "pages": ["index", "data-model", "caching"] } diff --git a/product-docs/14-caching.md b/product-docs/14-caching.md new file mode 100644 index 0000000..ed64463 --- /dev/null +++ b/product-docs/14-caching.md @@ -0,0 +1,387 @@ +# AuthN / AuthZ Cache + +## Status: Active v1 +## Date: 2026-07-31 + +This document describes Atom's Redis-backed cache for authentication and authorization inputs. The source of truth for overall product requirements is [Atom Product Requirements Document](./PRD.md), and authorization terminology is defined in [Atom access model](./11-access-model-simplification.md). + +--- + +## Goal + +Atom is often used as an external Identity Provider. Every request from every downstream service hits Atom's auth path. Without a cache, each request costs: + +- **JWT auth** — a three-way join (`sessions ⨝ entities ⨝ tenants`). +- **API-key auth** — a three-way join (`credentials ⨝ entities ⨝ tenants`) plus a KDF verification. +- **Every authorization decision** — a recursive CTE call (`subject_effective_grants(uuid)`) that flattens direct policies, role assignments, and group membership. + +That is 3–4 Postgres round trips per request, one of them a recursive CTE. Under IdP-scale load this becomes the bottleneck. + +The cache targets these hot reads specifically, keeping Postgres as the single source of truth. + +--- + +## Design Principles + +1. **Cache inputs, never decisions.** The permit/deny outcome is never cached. Only the raw data the PDP consumes is cached, so a policy change takes effect on the next request without needing to invalidate a combinatorial `(subject, action, object)` space. +2. **Correctness over hit rate.** A revoked token, disabled entity, or removed grant must stop working immediately. Every mutation that could affect a cached value invalidates the affected keys through a race-safe barrier. +3. **Fail-safe reads.** If Redis is unreachable, reads fall through to Postgres. Auth still works — just slower. +4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate the cache would risk serving stale grants after a revoke. +5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. + +`cache: None` means *caching is not configured*, and it is the one state in which every mutation guard degrades to a pass-through. An **enabled** cache that merely cannot reach Redis is a different state and never collapses into `None` — otherwise a replica that booted during an outage would mutate grants, sessions, and credentials without invalidating entries its peers were still serving, and a revoke on one replica would stay authorized on another. Unreachable Redis is a runtime condition: the client is retained, reads degrade to misses, and `begin` fails so security-sensitive mutations are refused until Redis returns. + +--- + +## What Is Cached + +Six categories, each keyed by a UUID under the `atom:v1:` namespace. + +| Category | Key format | Payload shape (DTO) | Answers | +|---|---|---|---| +| `Session` | `atom:v1:session:` | `SessionCacheEntry` | Is this JWT's session alive? | +| `EntityStatus` | `atom:v1:entity_status:` | `EntityStatusCacheEntry` | Is the user active? Which tenant? | +| `TenantStatus` | `atom:v1:tenant_status:` | `TenantStatusCacheEntry` | Is the tenant active? | +| `Credential` | `atom:v1:credential:` | `CredentialCacheEntry` | API-key hash, status, expiry (never plaintext). | +| `CredentialCeiling` | `atom:v1:cred_ceiling:` | `CredentialCeiling` | Scoped-token permission cap. | +| `Grants` | `atom:v1:grants:` | `Vec` | The user's full flattened permission list. | + +DTOs are defined in [`src/cache/entries.rs`](../src/cache/entries.rs). Key builders in [`src/cache/keys.rs`](../src/cache/keys.rs). + +The entity and tenant DTOs deliberately carry **no `deleted_at` field**. Both miss loaders already filter `deleted_at IS NULL`, so an entry can only ever be populated from a live row — a cached tombstone column would be `None` by construction and any check against it a no-op that merely *looked* like a tombstone check. Denying a subsequently soft-deleted entity or tenant is the delete path's invalidation duty, not the entry's. + +### What is *not* cached + +- **Passwords** — never used on the request path. Used only during `/login`, which mints a JWT; the JWT then uses the `Session` cache. +- **Plaintext API-key secrets** — only the hash used to verify them. See `CredentialCacheEntry` in [`src/cache/entries.rs`](../src/cache/entries.rs). +- **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. +- **Audit writes** — always go to Postgres. + +--- + +## How AuthN / AuthZ Use the Cache + +The three layers of a request, and which cache each layer hits: + +```mermaid +flowchart TD + Request[Incoming request
Authorization: Bearer ...] + Request --> Which{Token type?} + + Which -->|JWT| JWT[JWT auth] + Which -->|API key| API[API-key auth] + + JWT --> J1[Session cache] + JWT --> J2[EntityStatus cache] + JWT --> J3[TenantStatus cache] + + API --> A1[Credential cache] + API --> A2[EntityStatus cache] + API --> A3[TenantStatus cache] + API --> A4[CredentialCeiling cache
only if scoped] + + J1 & J2 & J3 & A1 & A2 & A3 & A4 --> Authn[AuthContext built] + + Authn --> Authz{Authorization check
needed?} + Authz -->|Yes| G[Grants cache] + G --> PDP[PDP evaluates
allow / deny] + Authz -->|No| Done[Handler runs] + PDP --> Done +``` + +- JWT auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_jwt`. +- API-key auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_api_key`. +- Grants load: [`src/auth.rs`](../src/auth.rs) `AuthContext::effective_grants` and [`src/authz/engine.rs`](../src/authz/engine.rs) inside `load_decision_context`. + +The auth hot path batches the keys it can into **one pipelined round trip on a single pooled connection**, via `CacheClient::lookup_many` + `CacheClient::decode` — see [`src/cache/mod.rs`](../src/cache/mod.rs). Issued one at a time these would be three pool acquisitions and three serial round trips, each bounded by `op_timeout`, before any request work started. + +Only keys with no data dependency on each other can share a round trip: + +- **JWT** — session, entity, and tenant are all known up front, since the tenant key comes from the token's `tid` claim. All three go in one round trip. +- **API-key** — the credential key is read first and alone, because it gates everything after it; on a credential hit the entity read likewise precedes the tenant key, which is derived from the *entity's* current tenant rather than from the credential (see the note on `CredentialCacheEntry` above). On the cold path, where both are already known from the loaded row, the entity and tenant reads are batched. + +--- + +## Consistency Model + +Every cached entry is a Redis hash with three fields: + +- `v` — an integer version, bumped on every mutation that could affect the entry. +- `dirty` — `"1"` while a mutation is in flight, absent otherwise. +- `p` — the serialized payload, present only when the entry holds a valid value. + +Four atomic Lua scripts implement a per-key **mutation barrier** that closes races a plain cache-aside plus post-commit `DEL` cannot. + +### The three primitives + +| Primitive | When it runs | What it does | +|---|---|---| +| `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, clears `p`, and re-applies the entry's expiry. Best-effort. | +| `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | +| `discard` | After a reader fails to deserialize a payload | Clears **only** `p`, and only if `dirty=0` and `v` still equals what that reader observed. Never touches the barrier fields. | + +### Read path + +```mermaid +sequenceDiagram + participant R as Request + participant C as CacheClient + participant P as Postgres + + R->>C: lookup(key) + alt Hit and not dirty + C-->>R: value + else Miss / dirty / Redis down + C-->>R: (Miss, version=N) + R->>P: load(key) + P-->>R: value + R->>C: try_populate(key, version=N, value) + Note over C: Writes only if version still N
and not dirty + end +``` + +### Write path (mutation) + +```mermaid +sequenceDiagram + participant W as Mutation + participant C as CacheClient + participant P as Postgres + + W->>C: begin(keys) + Note over C: bump v, set dirty=1 + alt Redis unreachable + C-->>W: Err service_unavailable + Note over W: Mutation refused + else OK + C-->>W: Ok + W->>P: UPDATE / DELETE ... + P-->>W: committed + W->>C: end(keys) + Note over C: bump v again, clear dirty,
clear p, re-apply expiry + end +``` + +### The races this prevents + +1. **Read-before-mutation.** A reader observes version `N`, starts loading from Postgres; a mutation runs to completion, bumping to `N+2`. The reader's `try_populate` presents `N` — rejected on version mismatch. +2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). +3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. `end` re-applies `PEXPIRE` on the entry, since `HINCRBY` would otherwise recreate an already-expired key and leave it immortal. +4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — e.g. the JWT miss loader joins tenants through `entities.tenant_id`, so it returns the entity's *current* tenant's status, which is not necessarily the tenant the token's `tid` claim points to when the token outlived a tenant move. Populates now write only when the observed version's key matches the key the payload describes. +5. **Cleanup destroying a live barrier.** Discarding a corrupt payload must not take `v` and `dirty` with it. Deleting the whole hash would erase a concurrent mutation's barrier: the next reader would find an absent key, observe version `0`, load pre-commit state, and populate it successfully — the barrier defeated by a cleanup path. `discard` is version-guarded and clears only `p`; `end` also clears `p` defensively, so whatever happened to the key mid-mutation, the next reader reloads cleanly. + +The first primitive that fails self-heals: `begin` failing refuses the mutation; `end` failing leaves the entry dirty until barrier-TTL expiry; `try_populate` failing just leaves the entry as a miss for the next reader to reload. + +--- + +## Invalidation Map + +Which mutation invalidates which category: + +| Mutation | Invalidates | Where | +|---|---|---| +| Logout | `Session` | [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Password reset | `Session` (bulk, per active session) | [`src/identity/service.rs`](../src/identity/service.rs) | +| Entity update | `EntityStatus` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Entity activate / deactivate | `EntityStatus` | [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Entity delete | `EntityStatus` + `Session` + `Credential` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Entity restore | `EntityStatus` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Tenant update | `TenantStatus` | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant delete | `TenantStatus` + child `Session`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant restore | `TenantStatus` + reactivated `Credential`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant create | `Grants` (of the creator) | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Credential revoke / rotate | `Credential` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Credential scope change | `CredentialCeiling` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs) | +| Role assignment (create / delete) | `Grants` for each affected subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Direct policy (create / delete) | `Grants` for the subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Role permission-block change | `Grants` for every assignee of the role | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | [`src/graphql/groups.rs`](../src/graphql/groups.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | + +Tenant creation is on this list because `create_tenant` bootstraps a tenant-admin role, role assignment, and membership for the creator in the same transaction — it grows the creator's own grant set, and the capability gate immediately above it has just warmed that exact key. + +`purgeTenant` performs **no** cache invalidation. It is reachable only for an already-soft-deleted tenant, and the soft delete invalidated `TenantStatus` and the members' sessions, so the tenant is already denied at the lifecycle check that runs before grant matching. Residual `Grants` entries naming the purged tenant are therefore inert rather than dangerous — but this rests on the soft delete having run first, and is worth revisiting if purge ever becomes reachable directly. + +The REST paths (`delete_entity`, `add_group_member`, `remove_group_member`) invalidate through the same helpers as the GraphQL resolvers — entity deletion is consolidated into a single service entry point at [`src/identity/service.rs`](../src/identity/service.rs), and both group-member handlers wrap their mutation in `guarded_mutation`. + +### Race-safe enumeration + +Group and role invalidations must enumerate **who is affected** (every entity in a group closure, every assignee of a role). If enumeration happens outside a transaction, a concurrent `add_group_member` can slip a new member in *between* enumeration and mutation — that new member's `grants` key is never invalidated, and a stale entry survives until TTL. + +To close this, the enumeration functions lock the relevant rows `FOR UPDATE` inside the caller's transaction: + +- [`lock_group_closures_and_collect_grants_keys`](../src/authz/repo.rs) — locks every group in the closure of the given roots (id-sorted, so it cannot deadlock against itself), then returns the affected `grants` keys. +- [`lock_role_and_collect_grants_keys`](../src/authz/repo.rs) — locks the role row, then locks the closure of every group assigned to it. + +Callers pair these with `guarded_tx_mutation` (see below) rather than `guarded_mutation`, because the enumeration must happen inside the same open transaction as the mutation itself. + +--- + +## API Surface + +### Reading + +```rust +// Cache-aside with a fallback loader. Callers use this at read sites. +crate::cache::cached_or_load( + cache, // Option<&CacheClient> + CacheCategory::Grants, + &cache::keys::grants(subject_id), + || repo::effective_grants_for_subject(pool, subject_id), +).await? +``` + +Defined at [`src/cache/mod.rs`](../src/cache/mod.rs) `cached_or_load`. Tolerant of `cache: None` — falls through to `loader`. + +### Writing (single category, single call site) + +```rust +crate::cache::invalidate::guarded_mutation( + cache, + CacheCategory::Credential, + std::slice::from_ref(&cache::keys::credential(credential_id)), + || async { /* Postgres mutation */ }, +).await? +``` + +Defined at [`src/cache/invalidate.rs`](../src/cache/invalidate.rs) `guarded_mutation`. + +### Writing (single-category, mutation owns a `Transaction`) + +Use `guarded_tx_mutation` when the affected keys can only be enumerated **under the locks the mutation itself takes** — every group-closure and role mutation. It opens the transaction, runs `collect_keys` (which locks and enumerates), establishes the barrier on that key set, runs `mutate` on the same transaction, commits, and clears the barrier — in that fixed order, regardless of outcome. Stable Rust has no async closures, so the closures return a boxed future borrowing the transaction: + +```rust +crate::cache::invalidate::guarded_tx_mutation( + cache, + CacheCategory::Grants, + &state.pool, + |tx| Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[group_id]).await + }), + |tx| Box::pin(async move { do_mutation_in_tx(tx).await }), +).await? +``` + +Defined at [`src/cache/invalidate.rs`](../src/cache/invalidate.rs) `guarded_tx_mutation`. Callers hold the `cache: None` fallback themselves because the uncached path is usually a different repo function (typically the non-`_in_tx` variant that opens its own transaction). + +### Writing (multi-category, mutation owns a `Transaction`) + +Use `begin_all` / `end_all` for mutations spanning several categories on one open transaction (e.g. `deleteEntity` invalidates `EntityStatus` + `Session` + `Credential`): + +```rust +let mut tx = pool.begin().await?; +let (session_ids, credential_ids) = + repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by).await?; +let session_keys: Vec = session_ids.iter().map(|id| keys::session(*id)).collect(); +let credential_keys: Vec = credential_ids.iter().map(|id| keys::credential(*id)).collect(); +let entity_status_keys = [keys::entity_status(id)]; +let groups = [ + (CacheCategory::EntityStatus, entity_status_keys.as_slice()), + (CacheCategory::Session, session_keys.as_slice()), + (CacheCategory::Credential, credential_keys.as_slice()), +]; + +cache::invalidate::begin_all(cache, &groups).await?; +let outcome = repo::finish_entity_deletion_in_tx(&mut tx, id).await; +let outcome = match outcome { + Ok(()) => tx.commit().await, + Err(e) => Err(e), +}; +cache::invalidate::end_all(cache, &groups).await; +outcome? +``` + +--- + +## Failure Modes + +| Failure | Behaviour | Impact | +|---|---|---| +| Redis unreachable at startup | The client is retained, never downgraded to `cache: None`. `ATOM_CACHE_FAIL_FAST_ON_STARTUP` decides whether to abort startup instead of booting into the refusing state below. | Reads fall through to Postgres; security-sensitive mutations are refused until Redis recovers. | +| Cache config invalid at startup | Fatal regardless of `ATOM_CACHE_FAIL_FAST_ON_STARTUP` — an unparseable `ATOM_CACHE_REDIS_URL` cannot recover by retrying. | Startup fails. | +| Redis unreachable during read | Treated as `Lookup::Unavailable`. Falls through to Postgres loader. | Auth works, slower. | +| Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | The mutation does not commit. | +| Redis unreachable during `end` | Best-effort — logged, not surfaced. Entry stays dirty until barrier TTL. | Entry reloads on next reader; slight perf hit until then. | +| Redis unreachable during `try_populate` | Best-effort — dropped silently. | Next reader still gets a miss and retries. | +| Corrupt payload | Treated as a miss; only the payload field is cleared, version-guarded, so a concurrent mutation's barrier survives. | One extra Postgres round trip. | + +--- + +## Configuration + +Set via environment variables — see [`.env.example`](../.env.example) for the full list. + +| Variable | Meaning | +|---|---| +| `ATOM_CACHE_ENABLED` | Master switch. `false` disables all caching (call sites pass `cache: None`). | +| `ATOM_CACHE_REDIS_URL` | Redis connection URL. | +| `ATOM_CACHE_POOL_MAX_SIZE` | Max Redis connections. | +| `ATOM_CACHE_FAIL_FAST_ON_STARTUP` | `true` aborts startup when Redis is unreachable. Default `false` — boot and refuse security-sensitive mutations until Redis recovers. | +| `ATOM_CACHE_CONNECT_TIMEOUT_MS` | Startup PING timeout. | +| `ATOM_CACHE_OP_TIMEOUT_MS` | Per-operation timeout. | +| `ATOM_CACHE_TTL_SESSION_SECS` | TTL for `Session` entries. | +| `ATOM_CACHE_TTL_ENTITY_STATUS_SECS` | TTL for `EntityStatus` entries. | +| `ATOM_CACHE_TTL_TENANT_STATUS_SECS` | TTL for `TenantStatus` entries. | +| `ATOM_CACHE_TTL_CREDENTIAL_SECS` | TTL for `Credential` entries. | +| `ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS` | TTL for `CredentialCeiling` entries. | +| `ATOM_CACHE_TTL_GRANTS_SECS` | TTL for `Grants` entries. | + +Config struct: [`src/config.rs`](../src/config.rs) `CacheConfig` / `CacheTtlConfig`. + +Every `ATOM_CACHE_TTL_*` value must be greater than zero and no more than **86400 seconds (24h)**, rejected at startup otherwise. The upper bound exists because the barrier expiry is derived as `entry_ttl * 5`, which has to stay representable — without the bound, a nonsensically large TTL would boot and serve reads happily, then fail on the first security-sensitive mutation. + +TTLs are the residual staleness bound if invalidation is missed entirely (e.g. barrier TTL expired before `end` completed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. + +--- + +## Metrics + +All cache operations emit metrics through [`src/metrics.rs`](../src/metrics.rs): + +| Metric | Labels | Values | +|---|---|---| +| `atom_cache_lookup_total` | `category`, `outcome` | `hit`, `miss`, `error` | +| `atom_cache_invalidation_total` | `category`, `outcome` | `ok`, `error` | +| `atom_cache_populate_total` | `category`, `outcome` | `applied`, `stale`, `error` | + +Category labels are fixed enum variants, so cardinality is bounded. + +The populate metric splits `applied` (the write took) from `stale` (rejected by the barrier: dirty entry, or the version moved since the caller's `lookup`). A hit rate stuck at zero can then be told apart from every write being rejected by a stuck barrier — otherwise indistinguishable. + +--- + +## Extending — Adding a New Cached Category + +The cache client (Redis pool, Lua scripts, barrier, timeouts, metrics) is fully generic. Only the **registry** of categories is centralised. To add a new one: + +1. Add a variant to `CacheCategory` in [`src/cache/mod.rs`](../src/cache/mod.rs), plus cases in `as_str()` and `ttl()`. +2. Add a `_secs: u64` field to `CacheTtlConfig` in [`src/config.rs`](../src/config.rs), and a matching env var. +3. Add a key builder to [`src/cache/keys.rs`](../src/cache/keys.rs). +4. (Optional) add a DTO to [`src/cache/entries.rs`](../src/cache/entries.rs) — the client is generic over any `Serialize + DeserializeOwned` type, so a bespoke DTO is only needed if the DB row shape is not directly usable. +5. At each read site: call `cached_or_load(cache, CacheCategory::, &keys::(id), || loader)`. +6. At each write site that could affect the entry: wrap the mutation in `guarded_mutation` — or `guarded_tx_mutation` if the affected keys can only be enumerated under the mutation's own locks, or `begin_all` / `end_all` if the mutation spans several categories on one open `Transaction`. + +The design is deliberately explicit — the enum is not `Other(String)` — because: + +- Bounded label cardinality keeps metrics well-behaved. +- All TTLs are auditable in one file. +- No handler can silently invent a new cache category that collides with an existing one. + +--- + +## Testing + +- **Barrier semantics (Redis-gated unit tests):** [`src/cache/mod.rs`](../src/cache/mod.rs) `#[cfg(test)] mod tests`. Covers the read-before-mutation race, the read-during-dirty-window race, and that a corrupt-payload cleanup leaves a concurrent barrier intact. +- **End-to-end invalidation matrix:** [`tests/m25_cache_invalidation.rs`](../tests/m25_cache_invalidation.rs). Covers every mutation → invalidation pairing listed above, plus the cross-key poisoning and enumeration races. + +Both suites are `#[ignore]` and require `ATOM_TEST_REDIS_URL`; run with `cargo test -- --include-ignored`. + +Redis must be **flushed between test binaries**, alongside the per-binary database recreate — see `run_one` in [`.github/workflows/rust.yml`](../.github/workflows/rust.yml). The suite caches under keys derived from the fixed seeded admin id, so without a flush the admin's grant expansion outlives the database it was derived from and a later binary authorizes against a tenant graph that no longer exists. + +--- + +## Related Documents + +- [Atom Product Requirements Document](./PRD.md) +- [Atom access model](./11-access-model-simplification.md) +- [Scoped Access Tokens](./13-access-tokens.md) diff --git a/src/audit.rs b/src/audit.rs index 9f16e7c..c22b1e0 100644 --- a/src/audit.rs +++ b/src/audit.rs @@ -205,7 +205,7 @@ pub async fn commit_with_audit( /// Enqueues a domain event outbox row inside an existing DB transaction for /// non-audited operations (e.g. create mutations), keeping the mutation and outbox /// event strictly atomic. -async fn observe_in_tx( +pub(crate) async fn observe_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, events_enabled: bool, meta: &AuditMeta<'_>, @@ -256,7 +256,7 @@ pub async fn commit_with_observation( /// Emits an audit log tracing line for a successful non-audited operation (observe path) /// after its database transaction has successfully committed. -fn log_observe_allow(meta: &AuditMeta<'_>, details: &Value) { +pub(crate) fn log_observe_allow(meta: &AuditMeta<'_>, details: &Value) { let event = AuditEvent { actor_entity_id: meta.actor_entity_id, tenant_id: meta.tenant_id, diff --git a/src/auth.rs b/src/auth.rs index 2948eaf..4f15ecc 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -12,6 +12,12 @@ use sqlx::PgPool; use uuid::Uuid; use crate::{ + cache::{ + entries::{ + CredentialCacheEntry, EntityStatusCacheEntry, SessionCacheEntry, TenantStatusCacheEntry, + }, + keys as cache_keys, CacheCategory, Lookup, + }, error::{db_err, AppError}, keys::{ActiveKeys, LoadedKey}, models::enums::{ @@ -55,6 +61,14 @@ pub struct AuthContext { /// The loaded permission ceiling for a scoped access token; `None` for /// unscoped tokens and JWT/session auth (no cap). pub ceiling: Option>, + /// The cache backing this request's grant-expansion lookups (see + /// `src/cache/mod.rs`). Carried here — rather than threaded as an extra + /// parameter through every capability gate (`has_capability_in_scope`, + /// `require_any_capability`, ...) and through `authz::engine::evaluate`/ + /// `explain` — because those have dozens of call sites across the + /// codebase, while `AuthContext` is constructed in exactly a few places. + /// `None` reproduces pre-caching behavior exactly. + pub cache: Option>, } /// Extractor that requires the authenticated entity to hold a `manage` policy @@ -64,6 +78,28 @@ pub struct RequireManage(pub AuthContext); // ─── JWT ────────────────────────────────────────────────────────────────────── +/// The parsed form of a primary key, ready to sign with. Derived from a +/// [`LoadedKey`] via [`JwtSigner::from_key`]. +/// +/// Exists so a caller holding the `ActiveKeys` read guard can take what it +/// needs and release the guard before doing I/O, without cloning `LoadedKey` +/// — which would duplicate the raw PKCS8 private-key PEM on the heap for the +/// duration. This holds only the parsed key that signing requires anyway. +pub struct JwtSigner { + kid: String, + key: EncodingKey, +} + +impl JwtSigner { + pub fn from_key(primary: &LoadedKey) -> Result { + Ok(Self { + kid: primary.kid.clone(), + key: EncodingKey::from_ec_pem(primary.private_key_pem.as_bytes()) + .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}")))?, + }) + } +} + pub fn encode_jwt( entity_id: Uuid, session_id: Uuid, @@ -72,10 +108,30 @@ pub fn encode_jwt( expiry_secs: u64, issuer: &str, audience: &str, +) -> Result { + encode_jwt_with( + entity_id, + session_id, + tenant_id, + &JwtSigner::from_key(primary)?, + expiry_secs, + issuer, + audience, + ) +} + +pub fn encode_jwt_with( + entity_id: Uuid, + session_id: Uuid, + tenant_id: Option, + signer: &JwtSigner, + expiry_secs: u64, + issuer: &str, + audience: &str, ) -> Result { let header = Header { alg: Algorithm::ES256, - kid: Some(primary.kid.clone()), + kid: Some(signer.kid.clone()), ..Header::default() }; @@ -90,10 +146,7 @@ pub fn encode_jwt( exp: now + expiry_secs as usize, }; - let encoding_key = EncodingKey::from_ec_pem(primary.private_key_pem.as_bytes()) - .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}")))?; - - encode(&header, &claims, &encoding_key) + encode(&header, &claims, &signer.key) .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}"))) } @@ -163,31 +216,28 @@ async fn auth_from_token(state: &AppState, token: &str) -> Result Result { - let keys = state.keys.read().await; - let claims = decode_jwt( - token, - &keys, - &state.config.jwt_issuer, - &state.config.jwt_audience, - )?; - drop(keys); - - let entity_id: Uuid = claims - .sub - .parse() - .map_err(|_| AppError::unauthorized("invalid entity id in token"))?; - let session_id: Uuid = claims - .sid - .parse() - .map_err(|_| AppError::unauthorized("invalid session id in token"))?; - let tenant_id: Option = claims - .tid - .as_deref() - .map(|s| s.parse()) - .transpose() - .map_err(|_| AppError::unauthorized("invalid tenant id in token"))?; +/// A snapshot of the fields `auth_from_jwt` needs, regardless of whether they +/// came from a cache hit or a fresh Postgres load — one canonical check +/// function (`check_session_entity_tenant`) validates either shape +/// identically, so the deny logic can never drift between the cached and +/// uncached paths. +struct SessionEntityTenantSnapshot { + session_entity_id: Uuid, + revoked_at: Option>, + expires_at: chrono::DateTime, + entity_tenant_id: Option, + entity_status: EntityStatus, + tenant_status: Option, +} +/// The existing session/entity/tenant join, unchanged from before caching was +/// introduced. This remains the single cache-miss loader — it is never split +/// into per-field queries. +async fn load_session_entity_tenant( + pool: &PgPool, + session_id: Uuid, + entity_id: Uuid, +) -> Result { use sqlx::Row; let row = sqlx::query( r#"SELECT s.revoked_at, @@ -204,7 +254,7 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result AppError::unauthorized("session not found"), @@ -215,48 +265,260 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result = row .try_get("expires_at") .map_err(|_| AppError::unauthorized("corrupt session"))?; + let entity_status: EntityStatus = row + .try_get("entity_status") + .map_err(|_| AppError::unauthorized("corrupt entity"))?; + let entity_tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); + let tenant_status: Option = row + .try_get::, _>("tenant_status") + .unwrap_or(None); + + Ok(SessionEntityTenantSnapshot { + session_entity_id: entity_id, + revoked_at, + expires_at, + entity_tenant_id, + entity_status, + tenant_status, + }) +} - if revoked_at.is_some() { +/// The one canonical set of deny checks for JWT/session authentication — +/// consumed identically whether `snapshot` came from a cache hit or a fresh +/// Postgres load. +fn check_session_entity_tenant( + snapshot: &SessionEntityTenantSnapshot, + expected_entity_id: Uuid, + expected_tenant_id: Option, +) -> Result<(), AppError> { + if snapshot.session_entity_id != expected_entity_id { + return Err(AppError::unauthorized("session not found")); + } + if snapshot.revoked_at.is_some() { return Err(AppError::unauthorized("session revoked")); } - if expires_at < Utc::now() { + if snapshot.expires_at < Utc::now() { return Err(AppError::unauthorized("session expired")); } - - let entity_status: EntityStatus = row - .try_get("entity_status") - .map_err(|_| AppError::unauthorized("corrupt entity"))?; - if entity_status != EntityStatus::Active { + if snapshot.entity_status != EntityStatus::Active { return Err(AppError::unauthorized("entity is not active")); } - - let entity_tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); - if tenant_id != entity_tenant_id { + if expected_tenant_id != snapshot.entity_tenant_id { return Err(AppError::unauthorized("token tenant does not match entity")); } - if let Some(tenant_status) = row - .try_get::, _>("tenant_status") - .unwrap_or(None) - { - if tenant_status != TenantStatus::Active { + if let Some(tenant_status) = &snapshot.tenant_status { + if *tenant_status != TenantStatus::Active { return Err(AppError::unauthorized("tenant is not active")); } } + Ok(()) +} - Ok(AuthContext { +fn jwt_auth_context( + state: &AppState, + entity_id: Uuid, + session_id: Uuid, + snapshot: &SessionEntityTenantSnapshot, +) -> AuthContext { + AuthContext { entity_id, - tenant_id: entity_tenant_id, + tenant_id: snapshot.entity_tenant_id, session_id: Some(session_id), + cache: state.cache.clone(), ..Default::default() - }) + } } -async fn auth_from_api_key(state: &AppState, key: &str) -> Result { - let (cred_id, secret_bytes) = - parse_api_key(key).ok_or_else(|| AppError::unauthorized("malformed api key"))?; +async fn auth_from_jwt(state: &AppState, token: &str) -> Result { + let keys = state.keys.read().await; + let claims = decode_jwt( + token, + &keys, + &state.config.jwt_issuer, + &state.config.jwt_audience, + )?; + drop(keys); - use sqlx::Row; + let entity_id: Uuid = claims + .sub + .parse() + .map_err(|_| AppError::unauthorized("invalid entity id in token"))?; + let session_id: Uuid = claims + .sid + .parse() + .map_err(|_| AppError::unauthorized("invalid session id in token"))?; + let tenant_id: Option = claims + .tid + .as_deref() + .map(|s| s.parse()) + .transpose() + .map_err(|_| AppError::unauthorized("invalid tenant id in token"))?; + + let Some(cache) = &state.cache else { + let snapshot = load_session_entity_tenant(&state.pool, session_id, entity_id).await?; + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + return Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)); + }; + + let session_key = cache_keys::session(session_id); + let entity_key = cache_keys::entity_status(entity_id); + let tenant_key = tenant_id.map(cache_keys::tenant_status); + + // All three keys are known up front — the tenant key comes from the `tid` + // claim, not from the entity entry as it does in `auth_from_api_key` — so + // they are read in one pipelined round trip rather than three serial ones + // on three separate pooled connections, before any request work starts. + let mut batch_keys: Vec<&str> = vec![&session_key, &entity_key]; + if let Some(key) = &tenant_key { + batch_keys.push(key); + } + let mut raw = cache.lookup_many(&batch_keys).await.into_iter(); + let session_raw = raw.next().unwrap_or_default(); + let entity_raw = raw.next().unwrap_or_default(); + let tenant_raw = raw.next(); + + let session_lookup = cache + .decode::(CacheCategory::Session, &session_key, session_raw) + .await; + let entity_lookup = cache + .decode::(CacheCategory::EntityStatus, &entity_key, entity_raw) + .await; + let tenant_lookup = match (&tenant_key, tenant_raw) { + (Some(key), Some(raw)) => Some( + cache + .decode::(CacheCategory::TenantStatus, key, raw) + .await, + ), + _ => None, + }; + + // Full cache hit across every key this token needs: validate entirely in + // memory, no Postgres round trip. + if let (Lookup::Hit(session), Lookup::Hit(entity)) = (&session_lookup, &entity_lookup) { + let tenant_hit = match (&tenant_key, &tenant_lookup) { + (None, _) => Some(None), + (Some(_), Some(Lookup::Hit(tenant))) => Some(Some(tenant)), + _ => None, + }; + if let Some(tenant_opt) = tenant_hit { + let snapshot = SessionEntityTenantSnapshot { + session_entity_id: session.entity_id, + revoked_at: session.revoked_at, + expires_at: session.expires_at, + entity_tenant_id: entity.tenant_id, + entity_status: entity.status.clone(), + tenant_status: tenant_opt.map(|t| t.status.clone()), + }; + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + return Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)); + } + } + + // Any miss/dirty/unavailable key: fall back to the existing combined + // query, unchanged, then best-effort populate whichever entries missed. + let snapshot = load_session_entity_tenant(&state.pool, session_id, entity_id).await?; + + if let Lookup::Miss { version } = session_lookup { + let entry = SessionCacheEntry { + entity_id, + revoked_at: snapshot.revoked_at, + expires_at: snapshot.expires_at, + }; + cache + .try_populate(CacheCategory::Session, &session_key, version, &entry) + .await; + } + if let Lookup::Miss { version } = entity_lookup { + let entry = EntityStatusCacheEntry { + status: snapshot.entity_status.clone(), + tenant_id: snapshot.entity_tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + // `tenant_key` is derived from the token's `tid` claim, but + // `load_session_entity_tenant` joins tenants through `e.tenant_id` — so + // `snapshot.tenant_status` describes the *entity's current* tenant. The + // two differ whenever a token outlives a tenant move, and populating the + // claim's key with the current tenant's payload would write one tenant's + // status under another tenant's key (e.g. marking a suspended tenant + // active for everyone, off a request that `check_session_entity_tenant` + // is about to reject anyway). Only populate when the version we observed + // belongs to the key the payload actually describes. + if let (Some(tenant_key), Some(Lookup::Miss { version })) = (&tenant_key, tenant_lookup) { + if let (Some(entity_tenant_id), Some(tenant_status)) = + (snapshot.entity_tenant_id, &snapshot.tenant_status) + { + if *tenant_key == cache_keys::tenant_status(entity_tenant_id) { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + }; + cache + .try_populate(CacheCategory::TenantStatus, tenant_key, version, &entry) + .await; + } + } + } + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)) +} + +/// A snapshot of every field `auth_from_api_key` needs, regardless of whether +/// it came from cache hits or a fresh Postgres load — one canonical function +/// (`verify_api_key_snapshot`) validates either shape identically. +struct CredentialSnapshot { + entity_id: Uuid, + tenant_id: Option, + secret_hash: Option, + secret_lookup_hash: Option>, + status: CredentialStatus, + expires_at: Option>, + scoped: bool, + entity_status: EntityStatus, + tenant_status: Option, +} + +impl CredentialSnapshot { + fn from_cache_entries( + cred: &CredentialCacheEntry, + entity: &EntityStatusCacheEntry, + tenant: Option<&TenantStatusCacheEntry>, + ) -> Self { + Self { + entity_id: cred.entity_id, + // From the entity entry, not `cred` — see `CredentialCacheEntry`'s + // doc comment for why the credential entry carries no tenant_id + // of its own. + tenant_id: entity.tenant_id, + secret_hash: cred.secret_hash.clone(), + secret_lookup_hash: cred.secret_lookup_hash.clone(), + status: cred.status.clone(), + expires_at: cred.expires_at, + scoped: cred.scoped, + entity_status: entity.status.clone(), + tenant_status: tenant.map(|t| t.status.clone()), + } + } +} + +fn credential_cache_entry(snapshot: &CredentialSnapshot) -> CredentialCacheEntry { + CredentialCacheEntry { + entity_id: snapshot.entity_id, + status: snapshot.status.clone(), + secret_hash: snapshot.secret_hash.clone(), + secret_lookup_hash: snapshot.secret_lookup_hash.clone(), + expires_at: snapshot.expires_at, + scoped: snapshot.scoped, + } +} + +/// The existing credential/entity/tenant join, unchanged from before caching +/// was introduced. This remains the single cache-miss loader for all three +/// cached entities it touches — it is never split into per-field queries. +async fn load_credential_row(pool: &PgPool, cred_id: Uuid) -> Result { + use sqlx::Row; let row = sqlx::query( r#"SELECT c.entity_id, c.secret_hash, @@ -276,94 +538,125 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result AppError::unauthorized("api key not found"), other => AppError::Database(other), })?; - // Verify the secret before any state checks, so a caller holding only a - // credential ID cannot learn the token's revoked/expired status. - // - // Verifier: keyed HMAC digest when present (tokens minted with a KEK); - // argon2 hash otherwise. See create_access_token for the rationale. - let lookup_hash: Option> = row.try_get("secret_lookup_hash").unwrap_or(None); - let had_lookup_hash = lookup_hash.is_some(); - let verified = match lookup_hash { + Ok(CredentialSnapshot { + entity_id: row.try_get("entity_id").map_err(db_err)?, + tenant_id: row.try_get("tenant_id").unwrap_or(None), + secret_hash: row.try_get("secret_hash").unwrap_or(None), + secret_lookup_hash: row.try_get("secret_lookup_hash").unwrap_or(None), + status: row.try_get("status").map_err(db_err)?, + expires_at: row.try_get("expires_at").unwrap_or(None), + scoped: row.try_get("scoped").unwrap_or(false), + entity_status: row + .try_get("entity_status") + .map_err(|_| AppError::unauthorized("corrupt entity"))?, + tenant_status: row + .try_get::, _>("tenant_status") + .unwrap_or(None), + }) +} + +/// The one canonical set of deny checks for API-key authentication — +/// consumed identically whether `snapshot` came from cache hits or a fresh +/// Postgres load. Secret verification runs before any state check, so a +/// caller holding only a credential ID cannot learn the token's +/// revoked/expired status — caching the row does not change this order. +fn verify_api_key_snapshot( + snapshot: &CredentialSnapshot, + secret_bytes: &[u8], + kek: Option<&crate::config::SecretBytes>, +) -> Result<(), AppError> { + let verified = match &snapshot.secret_lookup_hash { Some(stored) => { - let kek = state - .config - .signing_keys - .key_encryption_key - .as_ref() - .ok_or_else(|| AppError::unauthorized("invalid api key"))?; - crate::crypto::hmac_sha256_verify(kek.expose(), &secret_bytes, &stored) + let kek = kek.ok_or_else(|| AppError::unauthorized("invalid api key"))?; + crate::crypto::hmac_sha256_verify(kek.expose(), secret_bytes, stored) } None => { - let hash: Option = row.try_get("secret_hash").unwrap_or(None); - let hash = hash.ok_or_else(|| AppError::unauthorized("invalid credential"))?; + let hash = snapshot + .secret_hash + .as_deref() + .ok_or_else(|| AppError::unauthorized("invalid credential"))?; use argon2::{ password_hash::{PasswordHash, PasswordVerifier}, Argon2, }; - let parsed = PasswordHash::new(&hash) + let parsed = PasswordHash::new(hash) .map_err(|_| AppError::unauthorized("invalid credential"))?; Argon2::default() - .verify_password(&secret_bytes, &parsed) + .verify_password(secret_bytes, &parsed) .is_ok() } }; if !verified { return Err(AppError::unauthorized("invalid api key")); } - - let status: CredentialStatus = row.try_get("status").map_err(db_err)?; - if status != CredentialStatus::Active { + if snapshot.status != CredentialStatus::Active { return Err(AppError::unauthorized("api key revoked")); } - - let expires_at: Option> = row.try_get("expires_at").unwrap_or(None); - if let Some(exp) = expires_at { + if let Some(exp) = snapshot.expires_at { if exp < Utc::now() { return Err(AppError::unauthorized("api key expired")); } } - - let entity_status: EntityStatus = row - .try_get("entity_status") - .map_err(|_| AppError::unauthorized("corrupt entity"))?; - if entity_status != EntityStatus::Active { + if snapshot.entity_status != EntityStatus::Active { return Err(AppError::unauthorized("entity is not active")); } - if let Some(tenant_status) = row - .try_get::, _>("tenant_status") - .unwrap_or(None) - { - if tenant_status != TenantStatus::Active { + if let Some(tenant_status) = &snapshot.tenant_status { + if *tenant_status != TenantStatus::Active { return Err(AppError::unauthorized("tenant is not active")); } } + Ok(()) +} - let entity_id: Uuid = row.try_get("entity_id").map_err(db_err)?; - - let tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); +/// Runs the shared post-verification steps (opportunistic verifier upgrade, +/// usage stamp, scoped-ceiling load) and builds the `AuthContext`. Shared by +/// every `auth_from_api_key` path (cached or not) so these never drift. +async fn finish_api_key_auth( + state: &AppState, + cred_id: Uuid, + secret_bytes: &[u8], + snapshot: &CredentialSnapshot, +) -> Result { + verify_api_key_snapshot( + snapshot, + secret_bytes, + state.config.signing_keys.key_encryption_key.as_ref(), + )?; // Opportunistic verifier upgrade: a token minted without a KEK verifies via // argon2, paying the KDF on every request. Once a KEK is configured, swap to // the keyed digest on first successful use. Best-effort — a failed upgrade - // must never fail an otherwise-valid authentication. - if !had_lookup_hash { + // must never fail an otherwise-valid authentication. Guarded by the cache + // barrier (not a bare DEL) so a concurrent reader can't repopulate the + // credential cache entry with the stale verifier after this commits. + if snapshot.secret_lookup_hash.is_none() { if let Some(kek) = state.config.signing_keys.key_encryption_key.as_ref() { - let digest = crate::crypto::hmac_sha256(kek.expose(), &secret_bytes); - if let Err(err) = sqlx::query( - "UPDATE credentials SET secret_lookup_hash = $1, secret_hash = NULL WHERE id = $2", + let digest = crate::crypto::hmac_sha256(kek.expose(), secret_bytes); + let credential_key = cache_keys::credential(cred_id); + let result = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + CacheCategory::Credential, + std::slice::from_ref(&credential_key), + || async { + sqlx::query( + "UPDATE credentials SET secret_lookup_hash = $1, secret_hash = NULL WHERE id = $2", + ) + .bind(digest) + .bind(cred_id) + .execute(&state.pool) + .await + .map_err(AppError::Database) + }, ) - .bind(digest) - .bind(cred_id) - .execute(&state.pool) - .await - { + .await; + if let Err(err) = result { tracing::warn!( credential_id = %cred_id, error = %err, @@ -376,6 +669,7 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result Result Result { + let (cred_id, secret_bytes) = + parse_api_key(key).ok_or_else(|| AppError::unauthorized("malformed api key"))?; + + let Some(cache) = &state.cache else { + let row = load_credential_row(&state.pool, cred_id).await?; + return finish_api_key_auth(state, cred_id, &secret_bytes, &row).await; + }; + + let credential_key = cache_keys::credential(cred_id); + let credential_lookup = cache + .lookup::(CacheCategory::Credential, &credential_key) + .await; + + if let Lookup::Hit(cred_entry) = &credential_lookup { + let entity_key = cache_keys::entity_status(cred_entry.entity_id); + let entity_lookup = cache + .lookup::(CacheCategory::EntityStatus, &entity_key) + .await; + + // Tenant context is derived from the entity's own (freshly + // invalidated) cache entry, never from the credential entry: a + // tenant move invalidates `entity_status` but not `credential`, so a + // stale `cred_entry` would otherwise check the entity's *former* + // tenant's status and hand the request an `AuthContext.tenant_id` + // for a tenant the entity no longer belongs to. + if let Lookup::Hit(entity_entry) = &entity_lookup { + let tenant_key = entity_entry.tenant_id.map(cache_keys::tenant_status); + let tenant_lookup = match &tenant_key { + Some(k) => Some( + cache + .lookup::(CacheCategory::TenantStatus, k) + .await, + ), + None => None, + }; + let tenant_hit = match (&tenant_key, &tenant_lookup) { + (None, _) => Some(None), + (Some(_), Some(Lookup::Hit(t))) => Some(Some(t)), + _ => None, + }; + if let Some(tenant_opt) = tenant_hit { + let snapshot = + CredentialSnapshot::from_cache_entries(cred_entry, entity_entry, tenant_opt); + return finish_api_key_auth(state, cred_id, &secret_bytes, &snapshot).await; + } + } + + // Credential hit but entity/tenant status missed (or the entity's + // current tenant's status wasn't cached): reuse the same combined + // query the cold-start path uses below, and best-effort populate + // whichever entries missed — always keyed off `row`'s freshly + // joined entity_id/tenant_id, mirroring the cold-start path exactly, + // never off the credential entry's stale copy. + let row = load_credential_row(&state.pool, cred_id).await?; + // `entity_key` (and the version observed for it) came from + // `cred_entry.entity_id`, which is the credential entry's own, + // possibly stale copy; the payload below comes from `row`. Populate + // only when the two agree, so entity A's key can never be given + // entity B's status and tenant. + if let Lookup::Miss { version } = entity_lookup { + if entity_key == cache_keys::entity_status(row.entity_id) { + let entry = EntityStatusCacheEntry { + status: row.entity_status.clone(), + tenant_id: row.tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + } + if let (Some(tenant_id), Some(tenant_status)) = (row.tenant_id, &row.tenant_status) { + let tenant_key = cache_keys::tenant_status(tenant_id); + if let Lookup::Miss { version } = cache + .lookup::(CacheCategory::TenantStatus, &tenant_key) + .await + { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + }; + cache + .try_populate(CacheCategory::TenantStatus, &tenant_key, version, &entry) + .await; + } + } + return finish_api_key_auth(state, cred_id, &secret_bytes, &row).await; + } + + // Credential missed (or dirty/unavailable): one combined query loads + // everything; populate the credential entry and, best-effort, the + // entity/tenant entries too (each with its own freshly-observed version). + let row = load_credential_row(&state.pool, cred_id).await?; + + if let Lookup::Miss { version } = credential_lookup { + let entry = credential_cache_entry(&row); + cache + .try_populate(CacheCategory::Credential, &credential_key, version, &entry) + .await; + } + // Both keys are already known from `row`, so their version reads go out + // in one pipelined round trip instead of two serial ones. + let entity_key = cache_keys::entity_status(row.entity_id); + let tenant_key = row.tenant_id.map(cache_keys::tenant_status); + let mut batch_keys: Vec<&str> = vec![&entity_key]; + if let Some(key) = &tenant_key { + batch_keys.push(key); + } + let mut raw = cache.lookup_many(&batch_keys).await.into_iter(); + let entity_raw = raw.next().unwrap_or_default(); + let tenant_raw = raw.next(); + + if let Lookup::Miss { version } = cache + .decode::(CacheCategory::EntityStatus, &entity_key, entity_raw) + .await + { + let entry = EntityStatusCacheEntry { + status: row.entity_status.clone(), + tenant_id: row.tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + if let (Some(tenant_key), Some(tenant_raw), Some(tenant_status)) = + (&tenant_key, tenant_raw, &row.tenant_status) + { + if let Lookup::Miss { version } = cache + .decode::(CacheCategory::TenantStatus, tenant_key, tenant_raw) + .await + { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + }; + cache + .try_populate(CacheCategory::TenantStatus, tenant_key, version, &entry) + .await; + } + } + + finish_api_key_auth(state, cred_id, &secret_bytes, &row).await +} + impl AuthContext { /// Load the authenticated entity's canonical grant expansion for one - /// authorization decision. This deliberately hits the DB on each gate/PDP - /// entry so policy writes take effect for the next check, even inside a - /// multi-field GraphQL request. + /// authorization decision. This deliberately hits the DB (or, when + /// caching is enabled, the cache-aside path backed by `self.cache`) on + /// each gate/PDP entry so policy writes take effect for the next check, + /// even inside a multi-field GraphQL request. This is the single + /// canonical cache-aware grants loader; `authz::engine`'s delegated + /// (non-self-check) path shares the same cache category via + /// `load_decision_context`, keyed the same way. pub async fn effective_grants( &self, pool: &PgPool, ) -> Result>, AppError> { - crate::authz::repo::effective_grants_for_subject(pool, self.entity_id) - .await - .map(std::sync::Arc::new) + crate::cache::cached_or_load( + self.cache.as_deref(), + CacheCategory::Grants, + &cache_keys::grants(self.entity_id), + || crate::authz::repo::effective_grants_for_subject(pool, self.entity_id), + ) + .await + .map(std::sync::Arc::new) } /// The permission ceiling to apply when `subject_id` is the token owner diff --git a/src/authz/engine.rs b/src/authz/engine.rs index f6a5c7f..96057b3 100644 --- a/src/authz/engine.rs +++ b/src/authz/engine.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::{ authz::conditions::conditions_match, + cache::{CacheCategory, CacheClient}, error::AppError, models::{ access::{ @@ -260,6 +261,7 @@ async fn load_decision_context( pool: &PgPool, req: &AuthzRequest, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { let Some(entity) = repo::load_authz_subject(pool, req.subject_id).await? else { return Ok(denied(AuthzResponse::deny("subject not found"), None, None)); @@ -342,12 +344,21 @@ async fn load_decision_context( // group membership already resolved recursively, each grant carrying its own // scope/effect/conditions. One match loop replaces the direct/role split. // Reuse the caller's freshly loaded grants when supplied so one PDP - // evaluation does not expand the same self-subject twice. + // evaluation does not expand the same self-subject twice. Otherwise (a + // delegated check about a subject other than the caller) this is the + // canonical cache-aware grants loader shared with `AuthContext:: + // effective_grants` — see `src/cache/mod.rs` for the consistency model. let grants = match cached_grants { Some(grants) => grants, - None => { - std::sync::Arc::new(repo::effective_grants_for_subject(pool, req.subject_id).await?) - } + None => std::sync::Arc::new( + crate::cache::cached_or_load( + cache, + CacheCategory::Grants, + &crate::cache::keys::grants(req.subject_id), + || repo::effective_grants_for_subject(pool, req.subject_id), + ) + .await?, + ), }; Ok(DecisionContext::Ready(Box::new(ReadyContext { @@ -382,28 +393,43 @@ fn scope_target<'a>( /// here from the caller's `AuthContext` (`ceiling_for`), never passed by hand, so /// a call site cannot forget to apply it. The caller's freshly loaded grants are /// reused when the checked subject is the caller itself. +/// +/// The caller's `AuthContext` carries the cache handle backing the +/// Redis-backed grant-expansion cache (see `src/cache/mod.rs`) — a `None` +/// cache (caching disabled, or `AuthContext::default()` in tests) reproduces +/// pre-caching behavior exactly. pub async fn evaluate( pool: &PgPool, req: &AuthzRequest, auth: &crate::auth::AuthContext, ) -> Result { + let cache = auth.cache.as_deref(); let cached_grants = if req.subject_id == auth.entity_id { Some(auth.effective_grants(pool).await?) } else { None }; - evaluate_prepared(pool, req, auth.ceiling_for(req.subject_id), cached_grants).await + evaluate_prepared( + pool, + req, + auth.ceiling_for(req.subject_id), + cached_grants, + cache, + ) + .await } /// Low-level PDP entry taking an explicit ceiling. For unit/parity tests and the /// engine's own internals; production code must call [`evaluate`], which derives -/// the ceiling from the authenticated context. +/// the ceiling from the authenticated context. Always uncached — tests exercise +/// the PDP's decision logic directly; cache behavior has its own dedicated test +/// suite against the real `evaluate`/`explain` entry points. pub async fn evaluate_with_ceiling( pool: &PgPool, req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, ) -> Result { - evaluate_prepared(pool, req, ceiling, None).await + evaluate_prepared(pool, req, ceiling, None, None).await } async fn evaluate_prepared( @@ -411,9 +437,10 @@ async fn evaluate_prepared( req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { let start = std::time::Instant::now(); - let result = evaluate_inner(pool, req, ceiling, cached_grants).await; + let result = evaluate_inner(pool, req, ceiling, cached_grants, cache).await; if let Ok(response) = &result { crate::metrics::record_decision(start.elapsed(), response.allowed); } @@ -425,8 +452,9 @@ async fn evaluate_inner( req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { - let ctx = match load_decision_context(pool, req, cached_grants).await? { + let ctx = match load_decision_context(pool, req, cached_grants, cache).await? { DecisionContext::Denied(denied) => return Ok(denied.response), DecisionContext::Ready(ctx) => ctx, }; @@ -552,13 +580,14 @@ pub async fn explain( req: &AuthzRequest, auth: &crate::auth::AuthContext, ) -> Result { + let cache = auth.cache.as_deref(); let cached_grants = if req.subject_id == auth.entity_id { Some(auth.effective_grants(pool).await?) } else { None }; let ceiling = auth.ceiling_for(req.subject_id); - let ctx = match load_decision_context(pool, req, cached_grants).await? { + let ctx = match load_decision_context(pool, req, cached_grants, cache).await? { DecisionContext::Denied(denied) => { return Ok(AuthzExplainResponse { allowed: false, diff --git a/src/authz/repo.rs b/src/authz/repo.rs index 8692372..a056b55 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use chrono::Utc; +use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::{PgPool, Postgres, Transaction}; use uuid::Uuid; @@ -826,7 +827,7 @@ fn parent_group_id_from_value(value: &Value) -> Result, AppError> { /// /// This is the single canonical grant representation consumed by the PDP and /// (incrementally) the other authorization readers. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct EffectiveGrant { /// The assignment that confers this grant: the `direct_policies.id` or the /// `role_assignments.id` row. With shared blocks this is what identifies @@ -861,7 +862,7 @@ pub struct EffectiveGrant { /// `scoped` records intent independently of `entries`: a scoped token whose limit /// rows were deleted yields `entries = []` and must fail closed (deny everything), /// never silently widen to the owner's full authority. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CredentialCeiling { pub entries: Vec, } @@ -993,8 +994,9 @@ pub async fn create_role_with_assignments( ensure_entities_exist(pool, member_entity_ids).await?; if child_role_ids.is_empty() { + let mut conn = pool.acquire().await.map_err(db_err)?; crate::guardrails::validate_role_assignment_plan( - pool, + &mut conn, member_entity_ids, capability_ids, req.tenant_id, @@ -1004,8 +1006,9 @@ pub async fn create_role_with_assignments( .await?; } else { validate_composite_children(pool, id, req.tenant_id, child_role_ids).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; crate::guardrails::validate_composite_role_assignment_plan( - pool, + &mut conn, member_entity_ids, child_role_ids, req.tenant_id, @@ -1226,10 +1229,56 @@ pub async fn replace_role_permission_block_links_with_audit( role_id: Uuid, permission_block_ids: &[Uuid], ) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + let tenant_id = replace_role_permission_block_links_in_tx( + &mut tx, + events_enabled, + actor_id, + role_id, + permission_block_ids, + ) + .await?; + tx.commit().await.map_err(db_err)?; + // `_in_tx` already enqueued the outbox row via `observe_in_tx` before + // returning — this is the post-commit stdout observability log + // `commit_with_observation` would otherwise provide. + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role", + target_id: Some(role_id), + event: "role.permission_blocks.replace", + }, + &serde_json::json!({ "permission_block_ids": permission_block_ids }), + ); + Ok(()) +} + +/// Body of [`replace_role_permission_block_links`]; caller contract per +/// [`create_role_assignment_in_tx`]. The resolver must already hold the role +/// lock via [`lock_role_and_collect_grants_keys`] on this `tx` — `lock_role` +/// below just re-acquires it (same-transaction no-op). +/// +/// Every read runs on `tx`, never on a pooled connection: the validation +/// below is only meaningful under the role lock this transaction holds, and a +/// second connection acquired mid-transaction is a pool-exhaustion deadlock +/// under concurrency. +/// Returns the role's `tenant_id`, captured here rather than left for the +/// caller to re-derive post-commit — cheaper than an extra query, and safe +/// for any future case where the role itself stops existing by the time the +/// caller wants to log. +pub(crate) async fn replace_role_permission_block_links_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + role_id: Uuid, + permission_block_ids: &[Uuid], +) -> Result, AppError> { let role_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(role_id) - .fetch_optional(pool) + .fetch_optional(&mut **tx) .await .map_err(db_err)? .ok_or_else(|| AppError::not_found(format!("role {role_id} not found")))?; @@ -1247,7 +1296,7 @@ pub async fn replace_role_permission_block_links_with_audit( ) .bind(&unique_block_ids) .bind(role_tenant_id) - .fetch_one(pool) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if count != unique_block_ids.len() as i64 { @@ -1256,18 +1305,17 @@ pub async fn replace_role_permission_block_links_with_audit( )); } } - let mut tx = pool.begin().await.map_err(db_err)?; - lock_role(&mut tx, role_id).await?; + lock_role(tx, role_id).await?; // Validate under the role lock so a concurrent role assignment cannot commit // a prohibited combination against stale state: any other role-link or // assignment mutator blocks on this lock and re-validates against our result. // Runs on this transaction's own connection — borrowing a second one from // the pool here would deadlock a saturated pool. - crate::guardrails::validate_role_permission_block_links(&mut tx, role_id, &unique_block_ids) + crate::guardrails::validate_role_permission_block_links(&mut *tx, role_id, &unique_block_ids) .await?; sqlx::query("DELETE FROM role_permission_blocks WHERE role_id = $1") .bind(role_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -1279,12 +1327,12 @@ pub async fn replace_role_permission_block_links_with_audit( ) .bind(role_id) .bind(permission_block_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } - crate::audit::commit_with_observation( + crate::audit::observe_in_tx( tx, events_enabled, &crate::audit::AuditMeta { @@ -1296,7 +1344,8 @@ pub async fn replace_role_permission_block_links_with_audit( }, &serde_json::json!({ "permission_block_ids": unique_block_ids }), ) - .await + .await?; + Ok(role_tenant_id) } async fn insert_role_permission_block( @@ -2797,10 +2846,38 @@ pub async fn delete_role_with_audit( deleted_by: Option, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + let tenant_id = delete_role_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role", + target_id: Some(id), + event: "role.delete", + }, + &serde_json::json!({}), + ); + Ok(()) +} + +/// Body of [`delete_role`]; caller contract per +/// [`create_role_assignment_in_tx`] — the resolver must already hold the +/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. Returns +/// the role's `tenant_id`, captured here rather than left for the caller to +/// re-derive post-commit — a re-read after this commits would always miss +/// the now-deleted row. +pub(crate) async fn delete_role_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + deleted_by: Option, +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { @@ -2812,7 +2889,7 @@ pub async fn delete_role_with_audit( ) .bind(id) .bind(deleted_by) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; if result.rows_affected() == 0 { @@ -2826,8 +2903,8 @@ pub async fn delete_role_with_audit( event: "role.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(tenant_id) } pub async fn delete_role( @@ -2845,9 +2922,42 @@ pub async fn restore_role_with_audit( id: Uuid, restored_by: Option, ) -> Result<(), AppError> { - let _ = restored_by; let mut tx = pool.begin().await.map_err(db_err)?; + restore_role_in_tx(&mut tx, events_enabled, actor_id, id, restored_by).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation inside `restore_role_in_tx` via `observe_in_tx`. + let tenant_id = get_role(pool, id).await.ok().and_then(|r| r.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("role"), + target_id: Some(id), + event: "role.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} +/// Body of [`restore_role`]; caller contract per +/// [`create_role_assignment_in_tx`] — the resolver must already hold the +/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. +pub(crate) async fn restore_role_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result<(), AppError> { + let _ = restored_by; let tenant_info: Option<(Option, bool)> = sqlx::query_as( "SELECT r.tenant_id, (t.deleted_at IS NOT NULL) FROM roles r @@ -2855,7 +2965,7 @@ pub async fn restore_role_with_audit( WHERE r.id = $1 AND r.deleted_at IS NOT NULL", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let (tenant_id, _is_tenant_deleted) = match tenant_info { @@ -2877,20 +2987,18 @@ pub async fn restore_role_with_audit( WHERE id = $1 AND deleted_at IS NOT NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(restore_conflict)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("role"), + target_kind: "role", target_id: Some(id), event: "role.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; Ok(()) } @@ -2978,7 +3086,9 @@ pub async fn add_role_capability( let scope_ref = role.tenant_id.map(|tenant_id| tenant_id.to_string()); validate_capabilities_against_role_scope(pool, &scope_kind, scope_ref.as_deref(), &[cap_id]) .await?; - crate::guardrails::validate_role_capability(pool, role_id, cap_id).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; + crate::guardrails::validate_role_capability(&mut conn, role_id, cap_id).await?; + drop(conn); let mut tx = pool.begin().await.map_err(db_err)?; lock_role(&mut tx, role_id).await?; insert_role_capability_as_permission_block( @@ -3856,7 +3966,9 @@ pub async fn create_policy( pool: &PgPool, req: CreatePolicyBinding, ) -> Result { - crate::guardrails::validate_policy(pool, &req).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; + crate::guardrails::validate_policy(&mut conn, &req).await?; + drop(conn); let id = Uuid::new_v4(); let membership_tenant_id = req.tenant_id; let membership_entity_id = req.subject_id; @@ -3993,9 +4105,44 @@ pub async fn create_role_assignment_with_audit( req: CreateRoleAssignment, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; - lock_live_subject(&mut tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; - lock_role(&mut tx, req.role_id).await?; - validate_role_assignment(pool, &req).await?; + let assignment = create_role_assignment_in_tx(&mut tx, events_enabled, actor_id, req).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: assignment.tenant_id, + target_kind: "role_assignment", + target_id: Some(assignment.id), + event: "role_assignment.create", + }, + &serde_json::json!({}), + ); + Ok(assignment) +} + +/// Body of [`create_role_assignment`], callable directly against a caller-held +/// `tx` (not committed here). For a group subject, the caller must already +/// have run [`lock_group_closures_and_collect_grants_keys`] on this same `tx` +/// and called `cache.begin()` on the result — `lock_role`/`lock_live_subject` +/// below then just re-acquire those same locks (safe, same-transaction no-op). +/// Every other `_in_tx` twin in this module and `identity::repo` follows this +/// same convention: caller locks + begins the cache barrier first, this kind +/// of function re-acquires (never re-validates) those locks, and the caller +/// commits. +pub(crate) async fn create_role_assignment_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + req: CreateRoleAssignment, +) -> Result { + lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; + // Lock the role and validate under the lock so a concurrent block-link + // mutation cannot add a prohibited block against stale state: it blocks on + // this same lock and re-validates against the assignment we are inserting. + // Validation must run on `tx` for that to hold at all — the pool variant + // would neither see the locked state nor respect the locks. + lock_role(tx, req.role_id).await?; + validate_role_assignment_in_tx(tx, &req).await?; let assignment = sqlx::query_as::<_, RoleAssignment>( r#"INSERT INTO role_assignments (tenant_id, subject_kind, subject_id, role_id) @@ -4006,7 +4153,7 @@ pub async fn create_role_assignment_with_audit( .bind(req.subject_kind) .bind(req.subject_id) .bind(req.role_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -4018,7 +4165,7 @@ pub async fn create_role_assignment_with_audit( event: "role_assignment.create", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(assignment) } @@ -4029,61 +4176,16 @@ pub async fn create_role_assignment( create_role_assignment_with_audit(pool, false, None, req).await } -pub async fn delete_role_assignment_with_audit( - pool: &PgPool, - events_enabled: bool, - actor_id: Option, - id: Uuid, -) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; - let assignment_tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM role_assignments WHERE id = $1") - .bind(id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - let Some(tenant_id) = assignment_tenant_id else { - return Err(AppError::not_found(format!( - "role assignment {id} not found" - ))); - }; - let result = sqlx::query("DELETE FROM role_assignments WHERE id = $1") - .bind(id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - if result.rows_affected() == 0 { - return Err(AppError::not_found(format!( - "role assignment {id} not found" - ))); - } - let meta = crate::audit::AuditMeta { - actor_entity_id: actor_id, - tenant_id, - target_kind: "role_assignment", - target_id: Some(id), - event: "role_assignment.delete", - }; - let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) -} - -pub async fn delete_role_assignment(pool: &PgPool, id: Uuid) -> Result<(), AppError> { - delete_role_assignment_with_audit(pool, false, None, id).await -} - /// Returns `true` when a new assignment row was actually inserted, so callers /// can tell a real state change from an idempotent no-op and decide whether the /// operation is worth publishing as a domain event. pub(crate) async fn create_role_assignment_if_missing_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, req: &CreateRoleAssignment, ) -> Result { lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; lock_role(tx, req.role_id).await?; - validate_role_assignment_in_tx(pool, tx, req).await?; + validate_role_assignment_in_tx(tx, req).await?; let inserted = sqlx::query( r#"INSERT INTO role_assignments (tenant_id, subject_kind, subject_id, role_id) @@ -4185,20 +4287,116 @@ pub async fn get_role_assignment(pool: &PgPool, id: Uuid) -> Result Result<(), AppError> { + delete_role_assignment_with_audit(pool, false, None, id).await +} + +pub async fn delete_role_assignment_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + let tenant_id = delete_role_assignment_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }, + &serde_json::json!({}), + ); + Ok(()) +} + +/// Body of [`delete_role_assignment`]; caller contract per +/// [`create_role_assignment_in_tx`] — the group-subject resolver path must +/// already hold the subject's group closure lock on this `tx`. Returns the +/// assignment's `tenant_id`, captured here rather than left for the caller +/// to re-derive post-commit — the row is gone by then. +pub(crate) async fn delete_role_assignment_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result, AppError> { + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM role_assignments WHERE id = $1") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let Some(tenant_id) = tenant_id else { + return Err(AppError::not_found(format!( + "role assignment {id} not found" + ))); + }; + // A role assignment is a 'policy' protected object; the policy-object cleanup trigger + // sweeps the permission blocks targeting it when this row is deleted. + let result = sqlx::query("DELETE FROM role_assignments WHERE id = $1") + .bind(id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + if result.rows_affected() == 0 { + return Err(AppError::not_found(format!( + "role assignment {id} not found" + ))); + } + let meta = crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }; + let details = serde_json::json!({}); + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(tenant_id) +} + pub async fn create_direct_policy_with_audit( pool: &PgPool, events_enabled: bool, actor_id: Option, req: CreateDirectPolicy, ) -> Result { - validate_direct_policy(pool, &req).await?; - crate::guardrails::validate_direct_policy(pool, &req).await?; let mut tx = pool.begin().await.map_err(db_err)?; - lock_live_subject(&mut tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; + let policy = create_direct_policy_in_tx(&mut tx, events_enabled, actor_id, req).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: policy.tenant_id, + target_kind: "direct_policy", + target_id: Some(policy.id), + event: "direct_policy.create", + }, + &serde_json::json!({}), + ); + Ok(policy) +} + +/// Body of [`create_direct_policy`]; caller contract per +/// [`create_role_assignment_in_tx`]. Validates on `tx` rather than a pooled +/// connection — see [`replace_role_permission_block_links_in_tx`]. +pub(crate) async fn create_direct_policy_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + req: CreateDirectPolicy, +) -> Result { + validate_direct_policy_in_tx(tx, &req).await?; + crate::guardrails::validate_direct_policy(tx, &req).await?; + lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; let block_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM permission_blocks WHERE id = $1 FOR UPDATE") .bind(req.permission_block_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if block_tenant_id != Some(req.tenant_id) { @@ -4216,7 +4414,7 @@ pub async fn create_direct_policy_with_audit( .bind(req.subject_kind) .bind(req.subject_id) .bind(req.permission_block_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -4228,7 +4426,7 @@ pub async fn create_direct_policy_with_audit( event: "direct_policy.create", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(policy) } @@ -4314,10 +4512,35 @@ pub async fn delete_direct_policy_with_audit( id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + let tenant_id = delete_direct_policy_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "direct_policy", + target_id: Some(id), + event: "direct_policy.delete", + }, + &serde_json::json!({}), + ); + Ok(()) +} + +/// Body of [`delete_direct_policy`]; caller contract per +/// [`create_role_assignment_in_tx`]. Returns the policy's `tenant_id`, +/// captured here rather than left for the caller to re-derive post-commit — +/// the row is gone by then. +pub(crate) async fn delete_direct_policy_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result, AppError> { let policy_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM direct_policies WHERE id = $1") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = policy_tenant_id else { @@ -4327,13 +4550,16 @@ pub async fn delete_direct_policy_with_audit( "DELETE FROM direct_policies WHERE id = $1 RETURNING permission_block_id", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(block_id) = block_id else { return Err(AppError::not_found(format!("direct policy {id} not found"))); }; - delete_orphaned_blocks(&mut tx, &[block_id]).await?; + // The block is shared: GC it only if removing this policy left it + // unreferenced (mirrors delete_policy). Blocks targeting this policy *as an + // object* are swept by the policy-object cleanup trigger on the delete above. + delete_orphaned_blocks(tx, &[block_id]).await?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, @@ -4343,43 +4569,15 @@ pub async fn delete_direct_policy_with_audit( event: "direct_policy.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(tenant_id) } pub async fn delete_direct_policy(pool: &PgPool, id: Uuid) -> Result<(), AppError> { delete_direct_policy_with_audit(pool, false, None, id).await } -async fn validate_role_assignment( - pool: &PgPool, - req: &CreateRoleAssignment, -) -> Result<(), AppError> { - let role_tenant_id: Option = - sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") - .bind(req.role_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| AppError::bad_request("role assignment references unknown role"))?; - if role_tenant_id != req.tenant_id { - return Err(AppError::bad_request( - "role assignment tenantId must match role tenantId", - )); - } - validate_subject_boundary(pool, req.tenant_id, &req.subject_kind, req.subject_id).await?; - crate::guardrails::validate_role_assignment( - pool, - req.tenant_id, - req.subject_kind.clone(), - req.subject_id, - req.role_id, - ) - .await -} - async fn validate_role_assignment_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, req: &CreateRoleAssignment, ) -> Result<(), AppError> { @@ -4397,7 +4595,7 @@ async fn validate_role_assignment_in_tx( } validate_subject_boundary_in_tx(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; crate::guardrails::validate_role_assignment( - pool, + tx, req.tenant_id, req.subject_kind.clone(), req.subject_id, @@ -4406,11 +4604,14 @@ async fn validate_role_assignment_in_tx( .await } -async fn validate_direct_policy(pool: &PgPool, req: &CreateDirectPolicy) -> Result<(), AppError> { +async fn validate_direct_policy_in_tx( + tx: &mut Transaction<'_, Postgres>, + req: &CreateDirectPolicy, +) -> Result<(), AppError> { let block_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM permission_blocks WHERE id = $1") .bind(req.permission_block_id) - .fetch_optional(pool) + .fetch_optional(&mut **tx) .await .map_err(db_err)? .ok_or_else(|| { @@ -4421,7 +4622,7 @@ async fn validate_direct_policy(pool: &PgPool, req: &CreateDirectPolicy) -> Resu "direct policy tenantId must match permission block tenantId", )); } - validate_subject_boundary(pool, req.tenant_id, &req.subject_kind, req.subject_id).await + validate_subject_boundary_in_tx(tx, req.tenant_id, &req.subject_kind, req.subject_id).await } async fn validate_subject_boundary_in_tx( @@ -4484,66 +4685,6 @@ async fn validate_subject_boundary_in_tx( Ok(()) } -async fn validate_subject_boundary( - pool: &PgPool, - tenant_id: Option, - subject_kind: &SubjectKind, - subject_id: Uuid, -) -> Result<(), AppError> { - match subject_kind { - SubjectKind::Entity => { - let entity_tenant_id: Option = sqlx::query_scalar( - "SELECT tenant_id FROM entities WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(subject_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| AppError::bad_request("assignment references unknown entity"))?; - if let Some(tenant_id) = tenant_id { - let member: bool = sqlx::query_scalar( - r#"SELECT EXISTS ( - SELECT 1 FROM tenant_memberships - WHERE tenant_id = $1 AND entity_id = $2 AND status = 'active' - )"#, - ) - .bind(tenant_id) - .bind(subject_id) - .fetch_one(pool) - .await - .map_err(db_err)?; - if entity_tenant_id != Some(tenant_id) && !member { - return Err(AppError::bad_request( - "tenant assignment subject entity must belong to the tenant", - )); - } - } else if entity_tenant_id.is_some() { - return Err(AppError::bad_request( - "platform assignment cannot target tenant-owned entity", - )); - } - } - SubjectKind::Group => { - let group_tenant_id: Option = sqlx::query_scalar( - "SELECT tenant_id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(subject_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| { - AppError::bad_request("assignment references unknown principal group") - })?; - if group_tenant_id != tenant_id { - return Err(AppError::bad_request( - "assignment subject principal group must be in the same tenant", - )); - } - } - } - Ok(()) -} - pub async fn subject_role_assignments( pool: &PgPool, params: SubjectRoleAssignmentsQuery, @@ -5795,6 +5936,186 @@ pub async fn effective_grants_for_subject( .collect() } +/// Locks every group in the closure rooted at `root_group_ids` (each root +/// plus every descendant subgroup, via a `group_hierarchy` descent) `FOR +/// UPDATE` in `principal_groups` — in a stable, id-sorted order, so this can +/// never deadlock against another +/// call to this same function locking an overlapping closure — then returns +/// the member entity ids across that (now-locked) closure. +/// +/// # Why this exists +/// +/// A plain "enumerate current members, then invalidate" has a real race: a +/// group-subject mutation (direct policy / role assignment / role +/// block-links / group status-or-hierarchy change) can enumerate members at +/// one instant, while `identity::repo::add_group_member` concurrently +/// commits a *new* member in between that enumeration and the mutation's own +/// commit. That new member's `grants` key is never in the enumerated set, so +/// it never gets invalidated — a stale cached grant (or a stale cached +/// *lack* of one) can then survive until the grants TTL. (Reported by +/// external review, 2026-07-29.) +/// +/// `add_group_member`/`remove_group_member` already lock the group's row in +/// `principal_groups` `FOR UPDATE` before changing membership. Locking that +/// same row here — and holding it for the caller's entire transaction, +/// through commit — closes the race: neither side's transaction can commit +/// while the other holds the lock, so whichever runs first is fully visible +/// (including its own cache invalidation) before the other's enumeration or +/// commit proceeds. +/// +/// # What this does *not* cover +/// +/// The closure itself (`root_group_ids` plus descendants) is computed with +/// one unlocked read before any row is locked, so a concurrent *reparent* +/// that brings a brand-new subgroup into range between that read and the +/// lock isn't covered by this function — an accepted, documented residual +/// gap (bounded by the grants TTL, exactly as every enumeration-based +/// invalidation was before this fix). Closing that would mean locking +/// hierarchy structure itself for every membership mutation everywhere, a +/// far larger scope than the membership-add race this closes. +/// +/// # Contract for callers +/// +/// The returned ids are only exhaustive as long as the lock stays held: the +/// caller must keep `tx` open (no commit) from this call through the end of +/// its own mutation's commit, and should call `cache.begin()` on the +/// resulting keys *before* that commit — mirroring `guarded_mutation`'s +/// begin/mutate/end shape, just with `mutate` running against this +/// already-open, already-locked `tx` instead of opening its own. +pub async fn lock_group_closures_and_collect_member_ids( + tx: &mut Transaction<'_, Postgres>, + root_group_ids: &[Uuid], +) -> Result, AppError> { + if root_group_ids.is_empty() { + return Ok(Vec::new()); + } + let mut closure: Vec = sqlx::query_scalar( + r#"WITH RECURSIVE target_groups(id) AS ( + SELECT id FROM UNNEST($1::uuid[]) AS root(id) + UNION + SELECT gh.child_id + FROM group_hierarchy gh + JOIN target_groups tg ON tg.id = gh.parent_id + ) + SELECT id FROM target_groups"#, + ) + .bind(root_group_ids) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + closure.sort_unstable(); + closure.dedup(); + + // Locks whichever of these ids are principal groups (object groups have + // no members and simply won't match here — harmless, not an error). + sqlx::query("SELECT id FROM principal_groups WHERE id = ANY($1) ORDER BY id FOR UPDATE") + .bind(&closure) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + sqlx::query_scalar("SELECT DISTINCT entity_id FROM group_members WHERE group_id = ANY($1)") + .bind(&closure) + .fetch_all(&mut **tx) + .await + .map_err(db_err) +} + +/// [`lock_group_closures_and_collect_member_ids`], mapped straight to +/// `atom:v1:grants:*` cache keys — the form every locked group-subject +/// mutation call site actually wants. +pub async fn lock_group_closures_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + root_group_ids: &[Uuid], +) -> Result, AppError> { + Ok( + lock_group_closures_and_collect_member_ids(tx, root_group_ids) + .await? + .into_iter() + .map(crate::cache::keys::grants) + .collect(), + ) +} + +/// Like [`lock_group_closures_and_collect_grants_keys`], for a role: locks +/// the role row itself plus every group in the closure of every group +/// directly assigned this role, and returns the combined set of affected +/// `atom:v1:grants:*` keys (entity-direct assignees plus every locked +/// group's members). Entity-direct assignees need no lock of their own — +/// the affected key is exactly their `subject_id`, deterministic and +/// race-free. +/// +/// Locks the role row directly (`SELECT ... FOR UPDATE`, no `deleted_at` +/// filter) rather than going through the private, alive-only `lock_role` — +/// `restore_role` is a caller here and operates on an *already* +/// soft-deleted role, so requiring "alive" would always fail for it. Each +/// caller's own mutation separately validates/enforces whatever alive-ness +/// it needs; this only needs the lock (for closure consistency) and the +/// current assignee set (which was never conditioned on the role's own +/// status either, before this fix). +pub async fn lock_role_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + role_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar::<_, Uuid>("SELECT id FROM roles WHERE id = $1 FOR UPDATE") + .bind(role_id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let entity_subject_ids: Vec = sqlx::query_scalar( + "SELECT subject_id FROM role_assignments WHERE role_id = $1 AND subject_kind = 'entity'", + ) + .bind(role_id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + let group_subject_ids: Vec = sqlx::query_scalar( + "SELECT subject_id FROM role_assignments WHERE role_id = $1 AND subject_kind = 'group'", + ) + .bind(role_id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + let mut member_ids = lock_group_closures_and_collect_member_ids(tx, &group_subject_ids).await?; + member_ids.extend(entity_subject_ids); + member_ids.sort_unstable(); + member_ids.dedup(); + Ok(member_ids + .into_iter() + .map(crate::cache::keys::grants) + .collect()) +} + +/// Locks a role and a group (in that order) for the one mutation that needs +/// both in the same transaction: creating a role assignment for a group +/// subject. **Lock order matters here**: [`lock_role_and_collect_grants_keys`] +/// (used by `replaceRolePermissionBlocks`/`deleteRole`/`restoreRole`) always +/// locks the role first, then the closures of every group assigned it. If +/// `createRoleAssignment` locked the *subject group* first and only reached +/// the role lock afterward (inside `create_role_assignment_in_tx`'s own +/// `lock_role` call — which is exactly what the original code did), a +/// concurrent pair of requests could deadlock: one holding the descendant +/// group's row while waiting on the role, the other holding the role while +/// waiting on that same descendant group (reached via an ancestor group +/// already assigned the role). Postgres detects the cycle and aborts one +/// side. (Reported by external review, 2026-07-29.) +/// +/// Locking the role first here — before the group closure — makes +/// `createRoleAssignment`'s lock order match every role-mutation path +/// exactly, closing the inversion. `create_role_assignment_in_tx`'s own +/// `lock_role`/`lock_live_subject` calls afterward re-acquire the same locks +/// (a same-transaction no-op) — kept as-is since that function is also used, +/// unlocked, by the entity-subject path, which has no group lock to order +/// against. +pub async fn lock_role_then_group_closure_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + role_id: Uuid, + subject_group_id: Uuid, +) -> Result, AppError> { + lock_role(tx, role_id).await?; + lock_group_closures_and_collect_grants_keys(tx, &[subject_group_id]).await +} + pub async fn find_capability_ids_by_name( pool: &PgPool, name: &str, diff --git a/src/cache/entries.rs b/src/cache/entries.rs new file mode 100644 index 0000000..f86aa07 --- /dev/null +++ b/src/cache/entries.rs @@ -0,0 +1,66 @@ +//! Cache-transport DTOs. Deliberately distinct from the DB row models in +//! `crate::models` — each holds only the fields its read path actually needs, +//! never a full row dump, and never a plaintext secret. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::models::enums::{CredentialStatus, EntityStatus, TenantStatus}; + +/// The current SQL check backing `auth_from_jwt` verifies both the session id +/// *and* that it belongs to the claimed entity (`WHERE s.id = $1 AND +/// s.entity_id = $2`). `entity_id` is carried here so a cache hit can +/// re-verify that same invariant against the JWT's `sub` claim, rather than +/// trusting the key lookup alone. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionCacheEntry { + pub entity_id: Uuid, + pub revoked_at: Option>, + pub expires_at: DateTime, +} + +/// Shared between JWT and API-key authentication — one entity deactivation +/// invalidates both paths' view of the entity at once. +/// +/// Deliberately has no `deleted_at` field. Both miss loaders +/// (`auth::load_session_entity_tenant`, `auth::load_credential_row`) filter +/// `e.deleted_at IS NULL`, so an entry can only ever be populated from a +/// live row — a cached `deleted_at` would be `None` by construction and any +/// check against it a no-op. Denying a *subsequently* soft-deleted entity is +/// the soft-delete path's invalidation duty (`entity_status`, plus the +/// entity's sessions and credentials), not this entry's. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntityStatusCacheEntry { + pub status: EntityStatus, + pub tenant_id: Option, +} + +/// Has no `deleted_at` field, for the same reason as +/// [`EntityStatusCacheEntry`]: both miss loaders filter `t.deleted_at IS +/// NULL`, and denying a later-deleted tenant is the tenant delete path's +/// invalidation duty. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TenantStatusCacheEntry { + pub status: TenantStatus, +} + +/// Never carries the plaintext API-key secret — only what +/// `auth_from_api_key`'s existing verification step consumes. +/// +/// Deliberately has no `tenant_id` field: a credential's tenant is really the +/// owning entity's tenant, which can change (entity moved to another tenant) +/// independently of this entry ever being invalidated. A duplicated copy here +/// would go stale on that move with nothing to invalidate it, and +/// `auth_from_api_key` would have no way to know. Always read tenant context +/// from the entity's own `EntityStatusCacheEntry` instead, which *is* +/// invalidated on a tenant move. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CredentialCacheEntry { + pub entity_id: Uuid, + pub status: CredentialStatus, + pub secret_hash: Option, + pub secret_lookup_hash: Option>, + pub expires_at: Option>, + pub scoped: bool, +} diff --git a/src/cache/invalidate.rs b/src/cache/invalidate.rs new file mode 100644 index 0000000..cf1dc4d --- /dev/null +++ b/src/cache/invalidate.rs @@ -0,0 +1,130 @@ +//! The single orchestration primitive every security-sensitive mutation goes +//! through: establish the barrier on the affected cache keys, run the +//! Postgres mutation, then clear the barrier — regardless of the mutation's +//! outcome. See `src/cache/mod.rs` for the consistency model this +//! implements. +//! +//! Callers determine *which* keys are affected (a single subject, or an +//! enumerated set — see `authz::repo::affected_subject_ids_for_role` / +//! `affected_subject_ids_for_group`) before calling this; the enumeration +//! itself is domain SQL and does not belong here. + +use std::{future::Future, pin::Pin}; + +use sqlx::{PgPool, Postgres, Transaction}; + +use super::{CacheCategory, CacheClient}; +use crate::error::{db_err, AppError}; + +/// A boxed future borrowing the transaction it runs on — what +/// [`guarded_tx_mutation`]'s closures return. Stable Rust has no async +/// closures, so a closure that awaits while holding `&mut Transaction` has to +/// spell the borrow out by hand; call sites write `|tx| Box::pin(f(tx, ..))`. +pub type TxFuture<'a, T> = Pin> + Send + 'a>>; + +/// Runs `mutate` guarded by a cache barrier on `keys`. With caching disabled +/// (`cache: None`) this is a pure passthrough to `mutate`, byte-identical to +/// not having a cache at all. +/// +/// If establishing the barrier fails while caching is enabled, the mutation +/// is refused (`mutate` is never called) rather than committing a Postgres +/// change the cache cannot be told about — see the module-level consistency +/// model in `src/cache/mod.rs`. +pub async fn guarded_mutation( + cache: Option<&CacheClient>, + category: CacheCategory, + keys: &[String], + mutate: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + let Some(cache) = cache else { + return mutate().await; + }; + + cache.begin(category, keys).await?; + let result = mutate().await; + cache.end(category, keys).await; + result +} + +/// [`guarded_mutation`] for a mutation whose affected keys can only be +/// determined *under the locks the mutation itself takes* — every +/// group-closure and role mutation, where the set of affected `grants` keys +/// is the enumerated member set and a concurrent membership change would +/// otherwise slip past the enumeration (see +/// `authz::repo::lock_group_closures_and_collect_member_ids`). +/// +/// Opens the transaction, runs `collect_keys` on it (which locks and +/// enumerates), establishes the barrier on what it returned, runs `mutate` on +/// the same transaction, commits, and clears the barrier — in that order, +/// regardless of outcome. That ordering is the whole point of the helper: the +/// barrier must be established after the lock (so the enumeration is +/// complete) and cleared after the commit (so no reader repopulates from +/// pre-commit state), and every call site previously re-derived it by hand. +/// +/// Callers hold the `None` cache case themselves, since the uncached +/// fallback is a different repo function per call site rather than the same +/// one — usually the non-`_in_tx` variant that opens its own transaction. +pub async fn guarded_tx_mutation( + cache: &CacheClient, + category: CacheCategory, + pool: &PgPool, + collect_keys: K, + mutate: M, +) -> Result +where + K: for<'a> FnOnce(&'a mut Transaction<'static, Postgres>) -> TxFuture<'a, Vec>, + M: for<'a> FnOnce(&'a mut Transaction<'static, Postgres>) -> TxFuture<'a, T>, +{ + let mut tx = pool.begin().await.map_err(db_err)?; + let keys = collect_keys(&mut tx).await?; + cache.begin(category, &keys).await?; + let outcome = mutate(&mut tx).await; + let outcome = match outcome { + Ok(value) => tx.commit().await.map_err(db_err).map(|_| value), + Err(err) => Err(err), + }; + cache.end(category, &keys).await; + outcome +} + +/// Establishes a barrier on every `(category, keys)` group, in order. Used by +/// callers that already hold an open `Transaction` across their own +/// lock/enumerate/mutate/commit sequence (so the barrier can't be established +/// via a `FnOnce` closure the way [`guarded_mutation`] does). Prefer +/// [`guarded_tx_mutation`] for the single-category case it covers; `begin_all` +/// remains for mutations spanning several categories on one transaction. +/// +/// If a later group's barrier can't be established, the ones already +/// established are cleared immediately rather than left to self-heal on their +/// barrier TTL. Pair with [`end_all`], called unconditionally after the +/// mutation regardless of outcome. +pub async fn begin_all( + cache: &CacheClient, + groups: &[(CacheCategory, &[String])], +) -> Result<(), AppError> { + let mut established = Vec::with_capacity(groups.len()); + for &(category, keys) in groups { + match cache.begin(category, keys).await { + Ok(()) => established.push((category, keys)), + Err(err) => { + for (category, keys) in established { + cache.end(category, keys).await; + } + return Err(err); + } + } + } + Ok(()) +} + +/// Clears the barrier on every `(category, keys)` group established by a +/// prior [`begin_all`] call. Always best-effort, mirroring [`CacheClient::end`]. +pub async fn end_all(cache: &CacheClient, groups: &[(CacheCategory, &[String])]) { + for &(category, keys) in groups { + cache.end(category, keys).await; + } +} diff --git a/src/cache/keys.rs b/src/cache/keys.rs new file mode 100644 index 0000000..564b562 --- /dev/null +++ b/src/cache/keys.rs @@ -0,0 +1,30 @@ +//! Pure key-naming functions. Namespaced `atom:v1:` so an incompatible future +//! payload-shape change can roll out as `atom:v2:` without needing a flush. + +use uuid::Uuid; + +const NAMESPACE: &str = "atom:v1"; + +pub fn session(session_id: Uuid) -> String { + format!("{NAMESPACE}:session:{session_id}") +} + +pub fn entity_status(entity_id: Uuid) -> String { + format!("{NAMESPACE}:entity_status:{entity_id}") +} + +pub fn tenant_status(tenant_id: Uuid) -> String { + format!("{NAMESPACE}:tenant_status:{tenant_id}") +} + +pub fn credential(credential_id: Uuid) -> String { + format!("{NAMESPACE}:credential:{credential_id}") +} + +pub fn cred_ceiling(credential_id: Uuid) -> String { + format!("{NAMESPACE}:cred_ceiling:{credential_id}") +} + +pub fn grants(subject_id: Uuid) -> String { + format!("{NAMESPACE}:grants:{subject_id}") +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs new file mode 100644 index 0000000..dc29ccb --- /dev/null +++ b/src/cache/mod.rs @@ -0,0 +1,1222 @@ +//! Redis-backed cache for AuthN/AuthZ decision *inputs* (never decisions +//! themselves — see `src/authz/engine.rs` and `src/auth.rs` for what actually +//! decides allow/deny). Postgres remains the sole source of truth. +//! +//! # Consistency model +//! +//! Every cached entry is a Redis hash with three fields: `v` (an integer +//! version, bumped on every mutation that can affect the entry), `dirty` (an +//! integer *nesting counter* — above zero while one or more mutations are in +//! flight, zero or absent otherwise), and `p` (the serialized payload, +//! present only when the entry holds a valid value). +//! +//! Three primitives, each a small atomic Lua script, implement a per-key +//! mutation barrier that prevents four races that a plain cache-aside + +//! post-commit `DEL` cannot: a read started *before* a mutation repopulating +//! the cache with stale data after the mutation's invalidation ran, a read +//! started *during* the mutation's dirty window doing the same once the +//! mutation finishes, a lost invalidation silently resurrecting a revoked +//! value, and — the reason `dirty` is a counter rather than a flag — two +//! overlapping mutations on the same key finishing at different times: +//! +//! - `begin` — called before a security-sensitive Postgres mutation. +//! Increments the version and the dirty counter, clears any payload, and +//! bounds the barrier itself with an expiry so a lost `end` call self-heals +//! rather than leaving the entry dirty forever. +//! - `end` — called after the mutation (success or failure). Bumps the +//! version *again*, decrements the dirty counter, and clears any payload — +//! the next reader does a clean reload. The second version bump (beyond the +//! one `begin` already did) is what closes the dirty-window race below — it +//! is not merely decrementing a counter. +//! - `try_populate` — called by a cache-miss read after it finishes loading +//! from Postgres. Writes the payload only if the dirty counter is zero and +//! the version still matches what the reader observed before it started +//! loading; otherwise the write is silently discarded. +//! +//! The dirty-window race the version's second bump defeats (found by external +//! review, 2026-07-29 — the original `end` only cleared `dirty`, without +//! re-bumping the version): a reader's `lookup` can land *while* a mutation is +//! mid-flight (`dirty > 0`), observe the *post-`begin`* version, then proceed +//! to load from Postgres — possibly reading the pre-mutation state, since the +//! mutation's own Postgres write may not have committed yet. If `end` only +//! cleared `dirty` without moving the version again, that reader's later +//! `try_populate` call, run after `end`, would find the version *unchanged* +//! since the moment it was observed and would succeed — re-caching a stale +//! value for the mutation's category for a full TTL, exactly during a +//! revoke/policy-change race. Bumping the version in `end` too means any +//! version a reader could have observed during the dirty window is +//! guaranteed stale by the time `end` finishes, so `try_populate` always +//! rejects it — whether it runs while still dirty (rejected by the `dirty` +//! check) or after `end` (rejected by the version check). +//! +//! The overlapping-mutations race a counter (rather than a `0`/`1` flag) +//! defeats (also found by external review, 2026-07-29): if two +//! security-sensitive mutations both touch the same key — say, two role +//! changes affecting the same subject's `grants` entry — and a boolean `dirty` +//! flag is unconditionally cleared by whichever mutation's `end` runs first, +//! a reader landing after that first `end` but before the *second* mutation's +//! commit sees a clean, non-dirty entry and can `try_populate` a payload +//! loaded from the pre-second-mutation database state. That entry is wrong +//! the instant the second mutation commits, and if the second mutation's own +//! `end` is ever delayed or lost, nothing else corrects it before the TTL. +//! Making `dirty` a nesting counter — incremented by every `begin`, +//! decremented by every `end` — means the barrier only reads as clean once +//! *every* overlapping mutation on the key has called `end`, so a reader can +//! never land in the gap between one mutation's `end` and another's commit. +//! +//! Reads never depend on Redis being reachable: any error (timeout, +//! connection failure, corrupt payload) is treated as a miss and falls +//! through to the caller's Postgres loader. `begin` is the one exception — +//! while caching is enabled, a `begin` failure refuses the mutation rather +//! than committing a change the cache cannot be told about (see +//! `src/cache/invalidate.rs`). + +pub mod entries; +pub mod invalidate; +pub mod keys; + +use std::{future::Future, time::Duration}; + +use deadpool_redis::{Config as PoolConfig, Pool, Runtime}; +use serde::{de::DeserializeOwned, Serialize}; +use thiserror::Error; + +use crate::{ + config::{CacheConfig, CacheTtlConfig}, + error::AppError, + metrics, +}; + +/// Keys are chunked so a single Lua invocation never touches an unbounded +/// number of hash keys in one round trip. +const BULK_CHUNK_SIZE: usize = 500; + +const BEGIN_SCRIPT_SRC: &str = r#" +local ttl_ms = ARGV[1] +for i, key in ipairs(KEYS) do + redis.call('HINCRBY', key, 'v', 1) + redis.call('HINCRBY', key, 'dirty', 1) + redis.call('HDEL', key, 'p') + redis.call('PEXPIRE', key, ttl_ms) +end +return 1 +"#; + +// Re-applies `PEXPIRE` for the same reason `begin` sets it: `HINCRBY` +// recreates a key that has already expired, and a recreated barrier entry +// with no TTL would never be reclaimed. An `end` that lands after its own +// barrier expired (a long mutation, or a bulk invalidation chunking through +// many keys) would otherwise leak an immortal hash per key. +// +// `dirty` is a nesting counter, not a flag: `begin` increments it, `end` +// decrements it, and only 0 means "no mutation is still in flight" — see the +// module docs for why two overlapping mutations on the same key need this. +// The clamp back to 0 below guards against `dirty` drifting negative if `end` +// is ever called without a matching `begin` (should not happen, but a +// negative counter would otherwise require one *extra* `begin` to dig back +// out of before the barrier could ever be seen as dirty again). +// +// `HDEL p` is defensive rather than load-bearing: `begin` already cleared the +// payload, and `try_populate` refuses to write while dirty. It costs one call +// and guarantees that whatever happened to the key in between — including a +// reader repopulating it against a barrier that was destroyed out from under +// this mutation — the entry is left without a payload for the next reader to +// reload cleanly, which is what `end`'s contract has always claimed. +const END_SCRIPT_SRC: &str = r#" +local ttl_ms = ARGV[1] +for i, key in ipairs(KEYS) do + redis.call('HINCRBY', key, 'v', 1) + local remaining = redis.call('HINCRBY', key, 'dirty', -1) + if remaining < 0 then + redis.call('HSET', key, 'dirty', 0) + end + redis.call('HDEL', key, 'p') + redis.call('PEXPIRE', key, ttl_ms) +end +return 1 +"#; + +const TRY_POPULATE_SCRIPT_SRC: &str = r#" +local v = redis.call('HGET', KEYS[1], 'v') +if v == false then v = '0' end +local dirty = tonumber(redis.call('HGET', KEYS[1], 'dirty')) or 0 +if dirty > 0 or v ~= ARGV[1] then + return 'stale' +end +redis.call('HSET', KEYS[1], 'p', ARGV[2]) +redis.call('PEXPIRE', KEYS[1], ARGV[3]) +return 'applied' +"#; + +// Discards a corrupt payload without touching the barrier fields. Guarded by +// exactly the same version/dirty check as `try_populate`, and for the same +// reason: an unconditional `DEL` of the whole hash would take `v` and `dirty` +// with it, destroying an in-flight mutation's barrier. The next reader would +// then see an absent key, observe version 0, load pre-commit state, and +// populate it successfully — the barrier's whole purpose, defeated by a +// cleanup path. +const DISCARD_SCRIPT_SRC: &str = r#" +local v = redis.call('HGET', KEYS[1], 'v') +if v == false then v = '0' end +local dirty = tonumber(redis.call('HGET', KEYS[1], 'dirty')) or 0 +if dirty > 0 or v ~= ARGV[1] then + return 'skipped' +end +redis.call('HDEL', KEYS[1], 'p') +return 'discarded' +"#; + +/// Fixed, low-cardinality label for cache metrics and log lines. Never an ID, +/// action name, or arbitrary string — see `src/metrics.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CacheCategory { + Session, + EntityStatus, + TenantStatus, + Credential, + CredentialCeiling, + Grants, +} + +impl CacheCategory { + pub fn as_str(self) -> &'static str { + match self { + Self::Session => "session", + Self::EntityStatus => "entity_status", + Self::TenantStatus => "tenant_status", + Self::Credential => "credential", + Self::CredentialCeiling => "credential_ceiling", + Self::Grants => "grants", + } + } + + /// The configured entry TTL for this category — resolved here, once, so + /// callers never need to thread a `Duration` through read/write-path call + /// sites themselves. + fn ttl(self, cfg: &CacheTtlConfig) -> Duration { + let secs = match self { + Self::Session => cfg.session_secs, + Self::EntityStatus => cfg.entity_status_secs, + Self::TenantStatus => cfg.tenant_status_secs, + Self::Credential => cfg.credential_secs, + Self::CredentialCeiling => cfg.credential_ceiling_secs, + Self::Grants => cfg.grants_secs, + }; + Duration::from_secs(secs) + } +} + +/// The outcome of a cache read: a valid, non-dirty entry, or a miss carrying +/// the version a subsequent `try_populate` must present unchanged to write +/// successfully. `Unavailable` means the read itself failed (timeout, +/// connection error) — callers must fall through to Postgres and must not +/// attempt to populate afterward (the write would just fail too). +#[derive(Debug)] +pub enum Lookup { + Hit(T), + Miss { version: i64 }, + Unavailable, +} + +/// One key's unparsed entry from [`CacheClient::lookup_many`]. Opaque by +/// design — [`CacheClient::decode`] is the only way to read it, so the +/// dirty-bit and version handling can't be reimplemented per call site. +#[derive(Debug)] +pub struct RawLookup { + /// `None` when the read itself failed, which `decode` maps to + /// `Lookup::Unavailable`. + version: Option, + /// `None` for an absent *or* dirty entry — both are misses, and a dirty + /// entry's payload must never be served. + payload: Option>, +} + +impl Default for RawLookup { + /// An unavailable entry — the safe reading of a result that never arrived. + fn default() -> Self { + Self::unavailable() + } +} + +impl RawLookup { + fn unavailable() -> Self { + Self { + version: None, + payload: None, + } + } + + fn from_fields(fields: (Option, Option, Option>)) -> Self { + let (version, dirty, payload) = fields; + // `dirty` is a nesting counter (see `BEGIN_SCRIPT_SRC`/`END_SCRIPT_SRC`): + // any value above zero means at least one overlapping mutation on this + // key is still in flight. + let is_dirty = dirty.unwrap_or(0) > 0; + Self { + version: Some(version.unwrap_or(0)), + payload: payload.filter(|_| !is_dirty), + } + } +} + +#[derive(Debug, Error)] +pub enum CacheError { + #[error("cache operation timed out")] + Timeout, + #[error("cache pool error: {0}")] + Pool(#[from] deadpool_redis::PoolError), + #[error("cache redis error: {0}")] + Redis(#[from] redis::RedisError), +} + +#[derive(Debug)] +pub struct CacheClient { + pool: Pool, + op_timeout: Duration, + ttl: CacheTtlConfig, + begin_script: redis::Script, + end_script: redis::Script, + try_populate_script: redis::Script, + discard_script: redis::Script, +} + +impl CacheClient { + /// Builds the client and its connection pool *without* contacting Redis. + /// + /// An error here is a configuration error — an unparseable URL, an invalid + /// pool size — never a transient outage, so callers should treat it as + /// fatal. Reachability is a separate, retryable concern: see + /// [`Self::probe`] and `main::init_cache`. + pub fn build(cfg: &CacheConfig) -> anyhow::Result { + let pool_cfg = PoolConfig::from_url(&cfg.redis_url); + let mut pool_builder = pool_cfg + .builder() + .map_err(|e| anyhow::anyhow!("invalid ATOM_CACHE_REDIS_URL: {e}"))?; + pool_builder = pool_builder.max_size(cfg.pool_max_size as usize); + let pool = pool_builder + .runtime(Runtime::Tokio1) + .build() + .map_err(|e| anyhow::anyhow!("failed to build cache pool: {e}"))?; + + Ok(Self { + pool, + op_timeout: Duration::from_millis(cfg.op_timeout_ms), + ttl: cfg.ttl, + begin_script: redis::Script::new(BEGIN_SCRIPT_SRC), + end_script: redis::Script::new(END_SCRIPT_SRC), + try_populate_script: redis::Script::new(TRY_POPULATE_SCRIPT_SRC), + discard_script: redis::Script::new(DISCARD_SCRIPT_SRC), + }) + } + + /// One-shot reachability check with a single `PING`, bounded by + /// `connect_timeout_ms`. + pub async fn probe(&self, connect_timeout_ms: u64) -> anyhow::Result<()> { + tokio::time::timeout(Duration::from_millis(connect_timeout_ms), self.ping()) + .await + .map_err(|_| anyhow::anyhow!("cache connect timed out"))? + .map_err(|e| anyhow::anyhow!("cache connect failed: {e}")) + } + + /// [`build`](Self::build) plus a [`probe`](Self::probe) — fails unless + /// Redis is reachable right now. + pub async fn connect(cfg: &CacheConfig) -> anyhow::Result { + let client = Self::build(cfg)?; + client.probe(cfg.connect_timeout_ms).await?; + Ok(client) + } + + pub async fn ping(&self) -> Result<(), CacheError> { + let mut conn = self.get_conn().await?; + redis::cmd("PING") + .query_async::(&mut conn) + .await + .map_err(CacheError::from)?; + Ok(()) + } + + async fn get_conn(&self) -> Result { + tokio::time::timeout(self.op_timeout, self.pool.get()) + .await + .map_err(|_| CacheError::Timeout)? + .map_err(CacheError::from) + } + + /// Single-key read. Never errors outward — a failure of any kind becomes + /// `Lookup::Unavailable`. + pub async fn lookup( + &self, + category: CacheCategory, + key: &str, + ) -> Lookup { + let raw = self + .lookup_many(std::slice::from_ref(&key)) + .await + .pop() + .unwrap_or_else(RawLookup::unavailable); + self.decode(category, key, raw).await + } + + /// Reads every key in one pipelined round trip on a single pooled + /// connection, returning one [`RawLookup`] per key, in order. + /// + /// The auth hot path reads up to three keys before any request work + /// starts; issued one at a time that is three pool acquisitions and three + /// serial round trips, each bounded by `op_timeout`. Use this wherever the + /// keys have no data dependency on each other, then [`Self::decode`] each + /// result into its own type. Metrics are recorded by `decode`, not here, + /// so one batch can span several categories. + /// + /// Always returns exactly `keys.len()` entries; a transport failure yields + /// unavailable entries rather than a short vector. + pub async fn lookup_many(&self, keys: &[&str]) -> Vec { + if keys.is_empty() { + return Vec::new(); + } + let unavailable = || keys.iter().map(|_| RawLookup::unavailable()).collect(); + + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!(error = %err, "cache lookup unavailable"); + return unavailable(); + } + }; + + let mut pipe = redis::pipe(); + for key in keys { + pipe.cmd("HMGET").arg(*key).arg("v").arg("dirty").arg("p"); + } + let result = tokio::time::timeout( + self.op_timeout, + pipe.query_async::, Option, Option>)>>(&mut conn), + ) + .await; + + match result { + Ok(Ok(rows)) if rows.len() == keys.len() => { + rows.into_iter().map(RawLookup::from_fields).collect() + } + Ok(Ok(rows)) => { + tracing::warn!( + expected = keys.len(), + got = rows.len(), + "cache pipelined lookup returned an unexpected row count" + ); + unavailable() + } + Ok(Err(err)) => { + tracing::warn!(error = %err, "cache lookup failed"); + unavailable() + } + Err(_) => { + tracing::warn!("cache lookup timed out"); + unavailable() + } + } + } + + /// Parses one [`lookup_many`](Self::lookup_many) result into a typed + /// [`Lookup`], recording the read in the per-category metrics. A corrupt + /// payload is deleted and reported as a miss, exactly as in `lookup`. + pub async fn decode( + &self, + category: CacheCategory, + key: &str, + raw: RawLookup, + ) -> Lookup { + let Some(version) = raw.version else { + metrics::record_cache_lookup(category.as_str(), "error"); + return Lookup::Unavailable; + }; + let Some(payload) = raw.payload else { + metrics::record_cache_lookup(category.as_str(), "miss"); + return Lookup::Miss { version }; + }; + match rmp_serde::from_slice::(&payload) { + Ok(value) => { + metrics::record_cache_lookup(category.as_str(), "hit"); + Lookup::Hit(value) + } + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache payload corrupt; discarding"); + self.discard_payload(key, version).await; + metrics::record_cache_lookup(category.as_str(), "miss"); + Lookup::Miss { version } + } + } + } + + /// Clears a corrupt payload field, leaving the barrier (`v`/`dirty`) + /// intact, and only while the entry is still at `observed_version` and not + /// dirty. Best-effort: a failure just leaves the corrupt payload for the + /// next reader to trip over and re-attempt. + async fn discard_payload(&self, key: &str, observed_version: i64) { + let Ok(mut conn) = self.get_conn().await else { + return; + }; + let _ = tokio::time::timeout( + self.op_timeout, + self.discard_script + .key(key) + .arg(observed_version) + .invoke_async::(&mut conn), + ) + .await; + } + + /// Best-effort conditional write following a cache-miss load. Discarded + /// silently if the entry became dirty or its version moved on since the + /// caller observed `expected_version` — see the module docs. + pub async fn try_populate( + &self, + category: CacheCategory, + key: &str, + expected_version: i64, + value: &T, + ) { + let Ok(payload) = rmp_serde::to_vec(value) else { + tracing::warn!( + category = category.as_str(), + "cache payload serialize failed" + ); + metrics::record_cache_populate(category.as_str(), "error"); + return; + }; + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(_) => { + metrics::record_cache_populate(category.as_str(), "error"); + return; + } + }; + let ttl = category.ttl(&self.ttl); + let outcome = tokio::time::timeout( + self.op_timeout, + self.try_populate_script + .key(key) + .arg(expected_version) + .arg(payload) + .arg(ttl.as_millis() as i64) + .invoke_async::(&mut conn), + ) + .await; + // The script distinguishes `applied` from `stale` (barrier dirty, or + // the version moved since the caller's `lookup`). Both are normal, but + // only the split tells an operator whether a zero hit rate means "cold" + // or "every write is being rejected" — so it is recorded, not dropped. + match outcome { + Ok(Ok(result)) => { + let outcome = if result == "applied" { + "applied" + } else { + "stale" + }; + metrics::record_cache_populate(category.as_str(), outcome); + } + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache populate failed"); + metrics::record_cache_populate(category.as_str(), "error"); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache populate timed out"); + metrics::record_cache_populate(category.as_str(), "error"); + } + } + } + + /// Cache-aside read with a fallback loader: a hit returns immediately, a + /// miss loads via `loader` and best-effort populates the cache, and an + /// unavailable cache falls straight through to `loader`. + pub async fn get_or_load( + &self, + category: CacheCategory, + key: &str, + loader: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: Future>, + { + match self.lookup::(category, key).await { + Lookup::Hit(value) => Ok(value), + Lookup::Miss { version } => { + let value = loader().await?; + self.try_populate(category, key, version, &value).await; + Ok(value) + } + Lookup::Unavailable => loader().await, + } + } + + /// Increments `keys`' dirty counter before a security-sensitive Postgres + /// mutation — safe to call while another mutation on the same key is + /// already in flight, since the counter (not a flag) is what lets `end` + /// tell "this mutation is done" apart from "every overlapping mutation on + /// this key is done" (see module docs). **Fails the caller** if the + /// barrier cannot be established (Redis unreachable/timeout) — see module + /// docs and `src/cache/invalidate.rs`. A no-op that always succeeds when + /// `keys` is empty. + pub async fn begin(&self, category: CacheCategory, keys: &[String]) -> Result<(), AppError> { + if keys.is_empty() { + return Ok(()); + } + let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); + // One connection for every chunk, not one per chunk: a bulk + // invalidation re-acquiring from the pool per 500 keys competes with + // the request path for connections while already holding Postgres row + // locks. + let mut conn = self.get_conn().await.map_err(|err| { + tracing::warn!(category = category.as_str(), error = %err, "cache begin: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + AppError::service_unavailable("cache unavailable; refusing security-sensitive mutation") + })?; + for chunk in keys.chunks(BULK_CHUNK_SIZE) { + let mut invocation = self.begin_script.prepare_invoke(); + for key in chunk { + invocation.key(key); + } + invocation.arg(barrier_ttl.as_millis() as i64); + + let outcome = + tokio::time::timeout(self.op_timeout, invocation.invoke_async::(&mut conn)) + .await; + match outcome { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache begin failed"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return Err(AppError::service_unavailable( + "cache unavailable; refusing security-sensitive mutation", + )); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache begin timed out"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return Err(AppError::service_unavailable( + "cache unavailable; refusing security-sensitive mutation", + )); + } + } + } + metrics::record_cache_invalidation(category.as_str(), "ok"); + Ok(()) + } + + /// Bumps the version and decrements the dirty counter on `keys` after the + /// mutation (success or failure) — the entry only reads as clean once + /// every overlapping `begin` on the key has been matched by an `end`. + /// Always best-effort — never fails the caller. Left dirty entries + /// self-heal once the barrier TTL set by `begin` expires. The version + /// bump (not just the dirty decrement) is what stops a reader whose + /// `lookup` landed during the dirty window from repopulating a stale + /// value afterward — see the module docs. + pub async fn end(&self, category: CacheCategory, keys: &[String]) { + if keys.is_empty() { + return; + } + let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); + // One connection for every chunk — see `begin`. + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache end: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return; + } + }; + for chunk in keys.chunks(BULK_CHUNK_SIZE) { + let mut invocation = self.end_script.prepare_invoke(); + for key in chunk { + invocation.key(key); + } + invocation.arg(barrier_ttl.as_millis() as i64); + let outcome = + tokio::time::timeout(self.op_timeout, invocation.invoke_async::(&mut conn)) + .await; + match outcome { + Ok(Ok(_)) => metrics::record_cache_invalidation(category.as_str(), "ok"), + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache end failed"); + metrics::record_cache_invalidation(category.as_str(), "error"); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache end timed out"); + metrics::record_cache_invalidation(category.as_str(), "error"); + } + } + } + } +} + +/// The barrier key's own expiry: long enough to comfortably outlast any +/// realistic Postgres mutation + `end` call, so a lost `end` self-heals by +/// the whole entry expiring outright rather than staying dirty forever. +/// +/// Saturating rather than `*`: `Duration`'s multiplication panics on +/// overflow, and this runs inside `begin`/`end` — on the mutation path, long +/// after a nonsensically large `ATOM_CACHE_TTL_*` would have been accepted at +/// startup. `cache_from_env` bounds those values, so this is belt-and-braces. +fn barrier_ttl(entry_ttl: Duration) -> Duration { + entry_ttl.saturating_mul(5) +} + +/// `get_or_load`, but tolerant of caching being disabled entirely — the +/// common entry point for read paths, so call sites don't need to branch on +/// `Option<&CacheClient>` themselves. +pub async fn cached_or_load( + cache: Option<&CacheClient>, + category: CacheCategory, + key: &str, + loader: F, +) -> Result +where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: Future>, +{ + match cache { + Some(cache) => cache.get_or_load(category, key, loader).await, + None => loader().await, + } +} + +/// Redis-gated unit tests for the cache mechanism itself — key formatting, +/// serialization round trips, and the barrier primitives in isolation from +/// any AuthN/AuthZ call site. Requires `ATOM_TEST_REDIS_URL`; run with +/// `ATOM_TEST_REDIS_URL=redis://... cargo test -- --ignored`. +#[cfg(test)] +mod tests { + use super::*; + use serde::{Deserialize, Serialize}; + use uuid::Uuid; + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + struct Payload { + value: String, + } + + async fn test_client() -> CacheClient { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + let cfg = CacheConfig { + enabled: true, + redis_url: url, + ..CacheConfig::default() + }; + CacheClient::connect(&cfg) + .await + .expect("connect to test redis") + } + + fn unique_key(label: &str) -> String { + format!("atom:v1:test:{label}:{}", Uuid::new_v4()) + } + + /// Drives `main.rs`'s fail-fast-vs-degrade startup branching: `connect` + /// must return `Err` (not hang, not panic) against an unreachable Redis, + /// bounded by `connect_timeout_ms`. No live Redis needed — this + /// deliberately never reaches one — so it runs in default `cargo test`. + #[tokio::test] + async fn connect_fails_against_an_unreachable_redis() { + let cfg = CacheConfig { + enabled: true, + redis_url: "redis://127.0.0.1:1/0".into(), + connect_timeout_ms: 200, + op_timeout_ms: 50, + ..CacheConfig::default() + }; + let result = CacheClient::connect(&cfg).await; + assert!( + result.is_err(), + "connect must fail against an unreachable redis, not hang or succeed" + ); + } + + #[tokio::test] + #[ignore] + async fn ping_succeeds_against_reachable_redis() { + let client = test_client().await; + client.ping().await.expect("ping"); + } + + #[tokio::test] + #[ignore] + async fn lookup_on_absent_key_is_a_clean_miss_at_version_zero() { + let client = test_client().await; + let key = unique_key("lookup-miss"); + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => assert_eq!(version, 0), + other => panic!("expected a clean miss, got {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn try_populate_then_lookup_round_trips_the_payload() { + let client = test_client().await; + let key = unique_key("roundtrip"); + let value = Payload { + value: "hello".into(), + }; + + let version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected miss before populate, got {other:?}"), + }; + client + .try_populate(CacheCategory::Grants, &key, version, &value) + .await; + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_eq!(got, value), + other => panic!("expected a hit after populate, got {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn corrupt_payload_is_discarded_without_destroying_the_barrier() { + let client = test_client().await; + let key = unique_key("corrupt"); + + // Write a payload that isn't a validly-encoded `Payload` directly + // into the hash, bypassing `try_populate`, to simulate corruption. + let mut conn = client.get_conn().await.expect("conn"); + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("v") + .arg(1) + .arg("p") + .arg("not a valid payload") + .query_async(&mut conn) + .await + .expect("seed corrupt payload"); + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + other => panic!("expected corrupt payload to be a miss, got {other:?}"), + } + + // The corrupt payload must be gone so the next lookup doesn't repeat + // the same deserialize failure — but *only* the payload. Deleting the + // whole hash would take `v`/`dirty` with it, and a concurrent + // mutation's barrier along with them. + let (version, payload): (Option, Option>) = redis::cmd("HMGET") + .arg(&key) + .arg("v") + .arg("p") + .query_async(&mut conn) + .await + .expect("read back barrier fields"); + assert!( + payload.is_none(), + "corrupt payload should have been cleared" + ); + assert_eq!( + version, + Some(1), + "the version must survive: it is what a concurrent mutation's barrier rests on" + ); + } + + /// The reason [`DISCARD_SCRIPT_SRC`] is version-guarded: a corrupt-payload + /// cleanup racing a mutation must not clear the barrier that mutation just + /// established, or a reader could repopulate pre-commit state over it. + #[tokio::test] + #[ignore] + async fn corrupt_payload_cleanup_leaves_a_concurrent_barrier_intact() { + let client = test_client().await; + let key = unique_key("corrupt-vs-barrier"); + let keys = vec![key.clone()]; + + let mut conn = client.get_conn().await.expect("conn"); + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("v") + .arg(1) + .arg("p") + .arg("not a valid payload") + .query_async(&mut conn) + .await + .expect("seed corrupt payload"); + + // A mutation opens its barrier *after* the reader observed version 1. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + // The reader's cleanup now fires against its stale observed version. + client.discard_payload(&key, 1).await; + + let dirty: Option = redis::cmd("HGET") + .arg(&key) + .arg("dirty") + .query_async(&mut conn) + .await + .expect("read dirty"); + assert_eq!( + dirty.as_deref(), + Some("1"), + "the in-flight mutation's barrier must survive a late corrupt-payload cleanup" + ); + } + + #[tokio::test] + #[ignore] + async fn try_populate_rejects_a_stale_version() { + let client = test_client().await; + let key = unique_key("stale-version"); + let keys = vec![key.clone()]; + + // A concurrent mutation bumps the version between the reader's + // initial lookup and its populate attempt. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + client.end(CacheCategory::Grants, &keys).await; + + // The reader observed version 0 (before the mutation) and now tries + // to populate with that stale value. + let stale_value = Payload { + value: "stale".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, 0, &stale_value) + .await; + + // Must still be a miss — the stale write was discarded. + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + Lookup::Hit(got) => panic!("stale write should have been rejected, got {got:?}"), + other => panic!("unexpected lookup outcome: {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn dirty_entry_is_never_served_as_a_hit() { + let client = test_client().await; + let key = unique_key("dirty"); + let keys = vec![key.clone()]; + + let version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected miss, got {other:?}"), + }; + let value = Payload { + value: "before-mutation".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, version, &value) + .await; + // Confirm it's actually cached before dirtying it. + assert!(matches!( + client.lookup::(CacheCategory::Grants, &key).await, + Lookup::Hit(_) + )); + + // `begin` marks it dirty and clears the payload without an `end`. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + Lookup::Hit(got) => panic!("dirty entry served as a hit: {got:?}"), + other => panic!("unexpected outcome: {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn bulk_begin_and_end_cover_every_key_in_one_round_trip() { + let client = test_client().await; + let keys: Vec = (0..5).map(|i| unique_key(&format!("bulk-{i}"))).collect(); + + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + for key in &keys { + let (_, dirty, _): (Option, Option, Option>) = redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut client.get_conn().await.expect("conn")) + .await + .expect("hmget"); + assert_eq!( + dirty.as_deref(), + Some("1"), + "key {key} should be dirty after begin" + ); + } + + client.end(CacheCategory::Grants, &keys).await; + for key in &keys { + let (_, dirty, _): (Option, Option, Option>) = redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut client.get_conn().await.expect("conn")) + .await + .expect("hmget"); + assert_eq!( + dirty.as_deref(), + Some("0"), + "key {key} should be clean after end" + ); + } + } + + #[tokio::test] + #[ignore] + async fn get_or_load_hits_cache_on_second_call_without_invoking_loader() { + let client = test_client().await; + let key = unique_key("get-or-load"); + let calls = std::sync::atomic::AtomicUsize::new(0); + + let first: Payload = client + .get_or_load(CacheCategory::Grants, &key, || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async { + Ok(Payload { + value: "loaded".into(), + }) + } + }) + .await + .expect("first load"); + assert_eq!(first.value, "loaded"); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + + let second: Payload = client + .get_or_load(CacheCategory::Grants, &key, || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async { + Ok(Payload { + value: "should-not-be-called".into(), + }) + } + }) + .await + .expect("second load"); + assert_eq!( + second.value, "loaded", + "second call must be served from cache" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "loader must not run again on a cache hit" + ); + } + + /// The test that actually validates the consistency model, not just its + /// individual primitives: a reader that captured a version *before* a + /// concurrent mutation must not be able to repopulate the cache with its + /// now-stale result *after* the mutation commits and clears the barrier. + /// This is the exact race described in `src/cache/mod.rs`'s module docs. + #[tokio::test] + #[ignore] + async fn stale_reader_cannot_repopulate_after_a_concurrent_mutation_completes() { + let client = test_client().await; + let key = unique_key("race"); + let keys = vec![key.clone()]; + + // Reader observes the version before any mutation has happened. + let observed_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected initial miss, got {other:?}"), + }; + + // A concurrent mutation runs to completion while the reader's + // (simulated) Postgres load is still in flight: begin bumps the + // version and marks dirty, then end clears dirty after the mutation + // commits. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + client.end(CacheCategory::Grants, &keys).await; + + // The reader's stale load now finishes and attempts to populate with + // the version it observed *before* the mutation. + let stale_value = Payload { + value: "STALE — must never be visible".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, observed_version, &stale_value) + .await; + + // The cache must not have been poisoned with the stale value — the + // next reader must see a clean miss (correctness bound: even though + // the mutation revealed no new payload of its own, the stale write + // must never have applied), never the disallowed value. + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "stale reader was able to poison the cache after a concurrent mutation" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + } + + /// Regression test for a review finding: the test above only covers a + /// reader whose `lookup` landed *before* `begin` ran (a version `begin` + /// then bumps past). It does not cover a reader whose `lookup` lands + /// *during* the dirty window itself (after `begin`, before `end`) — that + /// reader observes the *post-`begin`* version, and if `end` only cleared + /// `dirty` without bumping the version again, that same version would + /// still match after `end`, so `try_populate` would wrongly accept a + /// value the reader may have loaded from Postgres before the mutation's + /// own write committed. This is the actual scenario the review + /// described: "a reader that starts during a mutation" — not one that + /// started before it. + #[tokio::test] + #[ignore] + async fn stale_reader_cannot_repopulate_during_the_mutations_dirty_window() { + let client = test_client().await; + let key = unique_key("dirty-window-race"); + let keys = vec![key.clone()]; + + // The mutation begins — the key is now dirty. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + // A reader's `lookup` lands *while the mutation is still in + // flight* — this is the case the previous test didn't cover. + let dirty_window_version = match client.lookup::(CacheCategory::Grants, &key).await + { + Lookup::Miss { version } => version, + other => panic!("expected a miss while dirty, got {other:?}"), + }; + + // The mutation finishes — `end` clears dirty (and, with the fix, + // bumps the version again). + client.end(CacheCategory::Grants, &keys).await; + + // The reader's (simulated) Postgres load — possibly stale, since it + // may have run before the mutation's own write committed — finishes + // and attempts to populate using the version it observed *during* + // the dirty window. + let stale_value = Payload { + value: "STALE — read during the dirty window, must never be visible".into(), + }; + client + .try_populate( + CacheCategory::Grants, + &key, + dirty_window_version, + &stale_value, + ) + .await; + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "a reader that started during the mutation's dirty window was able to poison \ + the cache once the mutation completed — this is exactly the P1 the barrier \ + model exists to prevent, and means `end` isn't invalidating a dirty-window \ + read's captured version" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + } + + /// Regression test for a review finding: the two tests above only cover a + /// single mutation on a key. When *two* security-sensitive mutations + /// overlap on the same key — e.g. two role changes affecting the same + /// subject's `grants` entry — a `dirty` field that is a flag rather than + /// a nesting counter is unconditionally cleared by whichever mutation's + /// `end` runs first, even though the second mutation is still in flight. + /// A reader landing in that gap sees a clean, non-dirty entry and can + /// `try_populate` a payload loaded before the second mutation's own + /// commit — wrong the instant that commit lands, and never corrected if + /// the second mutation's own `end` is ever delayed or lost. + #[tokio::test] + #[ignore] + async fn dirty_barrier_stays_up_until_every_overlapping_mutation_ends() { + let client = test_client().await; + let key = unique_key("overlapping-mutations"); + let keys = vec![key.clone()]; + + // Two mutations both touch this key concurrently — M1 begins first, + // then M2 begins while M1 is still in flight. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("M1 begin"); + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("M2 begin"); + + // M1 finishes first. With a flag (not a counter), this would clear + // `dirty` outright even though M2 hasn't committed yet. + client.end(CacheCategory::Grants, &keys).await; + + // A reader lands in the gap between M1's `end` and M2's commit. The + // barrier must still read as dirty — M2 is still in flight — so this + // must be a miss, not a hit, and the version it observes here must + // never successfully populate the cache. + let gap_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!( + "expected a miss while M2 is still in flight (M1's end must not have cleared \ + the barrier), got {other:?}" + ), + }; + let stale_value = Payload { + value: "STALE — read while a second overlapping mutation was still in flight".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, gap_version, &stale_value) + .await; + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "a reader landing between M1's `end` and M2's commit was able to poison the \ + cache — this means `dirty` is being treated as a flag instead of a nesting \ + counter, so the first of two overlapping mutations to finish clears the \ + barrier the second one still needs" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + + // M2 finishes. Only now should the barrier be fully clear. + client.end(CacheCategory::Grants, &keys).await; + let clean_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => { + panic!("expected a clean miss once every overlapping mutation ended, got {other:?}") + } + }; + let fresh_value = Payload { + value: "fresh, post-M2 value".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, clean_version, &fresh_value) + .await; + assert!( + matches!( + client.lookup::(CacheCategory::Grants, &key).await, + Lookup::Hit(got) if got == fresh_value + ), + "once every overlapping mutation has called `end`, the barrier must clear and a \ + fresh populate must succeed" + ); + } +} diff --git a/src/config.rs b/src/config.rs index e4a46ac..89f5a18 100644 --- a/src/config.rs +++ b/src/config.rs @@ -75,6 +75,7 @@ pub struct Config { pub certs_root_ca_key_path: Option, pub certs_leaf_default_ttl_secs: u64, pub certs_leaf_max_ttl_secs: u64, + pub cache: CacheConfig, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -100,6 +101,69 @@ impl Default for DbPoolConfig { } } +/// Upper bound on any single `ATOM_CACHE_TTL_*` value, enforced by +/// [`cache_from_env`]. +const MAX_CACHE_TTL_SECS: u64 = 24 * 60 * 60; + +/// Per-category TTLs, applied to cached entries as a defense-in-depth safety +/// net (not the primary invalidation mechanism — see `src/cache/mod.rs`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheTtlConfig { + pub session_secs: u64, + pub entity_status_secs: u64, + pub tenant_status_secs: u64, + pub credential_secs: u64, + pub credential_ceiling_secs: u64, + pub grants_secs: u64, +} + +impl Default for CacheTtlConfig { + fn default() -> Self { + Self { + session_secs: 60, + entity_status_secs: 60, + tenant_status_secs: 60, + credential_secs: 60, + credential_ceiling_secs: 60, + grants_secs: 60, + } + } +} + +/// Redis-backed cache for AuthN/AuthZ decision inputs. Off by default — this +/// is a pure performance optimization; every check works correctly with it +/// disabled, since Postgres remains authoritative. See `src/cache/mod.rs` for +/// the consistency model this configures. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheConfig { + pub enabled: bool, + pub redis_url: String, + pub pool_max_size: u32, + pub connect_timeout_ms: u64, + pub op_timeout_ms: u64, + /// When `enabled` and Redis is unreachable at startup: abort like an + /// unreachable Postgres would (`true`), or log and continue with caching + /// disabled (`false`, the recommended default — Redis is not a + /// correctness dependency for reads, only, while enabled, a write-path + /// dependency for security-sensitive mutations; see `src/cache/mod.rs`). + pub fail_fast_on_startup: bool, + pub ttl: CacheTtlConfig, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + enabled: false, + redis_url: String::new(), + pool_max_size: 20, + connect_timeout_ms: 2_000, + op_timeout_ms: 50, + fail_fast_on_startup: false, + ttl: CacheTtlConfig::default(), + } + } +} + #[derive(Clone, PartialEq, Eq)] pub struct SecretBytes(Vec); @@ -540,6 +604,7 @@ impl Config { certs_root_ca_key_path: std::env::var("ATOM_CERTS_ROOT_CA_KEY_PATH").ok(), certs_leaf_default_ttl_secs: env_u64("ATOM_CERTS_LEAF_DEFAULT_TTL_SECS", 2_592_000), certs_leaf_max_ttl_secs: env_u64("ATOM_CERTS_LEAF_MAX_TTL_SECS", 2_592_000), + cache: cache_from_env()?, public_base_url, }) } @@ -606,6 +671,7 @@ impl Config { certs_root_ca_key_path: None, certs_leaf_default_ttl_secs: 2_592_000, certs_leaf_max_ttl_secs: 2_592_000, + cache: CacheConfig::default(), } } } @@ -715,6 +781,78 @@ fn db_pool_from_env() -> Result { Ok(cfg) } +fn cache_from_env() -> Result { + let default = CacheConfig::default(); + let default_ttl = CacheTtlConfig::default(); + let cfg = CacheConfig { + enabled: env_bool_default("ATOM_CACHE_ENABLED", default.enabled), + redis_url: std::env::var("ATOM_CACHE_REDIS_URL").unwrap_or_default(), + pool_max_size: env_parse("ATOM_CACHE_POOL_MAX_SIZE", default.pool_max_size)?, + connect_timeout_ms: env_parse("ATOM_CACHE_CONNECT_TIMEOUT_MS", default.connect_timeout_ms)?, + op_timeout_ms: env_parse("ATOM_CACHE_OP_TIMEOUT_MS", default.op_timeout_ms)?, + fail_fast_on_startup: env_bool_default( + "ATOM_CACHE_FAIL_FAST_ON_STARTUP", + default.fail_fast_on_startup, + ), + ttl: CacheTtlConfig { + session_secs: env_parse("ATOM_CACHE_TTL_SESSION_SECS", default_ttl.session_secs)?, + entity_status_secs: env_parse( + "ATOM_CACHE_TTL_ENTITY_STATUS_SECS", + default_ttl.entity_status_secs, + )?, + tenant_status_secs: env_parse( + "ATOM_CACHE_TTL_TENANT_STATUS_SECS", + default_ttl.tenant_status_secs, + )?, + credential_secs: env_parse( + "ATOM_CACHE_TTL_CREDENTIAL_SECS", + default_ttl.credential_secs, + )?, + credential_ceiling_secs: env_parse( + "ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS", + default_ttl.credential_ceiling_secs, + )?, + grants_secs: env_parse("ATOM_CACHE_TTL_GRANTS_SECS", default_ttl.grants_secs)?, + }, + }; + if cfg.enabled { + if cfg.redis_url.trim().is_empty() { + anyhow::bail!("ATOM_CACHE_REDIS_URL must be set when ATOM_CACHE_ENABLED=true"); + } + if cfg.pool_max_size == 0 { + anyhow::bail!("ATOM_CACHE_POOL_MAX_SIZE must be greater than zero"); + } + if cfg.connect_timeout_ms == 0 { + anyhow::bail!("ATOM_CACHE_CONNECT_TIMEOUT_MS must be greater than zero"); + } + if cfg.op_timeout_ms == 0 { + anyhow::bail!("ATOM_CACHE_OP_TIMEOUT_MS must be greater than zero"); + } + let ttl = &cfg.ttl; + let ttls = [ + ttl.session_secs, + ttl.entity_status_secs, + ttl.tenant_status_secs, + ttl.credential_secs, + ttl.credential_ceiling_secs, + ttl.grants_secs, + ]; + if ttls.contains(&0) { + anyhow::bail!("ATOM_CACHE_TTL_* values must all be greater than zero"); + } + // Bounded at startup rather than left to fail on the mutation path: + // `cache::barrier_ttl` scales these by 5, and the resulting `Duration` + // has to stay representable. A day is already far beyond any sane + // staleness window for auth/authz state. + if ttls.iter().any(|secs| *secs > MAX_CACHE_TTL_SECS) { + anyhow::bail!( + "ATOM_CACHE_TTL_* values must not exceed {MAX_CACHE_TTL_SECS} seconds (24h)" + ); + } + } + Ok(cfg) +} + fn signing_keys_from_env() -> Result { let default = SigningKeyConfig::default(); Ok(SigningKeyConfig { @@ -1173,6 +1311,76 @@ mod tests { clear_hardening_env(); } + #[test] + fn cache_is_disabled_by_default() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + + let cfg = Config::from_env().expect("config"); + assert!(!cfg.cache.enabled, "caching must default off"); + assert!(!cfg.cache.fail_fast_on_startup); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_without_redis_url_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + + let err = Config::from_env().expect_err("cache enabled without a redis url"); + assert!(err.to_string().contains("ATOM_CACHE_REDIS_URL")); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_with_zero_ttl_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + std::env::set_var("ATOM_CACHE_REDIS_URL", "redis://localhost:6379/0"); + std::env::set_var("ATOM_CACHE_TTL_GRANTS_SECS", "0"); + + let err = Config::from_env().expect_err("zero grants ttl"); + assert!(err.to_string().contains("ATOM_CACHE_TTL")); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_with_zero_pool_size_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + std::env::set_var("ATOM_CACHE_REDIS_URL", "redis://localhost:6379/0"); + std::env::set_var("ATOM_CACHE_POOL_MAX_SIZE", "0"); + + let err = Config::from_env().expect_err("zero pool size"); + assert!(err.to_string().contains("ATOM_CACHE_POOL_MAX_SIZE")); + + clear_hardening_env(); + } + + #[test] + fn cache_disabled_ignores_missing_redis_url() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + // ATOM_CACHE_ENABLED left unset (false) — an absent/invalid redis url + // must not fail config parsing when caching is off. + let cfg = Config::from_env().expect("config"); + assert!(!cfg.cache.enabled); + assert!(cfg.cache.redis_url.is_empty()); + + clear_hardening_env(); + } + #[test] fn hot_path_allow_db_audit_opts_in_via_env() { let _guard = ENV_LOCK.lock().expect("env lock"); @@ -1302,6 +1510,18 @@ mod tests { "ATOM_GRPC_TLS_KEY_PATH", "ATOM_GRPC_TLS_CLIENT_CA_PATH", "ATOM_EMAIL_TEMPLATES_DIR", + "ATOM_CACHE_ENABLED", + "ATOM_CACHE_REDIS_URL", + "ATOM_CACHE_POOL_MAX_SIZE", + "ATOM_CACHE_CONNECT_TIMEOUT_MS", + "ATOM_CACHE_OP_TIMEOUT_MS", + "ATOM_CACHE_FAIL_FAST_ON_STARTUP", + "ATOM_CACHE_TTL_SESSION_SECS", + "ATOM_CACHE_TTL_ENTITY_STATUS_SECS", + "ATOM_CACHE_TTL_TENANT_STATUS_SECS", + "ATOM_CACHE_TTL_CREDENTIAL_SECS", + "ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS", + "ATOM_CACHE_TTL_GRANTS_SECS", ] { std::env::remove_var(name); } diff --git a/src/error.rs b/src/error.rs index 7f81c69..c8d762f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub enum AppError { message: String, retry_after_secs: u64, }, + #[error("{0}")] + ServiceUnavailable(String), #[error("database error")] Database(#[from] sqlx::Error), #[error("internal error")] @@ -65,6 +67,9 @@ impl AppError { retry_after_secs, } } + pub fn service_unavailable(msg: impl Into) -> Self { + AppError::ServiceUnavailable(msg.into()) + } } impl IntoResponse for AppError { @@ -90,6 +95,7 @@ impl IntoResponse for AppError { } return response; } + AppError::ServiceUnavailable(m) => (StatusCode::SERVICE_UNAVAILABLE, m.clone()), AppError::Database(e) => { if let sqlx::Error::Database(db) = e { match db.code().as_deref() { @@ -150,6 +156,7 @@ impl From for tonic::Status { AppError::Conflict(msg) => tonic::Status::already_exists(msg), AppError::PayloadTooLarge(msg) => tonic::Status::invalid_argument(msg), AppError::RateLimited { message, .. } => tonic::Status::resource_exhausted(message), + AppError::ServiceUnavailable(msg) => tonic::Status::unavailable(msg), AppError::Database(e) => { tracing::error!("db error: {e}"); tonic::Status::internal("database error") diff --git a/src/graphql/auth.rs b/src/graphql/auth.rs index 84e009b..dff9026 100644 --- a/src/graphql/auth.rs +++ b/src/graphql/auth.rs @@ -88,32 +88,55 @@ impl AuthMutation { let auth = require_auth(ctx)?; let state = ctx.data::()?; - let mut tx = state - .pool - .begin() - .await - .map_err(|e| gql_error(crate::error::db_err(e)))?; - if let Some(session_id) = auth.session_id { - repo::revoke_session_in_tx(&mut tx, session_id) + let event = audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: auth.tenant_id, + target_kind: Some("entity"), + target_id: Some(auth.entity_id), + event: "auth.logout", + outcome: AuditOutcome::Allow, + details: serde_json::json!({}), + }; + + match auth.session_id { + Some(session_id) => { + // The barrier must span the revoke *and* its commit, not just + // the revoke: releasing it before `commit_with_audit`'s + // internal `tx.commit()` lets a concurrent reader load the + // still-uncommitted (pre-revoke) row from Postgres and + // repopulate the cache with it, right after the barrier + // that was supposed to block exactly that — see + // `src/cache/mod.rs`. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + repo::revoke_session_in_tx(&mut tx, session_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &event, + ) + .await + }, + ) .await .map_err(gql_error)?; + } + None => { + let tx = state + .pool + .begin() + .await + .map_err(|e| gql_error(crate::error::db_err(e)))?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) + .await + .map_err(gql_error)?; + } } - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: auth.tenant_id, - target_kind: Some("entity"), - target_id: Some(auth.entity_id), - event: "auth.logout", - outcome: AuditOutcome::Allow, - details: serde_json::json!({}), - }, - ) - .await - .map_err(gql_error)?; Ok(true) } @@ -126,14 +149,33 @@ impl AuthMutation { )) })?; let state = ctx.data::()?; - let keys = state.keys.read().await; + // Derive the signer under the read guard and release it immediately: + // cloning `LoadedKey` to escape the guard would put a copy of the raw + // PKCS8 private-key PEM on the heap for every refresh, dropped without + // zeroization. Holding the guard across the mutation instead would + // block key rotation for the length of a DB transaction. + let signer = { + let keys = state.keys.read().await; + crate::auth::JwtSigner::from_key(&keys.primary).map_err(gql_error)? + }; - service::refresh_session( - &state.pool, - &state.config, - &keys.primary, - auth.entity_id, - session_id, + // Extends `expires_at` in place for the same session_id; without + // invalidating, a stale cached (shorter) expiry could cause a + // spurious "session expired" false-deny until the entry's own TTL + // catches up — not a security issue, but worth fixing. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || { + service::refresh_session( + &state.pool, + &state.config, + &signer, + auth.entity_id, + session_id, + ) + }, ) .await .map(Into::into) @@ -176,7 +218,8 @@ pub(crate) fn gql_error(err: AppError) -> async_graphql::Error { | AppError::Forbidden | AppError::Conflict(_) | AppError::PayloadTooLarge(_) - | AppError::RateLimited { .. } => async_graphql::Error::new(err.to_string()), + | AppError::RateLimited { .. } + | AppError::ServiceUnavailable(_) => async_graphql::Error::new(err.to_string()), } } diff --git a/src/graphql/credentials.rs b/src/graphql/credentials.rs index dbb2178..9530775 100644 --- a/src/graphql/credentials.rs +++ b/src/graphql/credentials.rs @@ -216,32 +216,39 @@ impl CredentialMutation { .into_iter() .map(permission_input_into_model) .collect::>>()?; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::replace_access_token_permissions_in_tx( - &mut tx, - owner_id, - credential_id, - permissions, - ) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: audit_tenant_id, - target_kind: Some("credential"), - target_id: Some(credential_id), - event: "credential.update", - outcome: AuditOutcome::Allow, - details: serde_json::json!({ - "entity_id": owner_id, - "kind": "access_token", - "delegated": delegated, - "credential_id": credential_id - }), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::CredentialCeiling, + std::slice::from_ref(&crate::cache::keys::cred_ceiling(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::replace_access_token_permissions_in_tx( + &mut tx, + owner_id, + credential_id, + permissions, + ) + .await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: audit_tenant_id, + target_kind: Some("credential"), + target_id: Some(credential_id), + event: "credential.update", + outcome: AuditOutcome::Allow, + details: serde_json::json!({ + "entity_id": owner_id, + "kind": "access_token", + "delegated": delegated, + "credential_id": credential_id + }), + }, + ) + .await }, ) .await @@ -256,27 +263,33 @@ impl CredentialMutation { let credential_id = parse_id(credential_id, "credentialId")?; let (owner_id, delegated, audit_tenant_id) = resolve_token_lifecycle_target(state, &auth, credential_id).await?; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::revoke_access_token_in_tx(&mut tx, owner_id, credential_id) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: audit_tenant_id, - target_kind: Some("credential"), - target_id: Some(credential_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({ - "entity_id": owner_id, - "kind": "access_token", - "delegated": delegated, - "credential_id": credential_id - }), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::revoke_access_token_in_tx(&mut tx, owner_id, credential_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: audit_tenant_id, + target_kind: Some("credential"), + target_id: Some(credential_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({ + "entity_id": owner_id, + "kind": "access_token", + "delegated": delegated, + "credential_id": credential_id + }), + }, + ) + .await }, ) .await @@ -390,22 +403,28 @@ impl CredentialMutation { } else { require_credential_management(state, &auth, entity_id).await? }; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::revoke_credential_in_tx(&mut tx, entity_id, credential_id) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id, - target_kind: Some("entity"), - target_id: Some(entity_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({"credential_id": credential_id}), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::revoke_credential_in_tx(&mut tx, entity_id, credential_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("entity"), + target_id: Some(entity_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({"credential_id": credential_id}), + }, + ) + .await }, ) .await diff --git a/src/graphql/entities.rs b/src/graphql/entities.rs index 8639955..eb1855d 100644 --- a/src/graphql/entities.rs +++ b/src/graphql/entities.rs @@ -298,23 +298,33 @@ impl EntityMutation { .await?; } - repo::update_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - entity_model::UpdateEntity { - name: input.name, - kind: parse_optional_entity_kind(input.kind), - alias: input.alias.into(), - tenant_id, - profile_id, - profile_version_id, - status: input.status.map(Into::into), - attributes: input.attributes, + // Status and tenant are both part of `atom:v1:entity_status:*`'s + // payload (see `src/cache/entries.rs`), so any update invalidates + // it — cheap and correct even when neither actually changed. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || { + repo::update_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + entity_model::UpdateEntity { + name: input.name, + kind: parse_optional_entity_kind(input.kind), + alias: input.alias.into(), + tenant_id, + profile_id, + profile_version_id, + status: input.status.map(Into::into), + attributes: input.attributes, + }, + "entity.update", + details.clone(), + ) }, - "entity.update", - details.clone(), ) .await } @@ -360,10 +370,10 @@ impl EntityMutation { ) .await?; } - repo::delete_entity_with_audit( + crate::identity::service::delete_entity( &state.pool, + state.cache.as_deref(), state.config.events.enabled(), - Some(auth.entity_id), id, Some(auth.entity_id), ) @@ -406,12 +416,21 @@ impl EntityMutation { let result = async { crate::auth::require_any_capability(&state.pool, &auth, &[("manage", Scope::Platform)]) .await?; - repo::restore_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + // Separate function from `update_entity`/`change_entity_status`, + // so it needs its own `entity_status` invalidation too. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || { + repo::restore_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + }, ) .await } @@ -669,23 +688,30 @@ async fn change_entity_status(ctx: &Context<'_>, id: ID, status: EntityStatus) - ], ) .await?; - repo::update_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - entity_id, - entity_model::UpdateEntity { - name: None, - kind: None, - alias: None, - tenant_id: None, - profile_id: None, - profile_version_id: None, - status: Some(status), - attributes: None, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(entity_id)), + || { + repo::update_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + entity_id, + entity_model::UpdateEntity { + name: None, + kind: None, + alias: None, + tenant_id: None, + profile_id: None, + profile_version_id: None, + status: Some(status), + attributes: None, + }, + event, + details.clone(), + ) }, - event, - details.clone(), ) .await } diff --git a/src/graphql/groups.rs b/src/graphql/groups.rs index 2c88b62..c6338dc 100644 --- a/src/graphql/groups.rs +++ b/src/graphql/groups.rs @@ -378,24 +378,92 @@ impl GroupMutation { event: "group.update", }; let details = serde_json::json!({}); + let status: Option = input.status.map(Into::into); + let status_changing = status.is_some(); let result = async { let existing = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, existing.tenant_id).await?; - repo::update_group_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - UpdateGroup { - name: input.name, - description: input.description, - status: input.status.map(Into::into), - attributes: input.attributes, - }, - "group.update", - details.clone(), - ) - .await + let update = UpdateGroup { + name: input.name, + description: input.description, + status, + attributes: input.attributes, + }; + // `subject_effective_grants`'s `subject_groups` CTE requires + // `status = 'active'` at every hop, so any status change (not + // just a move off "active") changes the recursive grant set for + // every member of this group's subtree. + if status_changing { + // Locked (not just enumerated) — see + // `authz::repo::lock_group_closures_and_collect_member_ids` + // for why a concurrent `add_group_member` needs this to be + // safe, not just a plain cache barrier. + let Some(cache) = state.cache.as_deref() else { + return repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + "group.update", + details.clone(), + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let details = details.clone(); + Box::pin(async move { + repo::update_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + update, + "group.update", + details, + ) + .await + }) + }, + ) + .await + .inspect(|group| { + // Only needed on this locked path — the non-status-change + // branch below already gets it from + // `update_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.update", + }, + &details, + ); + }) + } else { + repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + "group.update", + details.clone(), + ) + .await + } } .await; if let Err(ref err) = result { @@ -434,14 +502,61 @@ impl GroupMutation { let result = async { let group = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, group.tenant_id).await?; - repo::set_group_parent_with_audit( + // Reparenting `id` only changes what `id` and its descendants + // inherit from above — it never changes `group_hierarchy` rows + // below `id`, so `id`'s subtree is exactly what this mutation + // can affect. Locked (not just enumerated) — see + // `authz::repo::lock_group_closures_and_collect_member_ids` for + // why a concurrent `add_group_member` needs this to be safe. + let Some(cache) = state.cache.as_deref() else { + return repo::set_group_parent_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + parent_id, + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - parent_id, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::set_group_parent_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + parent_id, + ) + .await + }) + }, ) - .await + .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `set_group_parent_with_audit` + // itself. + repo::get_group(&state.pool, id).await.inspect(|group| { + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.parent.set", + }, + &serde_json::json!({ "parent_id": parent_id }), + ); + }) } .await; if let Err(ref err) = result { @@ -483,13 +598,53 @@ impl GroupMutation { let group = repo::get_group(&state.pool, id).await?; let tenant_id = group.tenant_id; require_group_manage_app(&state.pool, &auth, id, tenant_id).await?; - repo::remove_group_parent_with_audit( + // Locked, not just enumerated — see `set_group_parent` above. + let Some(cache) = state.cache.as_deref() else { + repo::remove_group_parent_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::remove_group_parent_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `remove_group_parent_with_audit` + // itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.parent.remove", + }, + &serde_json::json!({}), + ); Ok(tenant_id) } .await; @@ -538,14 +693,56 @@ impl GroupMutation { let existing = repo::get_group(&state.pool, id).await?; let tenant_id = existing.tenant_id; require_group_manage_app(&state.pool, &auth, id, tenant_id).await?; - repo::delete_group_with_audit( + // `group_hierarchy` rows aren't touched by a soft delete (only + // `deleted_at` is set), so enumeration is unaffected by timing. + // Locked, not just enumerated — see `set_group_parent` above. + let Some(cache) = state.cache.as_deref() else { + repo::delete_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::delete_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `delete_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.delete", + }, + &details, + ); Ok(tenant_id) } .await; @@ -580,14 +777,67 @@ impl GroupMutation { let result = async { crate::auth::require_any_capability(&state.pool, &auth, &[("manage", Scope::Platform)]) .await?; - repo::restore_group_with_audit( + // Locked, not just enumerated — see `set_group_parent` above (the + // lock works regardless of the group's own `deleted_at` status, + // so it applies here unchanged). + let Some(cache) = state.cache.as_deref() else { + return repo::restore_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::restore_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await + .await?; + // Mirrors `restore_group_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_group_with_audit` itself. + let tenant_id = repo::get_group(&state.pool, id) + .await + .ok() + .and_then(|g| g.tenant_id); + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("group"), + target_id: Some(id), + event: "group.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) } .await; if let Err(ref err) = result { @@ -664,12 +914,19 @@ impl GroupMutation { ], ) .await?; - repo::add_group_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - group_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + repo::add_group_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + group_id, + entity_id, + ) + }, ) .await?; Ok(tenant_id) @@ -718,12 +975,19 @@ impl GroupMutation { ], ) .await?; - repo::remove_group_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - group_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + repo::remove_group_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + group_id, + entity_id, + ) + }, ) .await?; Ok(tenant_id) @@ -774,21 +1038,66 @@ impl GroupMutation { let result = async { let group = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, group.tenant_id).await?; - repo::update_group_with_audit( + let update = UpdateGroup { + name: None, + description: None, + status: Some(status), + attributes: None, + }; + // Locked, not just enumerated — see `update_group` above. + let Some(cache) = state.cache.as_deref() else { + return repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + event, + details.clone(), + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - UpdateGroup { - name: None, - description: None, - status: Some(status), - attributes: None, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let details = details.clone(); + Box::pin(async move { + repo::update_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + update, + event, + details, + ) + .await + }) }, - event, - details.clone(), ) .await + .inspect(|group| { + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `update_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event, + }, + &details, + ); + }) } .await; if let Err(ref err) = result { diff --git a/src/graphql/policies.rs b/src/graphql/policies.rs index 310aaa5..092ec63 100644 --- a/src/graphql/policies.rs +++ b/src/graphql/policies.rs @@ -442,14 +442,61 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::replace_role_permission_block_links_with_audit( + // The role's assignees (entity and group) are locked and + // enumerated together as one step — see + // `authz::repo::lock_role_and_collect_grants_keys` — so a + // concurrent `add_group_member` on one of its assigned groups + // can't commit a membership change this misses. + let Some(cache) = state.cache.as_deref() else { + authz_repo::replace_role_permission_block_links_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + role_id, + &permission_block_ids, + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - role_id, - &permission_block_ids, + |tx| { + Box::pin(async move { + authz_repo::lock_role_and_collect_grants_keys(tx, role_id).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let permission_block_ids = permission_block_ids.clone(); + Box::pin(async move { + authz_repo::replace_role_permission_block_links_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + role_id, + &permission_block_ids, + ) + .await + }) + }, ) .await?; + // `_in_tx` already enqueued the outbox row before returning — this + // is the post-commit stdout observability log. Only needed on + // this locked path: the cache-disabled branch above already gets + // it from `replace_role_permission_block_links_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role", + target_id: Some(role_id), + event: "role.permission_blocks.replace", + }, + &serde_json::json!({ "permission_block_ids": permission_block_ids }), + ); Ok(tenant_id) } .await; @@ -494,14 +541,55 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_role_with_audit( + // See `replace_role_permission_blocks` above for why the role + // and its assignees are locked (not just enumerated) together. + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_role_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + |tx| { + Box::pin( + async move { authz_repo::lock_role_and_collect_grants_keys(tx, id).await }, + ) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_role_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `delete_role_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role", + target_id: Some(id), + event: "role.delete", + }, + &details, + ); Ok(tenant_id) } .await; @@ -536,14 +624,66 @@ impl PolicyMutation { let details = serde_json::json!({}); let result = async { require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - authz_repo::restore_role_with_audit( + // See `replace_role_permission_blocks` above for why the role + // and its assignees are locked (not just enumerated) together. + let Some(cache) = state.cache.as_deref() else { + return authz_repo::restore_role_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + |tx| { + Box::pin( + async move { authz_repo::lock_role_and_collect_grants_keys(tx, id).await }, + ) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::restore_role_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await + .await?; + // Mirrors `restore_role_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_role_with_audit` itself. + let tenant_id = authz_repo::get_role(&state.pool, id) + .await + .ok() + .and_then(|r| r.tenant_id); + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("role"), + target_id: Some(id), + event: "role.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) } .await; if let Err(ref err) = result { @@ -1015,18 +1155,91 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::create_role_assignment_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - CreateRoleAssignment { - tenant_id, - subject_kind, - subject_id, - role_id, - }, - ) - .await + let req = CreateRoleAssignment { + tenant_id, + subject_kind: subject_kind.clone(), + subject_id, + role_id, + }; + match subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::create_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + }, + ) + .await + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + return authz_repo::create_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + }; + // Locks the role *before* `subject_id`'s group closure — + // matching `replaceRolePermissionBlocks`/`deleteRole`/ + // `restoreRole`'s lock order exactly, since locking the + // group first here (with the role locked later, inside + // `create_role_assignment_in_tx`) can deadlock against + // those paths. See + // `authz::repo::lock_role_then_group_closure_and_collect_grants_keys`. + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_role_then_group_closure_and_collect_grants_keys( + tx, role_id, subject_id, + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::create_role_assignment_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + req, + ) + .await + }) + }, + ) + .await + .inspect(|assignment| { + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `create_role_assignment_with_audit` + // itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: assignment.tenant_id, + target_kind: "role_assignment", + target_id: Some(assignment.id), + event: "role_assignment.create", + }, + &details, + ); + }) + } + } } .await; if let Err(ref err) = result { @@ -1064,13 +1277,77 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_role_assignment_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await?; + match assignment.subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(assignment.subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::delete_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + }, + ) + .await?; + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[assignment.subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_role_assignment_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, + ) + .await?; + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `delete_role_assignment_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }, + &details, + ); + } + } Ok(tenant_id) } .await; @@ -1114,18 +1391,84 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::create_direct_policy_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - CreateDirectPolicy { - tenant_id, - subject_kind, - subject_id, - permission_block_id, - }, - ) - .await + let req = CreateDirectPolicy { + tenant_id, + subject_kind: subject_kind.clone(), + subject_id, + permission_block_id, + }; + match subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::create_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + }, + ) + .await + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + return authz_repo::create_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::create_direct_policy_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + req, + ) + .await + }) + }, + ) + .await + .inspect(|policy| { + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `create_direct_policy_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: policy.tenant_id, + target_kind: "direct_policy", + target_id: Some(policy.id), + event: "direct_policy.create", + }, + &details, + ); + }) + } + } } .await; if let Err(ref err) = result { @@ -1163,13 +1506,77 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_direct_policy_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await?; + match policy.subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(policy.subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::delete_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + }, + ) + .await?; + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[policy.subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_direct_policy_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, + ) + .await?; + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `delete_direct_policy_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "direct_policy", + target_id: Some(id), + event: "direct_policy.delete", + }, + &details, + ); + } + } Ok(tenant_id) } .await; diff --git a/src/graphql/schema.rs b/src/graphql/schema.rs index ca49453..f7c3667 100644 --- a/src/graphql/schema.rs +++ b/src/graphql/schema.rs @@ -519,6 +519,7 @@ mod tests { standby: None, }, None, + None, ) } diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 107dc10..2c95f0e 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -289,18 +289,31 @@ impl TenantMutation { &[("manage", Scope::Platform), ("create", Scope::Platform)], ) .await?; - tenant_repo::create_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_model::CreateTenant { - id, - name: input.name, - alias: input.alias, - tags: input.tags.unwrap_or_default(), - attributes: input.attributes.unwrap_or(serde_json::Value::Null), + // `create_tenant` bootstraps a tenant-admin role, role assignment + // and membership for the creator in the same transaction, so it + // grows the creator's own grant set. The capability gate directly + // above has just warmed that exact `grants` entry, so without this + // barrier a creator who isn't already a platform admin cannot + // manage the tenant they just created until the grants TTL lapses. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || { + tenant_repo::create_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_model::CreateTenant { + id, + name: input.name, + alias: input.alias, + tags: input.tags.unwrap_or_default(), + attributes: input.attributes.unwrap_or(serde_json::Value::Null), + }, + Some(auth.entity_id), + ) }, - Some(auth.entity_id), ) .await } @@ -392,16 +405,68 @@ impl TenantMutation { }; let details = serde_json::json!({}); - let result = async { + let result: std::result::Result<(), AppError> = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::soft_delete_tenant_with_audit( - &state.pool, + // `tenant_status` invalidation alone is *not* sufficient: this + // also bulk-revokes sessions, which would otherwise become a + // stale cache hit again the moment `restoreTenant` repopulates + // tenant_status as active. Credentials don't need the same + // treatment — `restore_tenant`'s own invalidation (see + // `reactivate_tenant_and_collect_credential_ids_in_tx`) already + // covers that side by the time a restore could matter. See + // `lock_tenant_and_collect_session_ids_in_tx` for why session ids + // are enumerated inside the same locked transaction, and why that + // lock-and-enumerate step must run before the barrier below is + // established, not after the tenant is flipped to `deleted`. + let Some(cache) = state.cache.as_deref() else { + tenant_repo::soft_delete_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + .await?; + return Ok(()); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let session_ids = + tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id).await?; + let session_keys: Vec = session_ids + .iter() + .map(|id| crate::cache::keys::session(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::deactivate_and_finish_tenant_soft_delete_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), - tenant_id, Some(auth.entity_id), + tenant_id, ) - .await + .await; + let outcome = match outcome { + Ok(_tenant) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + // `deactivate_and_finish_tenant_soft_delete_in_tx` already + // enqueued the outbox row before returning — this is the + // post-commit stdout observability log. Only needed on this + // locked path; the cache-disabled branch above already gets it + // from `soft_delete_tenant_with_audit` itself. + if outcome.is_ok() { + audit::log_observe_allow(&meta, &details); + } + outcome } .await; @@ -438,14 +503,82 @@ impl TenantMutation { let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::restore_tenant_with_audit( - &state.pool, + // Unlike the tenant-status-only case in `delete_tenant`, this + // also reactivates credentials, which need their own + // invalidation: `verify_api_key_snapshot` checks a credential's + // status first and isn't overridden by a fresher tenant_status + // check afterward, so a just-restored API key would keep + // getting denied until its own cache entry's TTL expires + // otherwise. See + // `reactivate_tenant_and_collect_credential_ids_in_tx` for why + // credential ids are enumerated inside the same locked + // transaction. + let Some(cache) = state.cache.as_deref() else { + return tenant_repo::restore_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let (tenant, credential_ids) = + tenant_repo::reactivate_tenant_and_collect_credential_ids_in_tx( + &mut tx, + tenant_id, + Some(auth.entity_id), + ) + .await?; + let credential_keys: Vec = credential_ids + .iter() + .map(|id| crate::cache::keys::credential(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Credential, &credential_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::finish_tenant_restore_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), tenant_id, - Some(auth.entity_id), ) - .await + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + outcome?; + // Mirrors `restore_tenant_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_tenant_with_audit` itself. + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: Some(tenant.id), + target_kind: Some("tenant"), + target_id: Some(tenant.id), + event: "tenant.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({ + "tenant_name": tenant.name, + }), + }, + ) + .await; + Ok(tenant) } .await; @@ -568,10 +701,16 @@ impl TenantMutation { async fn accept_tenant_invitation(&self, ctx: &Context<'_>, tenant_id: ID) -> Result { let auth = require_auth(ctx)?; let state = ctx.data::()?; - tenant_repo::accept_invitation( - &state.pool, - parse_id(tenant_id, "tenantId")?, - auth.entity_id, + let tenant_id = parse_id(tenant_id, "tenantId")?; + // Invitation acceptance grants tenant membership (and possibly a + // role) to `auth.entity_id` — easy to miss since it doesn't go + // through the "obvious" policy/role-assignment mutation entry + // points. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || tenant_repo::accept_invitation(&state.pool, tenant_id, auth.entity_id), ) .await .map_err(gql_error)?; @@ -585,10 +724,14 @@ impl TenantMutation { ) -> Result { let auth = require_auth(ctx)?; let state = ctx.data::()?; - let tenant_id = - tenant_repo::accept_invitation_token(&state.pool, &input.token, auth.entity_id) - .await - .map_err(gql_error)?; + let tenant_id = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || tenant_repo::accept_invitation_token(&state.pool, &input.token, auth.entity_id), + ) + .await + .map_err(gql_error)?; Ok(ID::from(tenant_id.to_string())) } @@ -650,12 +793,19 @@ impl TenantMutation { Scope::Tenant(tenant_id), ) .await?; - tenant_repo::remove_tenant_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + tenant_repo::remove_tenant_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + entity_id, + ) + }, ) .await } @@ -701,13 +851,20 @@ impl TenantMutation { Scope::Tenant(tenant_id), ) .await?; - tenant_repo::add_tenant_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - entity_id, - role_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + tenant_repo::add_tenant_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + entity_id, + role_id, + ) + }, ) .await } @@ -742,15 +899,100 @@ async fn change_tenant_status(ctx: &Context<'_>, id: ID, status: TenantStatus) - let status_detail = status.clone(); let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::change_tenant_status_with_audit( - &state.pool, + // Only `tenant_status`, not `grants`: the PDP's tenant-lifecycle deny + // check runs before grant matching (see `authz::engine:: + // load_decision_context`), so a stale tenant-membership-implicit + // grant inside a cached `grants` entry is harmless as long as this + // key itself invalidates. + let Some(cache) = state.cache.as_deref() else { + return tenant_repo::change_tenant_status_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, + ) + .await; + }; + if status == TenantStatus::Active { + // Re-enabling never revokes anything (see + // `change_tenant_status_in_tx`), so `tenant_status` is the only + // key that needs a barrier here. + return crate::cache::invalidate::guarded_mutation( + Some(cache), + crate::cache::CacheCategory::TenantStatus, + std::slice::from_ref(&crate::cache::keys::tenant_status(tenant_id)), + || { + tenant_repo::change_tenant_status_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, + ) + }, + ) + .await; + } + // Disabling/freezing also bulk-revokes the tenant's members' + // sessions (see `change_tenant_status_in_tx`) — lock the tenant and + // enumerate those session ids first, establish the barrier on both + // categories, then mutate. Mirrors `deleteTenant` exactly, and for + // the same reason: a session cache entry this call is about to + // revoke must never be left reachable as a stale hit, or it survives + // (with `revoked_at = None`) until the tenant is re-enabled and its + // own fresh `tenant_status` hit stops masking the stale session. + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let session_ids = + tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id).await?; + let session_keys: Vec = session_ids + .iter() + .map(|id| crate::cache::keys::session(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::change_tenant_status_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), tenant_id, status, event, ) - .await + .await; + let outcome = match outcome { + Ok(tenant) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| tenant), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + let tenant = outcome?; + // Only needed on this locked path — the `Active` branch above and + // the cache-disabled fallback both already get it from + // `change_tenant_status_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: Some(tenant_id), + target_kind: "tenant", + target_id: Some(tenant_id), + event, + }, + &serde_json::json!({ "status": status_detail.clone() }), + ); + Ok(tenant) } .await; if let Err(ref err) = result { diff --git a/src/grpc.rs b/src/grpc.rs index 54a8750..72377dc 100644 --- a/src/grpc.rs +++ b/src/grpc.rs @@ -816,6 +816,7 @@ mod tests { standby: None, }, None, + None, ) } } diff --git a/src/guardrails.rs b/src/guardrails.rs index 92f3043..68f3e13 100644 --- a/src/guardrails.rs +++ b/src/guardrails.rs @@ -1,4 +1,14 @@ -use sqlx::PgPool; +//! Assignment guardrails: the rules that decide whether a grant may be +//! created at all, independently of who is asking. +//! +//! Every entry point takes `&mut PgConnection` rather than `&PgPool`. Callers +//! run these validations under the row locks their mutation already holds, so +//! the reads must go through *that* transaction: a second pooled connection +//! neither sees the transaction's uncommitted state nor respects its locks, +//! and acquiring one while holding a transaction risks exhausting the pool +//! (every request holding one connection and waiting for a second). + +use sqlx::PgConnection; use uuid::Uuid; use crate::{ @@ -91,17 +101,20 @@ impl Rule { } } -pub async fn validate_policy(pool: &PgPool, req: &CreatePolicyBinding) -> Result<(), AppError> { - let assignments = assignments_for_policy(pool, req).await?; - validate_assignments(pool, &assignments).await +pub async fn validate_policy( + conn: &mut PgConnection, + req: &CreatePolicyBinding, +) -> Result<(), AppError> { + let assignments = assignments_for_policy(&mut *conn, req).await?; + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_capability( - pool: &PgPool, + conn: &mut PgConnection, role_id: Uuid, capability_id: Uuid, ) -> Result<(), AppError> { - let capability_names = capability_names(pool, &[capability_id]).await?; + let capability_names = capability_names(&mut *conn, &[capability_id]).await?; let rows = sqlx::query( r#"WITH RECURSIVE assigned_groups(edge_id, group_id) AS ( SELECT pb.id, pb.subject_id @@ -124,7 +137,7 @@ pub async fn validate_role_capability( JOIN entities e ON e.id = gm.entity_id"#, ) .bind(role_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; @@ -145,22 +158,22 @@ pub async fn validate_role_capability( })); } - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_assignment( - pool: &PgPool, + conn: &mut PgConnection, tenant_id: Option, subject_kind: SubjectKind, subject_id: Uuid, role_id: Uuid, ) -> Result<(), AppError> { - let entity_kinds = subject_entity_kinds(pool, subject_kind, subject_id).await?; + let entity_kinds = subject_entity_kinds(&mut *conn, subject_kind, subject_id).await?; if entity_kinds.is_empty() { return Ok(()); } - let role_capabilities = role_permission_assignments(pool, &[role_id]).await?; + let role_capabilities = role_permission_assignments(&mut *conn, &[role_id]).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -174,11 +187,11 @@ pub async fn validate_role_assignment( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_composite_role_assignment_plan( - pool: &PgPool, + conn: &mut PgConnection, entity_ids: &[Uuid], child_role_ids: &[Uuid], tenant_id: Option, @@ -197,14 +210,14 @@ pub async fn validate_composite_role_assignment_plan( let entity_kinds = sqlx::query_scalar::<_, String>("SELECT kind FROM entities WHERE id = ANY($1::uuid[])") .bind(&unique_entity_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; if entity_kinds.len() != unique_entity_ids.len() { return Err(AppError::bad_request("invalid member reference")); } - let role_capabilities = role_permission_assignments(pool, &unique_child_role_ids).await?; + let role_capabilities = role_permission_assignments(&mut *conn, &unique_child_role_ids).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -218,11 +231,11 @@ pub async fn validate_composite_role_assignment_plan( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_assignment_plan( - pool: &PgPool, + conn: &mut PgConnection, entity_ids: &[Uuid], capability_ids: &[Uuid], tenant_id: Option, @@ -243,14 +256,14 @@ pub async fn validate_role_assignment_plan( let entity_kinds = sqlx::query_scalar::<_, String>("SELECT kind FROM entities WHERE id = ANY($1::uuid[])") .bind(&unique_entity_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; if entity_kinds.len() != unique_entity_ids.len() { return Err(AppError::bad_request("invalid member reference")); } - let capability_names = capability_names(pool, &unique_capability_ids).await?; + let capability_names = capability_names(&mut *conn, &unique_capability_ids).await?; if capability_names.len() != unique_capability_ids.len() { return Err(AppError::bad_request("invalid capability reference")); } @@ -273,7 +286,7 @@ pub async fn validate_role_assignment_plan( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } /// Takes the caller's connection rather than the pool: every caller runs this @@ -281,7 +294,7 @@ pub async fn validate_role_assignment_plan( /// while holding one deadlocks a saturated pool (and hangs outright at /// `max_connections = 1`). pub async fn validate_group_member( - conn: &mut sqlx::PgConnection, + conn: &mut PgConnection, group_id: Uuid, entity_id: Uuid, ) -> Result<(), AppError> { @@ -340,11 +353,13 @@ pub async fn validate_group_member( } pub async fn validate_direct_policy( - pool: &PgPool, + conn: &mut PgConnection, req: &CreateDirectPolicy, ) -> Result<(), AppError> { - let entity_kinds = subject_entity_kinds(pool, req.subject_kind.clone(), req.subject_id).await?; - let permission_blocks = permission_block_assignments(pool, &[req.permission_block_id]).await?; + let entity_kinds = + subject_entity_kinds(&mut *conn, req.subject_kind.clone(), req.subject_id).await?; + let permission_blocks = + permission_block_assignments(&mut *conn, &[req.permission_block_id]).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -358,12 +373,12 @@ pub async fn validate_direct_policy( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } /// Takes the caller's connection, not the pool — see [`validate_group_member`]. pub async fn validate_role_permission_block_links( - conn: &mut sqlx::PgConnection, + conn: &mut PgConnection, role_id: Uuid, permission_block_ids: &[Uuid], ) -> Result<(), AppError> { @@ -407,13 +422,14 @@ pub async fn validate_role_permission_block_links( } async fn assignments_for_policy( - pool: &PgPool, + conn: &mut PgConnection, req: &CreatePolicyBinding, ) -> Result, AppError> { - let entity_kinds = subject_entity_kinds(pool, req.subject_kind.clone(), req.subject_id).await?; + let entity_kinds = + subject_entity_kinds(&mut *conn, req.subject_kind.clone(), req.subject_id).await?; let capability_names = match req.grant_kind { - GrantKind::Capability => capability_names(pool, &[req.grant_id]).await?, - GrantKind::Role => role_capability_names(pool, req.grant_id).await?, + GrantKind::Capability => capability_names(&mut *conn, &[req.grant_id]).await?, + GrantKind::Role => role_capability_names(&mut *conn, req.grant_id).await?, }; let (object_kind, object_type) = scope_to_object(req.scope_kind.clone(), req.scope_ref.as_deref()); @@ -436,30 +452,24 @@ async fn assignments_for_policy( .collect()) } -async fn validate_assignments<'e, E>( - executor: E, +async fn validate_assignments( + conn: &mut PgConnection, assignments: &[Assignment], -) -> Result<(), AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +) -> Result<(), AppError> { if assignments.is_empty() { return Ok(()); } - let rules = load_rules(executor).await?; + let rules = load_rules(&mut *conn).await?; decide(assignments, &rules).map_err(AppError::bad_request) } -async fn load_rules<'e, E>(executor: E) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn load_rules(conn: &mut PgConnection) -> Result, AppError> { use sqlx::Row; sqlx::query( r#"SELECT tenant_id, entity_kind, action_name AS capability_name, object_kind, object_type, decision, is_absolute FROM action_assignment_rules"#, ) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() @@ -478,14 +488,14 @@ where } async fn subject_entity_kinds( - pool: &PgPool, + conn: &mut PgConnection, subject_kind: SubjectKind, subject_id: Uuid, ) -> Result, AppError> { match subject_kind { SubjectKind::Entity => sqlx::query_scalar("SELECT kind FROM entities WHERE id = $1") .bind(subject_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err), SubjectKind::Group => sqlx::query_scalar( @@ -502,27 +512,24 @@ async fn subject_entity_kinds( WHERE gm.group_id IN (SELECT group_id FROM subject_groups)"#, ) .bind(subject_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err), } } -async fn capability_names<'e, E>(executor: E, ids: &[Uuid]) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn capability_names(conn: &mut PgConnection, ids: &[Uuid]) -> Result, AppError> { sqlx::query_scalar("SELECT name FROM actions WHERE id = ANY($1::uuid[])") .bind(ids) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err) } -async fn role_capability_names<'e, E>(executor: E, role_id: Uuid) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn role_capability_names( + conn: &mut PgConnection, + role_id: Uuid, +) -> Result, AppError> { sqlx::query_scalar( r#"SELECT DISTINCT c.name FROM (SELECT $1::uuid AS role_id) roles @@ -530,7 +537,7 @@ where JOIN actions c ON c.id = rc.capability_id"#, ) .bind(role_id) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err) } @@ -543,7 +550,7 @@ struct RoleCapabilityAssignment { } async fn role_permission_assignments( - pool: &PgPool, + conn: &mut PgConnection, role_ids: &[Uuid], ) -> Result, AppError> { if role_ids.is_empty() { @@ -602,7 +609,7 @@ async fn role_permission_assignments( WHERE rpb.role_id = ANY($1::uuid[])"#, ) .bind(role_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() @@ -616,13 +623,10 @@ async fn role_permission_assignments( .collect() } -async fn permission_block_assignments<'e, E>( - executor: E, +async fn permission_block_assignments( + conn: &mut PgConnection, permission_block_ids: &[Uuid], -) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +) -> Result, AppError> { if permission_block_ids.is_empty() { return Ok(Vec::new()); } @@ -678,7 +682,7 @@ where WHERE pb.id = ANY($1::uuid[])"#, ) .bind(permission_block_ids) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() diff --git a/src/health.rs b/src/health.rs index 5747aff..c9c4402 100644 --- a/src/health.rs +++ b/src/health.rs @@ -74,6 +74,7 @@ pub struct SystemStatus { pub migrations: ComponentCheck, pub signing_keys: ComponentCheck, pub certificate_issuer: ComponentCheck, + pub cache: ComponentCheck, pub db_pool: DbPoolStatus, pub signing_key_state: Option, pub audit_retention: AuditRetentionStatus, @@ -97,6 +98,7 @@ pub async fn readiness(state: &AppState) -> (StatusCode, Json) { let migrations = migrations_check(state).await; let (signing_keys, signing_key_state) = signing_keys_check(state).await; let certificate_issuer = certificate_issuer_check(state); + let cache = cache_check(state).await; let grpc_ready = grpc_check(state).await; let ready = readiness_ok( &database, @@ -124,6 +126,7 @@ pub async fn readiness(state: &AppState) -> (StatusCode, Json) { migrations, signing_keys, certificate_issuer, + cache, db_pool: db_pool_status(state), signing_key_state, audit_retention: audit_retention_status(state).await, @@ -260,6 +263,34 @@ fn certificate_issuer_check(state: &AppState) -> ComponentCheck { } } +/// Never part of the hard readiness requirement (see `readiness_ok`) — caching +/// is a performance optimization with a Postgres fallback for reads, so a down +/// Redis must not fail `/health/ready`. While enabled, an unreachable Redis +/// does refuse security-sensitive mutations (see `src/cache/mod.rs`); that +/// distinction is noted in the message rather than folded into the readiness +/// gate, which only concerns general request-serving availability. +async fn cache_check(state: &AppState) -> ComponentCheck { + let Some(cache) = &state.cache else { + return ComponentCheck { + status: ComponentStatus::Disabled, + message: "cache disabled".to_string(), + }; + }; + match cache.ping().await { + Ok(()) => ComponentCheck { + status: ComponentStatus::Ok, + message: "cache reachable".to_string(), + }, + Err(err) => ComponentCheck { + status: ComponentStatus::Degraded, + message: format!( + "cache unreachable: {err}; reads fall back to the database, but \ + security-sensitive mutations are refused until it recovers" + ), + }, + } +} + async fn grpc_check(state: &AppState) -> ComponentCheck { let status = state.grpc_status().await; match status.state { @@ -341,6 +372,7 @@ mod tests { migrations: check(ComponentStatus::Ok), signing_keys: check(ComponentStatus::Ok), certificate_issuer: check(ComponentStatus::Disabled), + cache: check(ComponentStatus::Disabled), db_pool: DbPoolStatus { max_connections: 0, min_connections: 0, diff --git a/src/identity/handlers.rs b/src/identity/handlers.rs index b48a64c..e2d8c6a 100644 --- a/src/identity/handlers.rs +++ b/src/identity/handlers.rs @@ -193,7 +193,7 @@ pub async fn reset_password( State(state): State, Json(req): Json, ) -> Result { - service::reset_password(&state.pool, req).await?; + service::reset_password(&state.pool, state.cache.as_deref(), req).await?; Ok(StatusCode::NO_CONTENT) } @@ -242,25 +242,43 @@ pub async fn logout( State(state): State, auth: AuthContext, ) -> Result { - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - if let Some(session_id) = auth.session_id { - repo::revoke_session_in_tx(&mut tx, session_id).await?; + let event = audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: auth.tenant_id, + target_kind: Some("entity"), + target_id: Some(auth.entity_id), + event: "auth.logout", + outcome: AuditOutcome::Allow, + details: serde_json::json!({}), + }; + + match auth.session_id { + Some(session_id) => { + // The barrier must span the revoke *and* its commit, not just + // the revoke: releasing it before `commit_with_audit`'s internal + // `tx.commit()` lets a concurrent reader load the still- + // uncommitted (pre-revoke) row from Postgres and repopulate the + // cache with it, right after the barrier that was supposed to + // block exactly that — see `src/cache/mod.rs`. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + repo::revoke_session_in_tx(&mut tx, session_id).await?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) + .await + }, + ) + .await?; + } + None => { + let tx = state.pool.begin().await.map_err(crate::error::db_err)?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) + .await?; + } } - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: auth.tenant_id, - target_kind: Some("entity"), - target_id: Some(auth.entity_id), - event: "auth.logout", - outcome: AuditOutcome::Allow, - details: serde_json::json!({}), - }, - ) - .await?; let mut response = Json(serde_json::json!({"authenticated": false})).into_response(); response.headers_mut().append( header::SET_COOKIE, @@ -398,7 +416,13 @@ pub async fn update_entity( scope_for_tenant(existing.tenant_id), ) .await?; - let entity = repo::update_entity(&state.pool, id, req).await?; + let entity = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || repo::update_entity(&state.pool, id, req), + ) + .await?; Ok(Json(entity)) } @@ -417,7 +441,14 @@ pub async fn delete_entity( ) .await?; } - repo::delete_entity(&state.pool, id, Some(auth.entity_id)).await?; + service::delete_entity( + &state.pool, + state.cache.as_deref(), + state.config.events.enabled(), + id, + Some(auth.entity_id), + ) + .await?; Ok(StatusCode::NO_CONTENT) } @@ -548,20 +579,31 @@ pub async fn revoke_credential( } else { require_credential_management(&state, &auth, entity_id).await? }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - service::revoke_credential_in_tx(&mut tx, entity_id, cred_id).await?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id, - target_kind: Some("entity"), - target_id: Some(entity_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({"credential_id": cred_id}), + // `revoke_credential` isn't kind-filtered — it can revoke an access-token + // credential too, not just password, so it must invalidate the same + // `atom:v1:credential:*` key `revoke_access_token` does. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(cred_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + service::revoke_credential_in_tx(&mut tx, entity_id, cred_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("entity"), + target_id: Some(entity_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({"credential_id": cred_id}), + }, + ) + .await }, ) .await?; @@ -770,7 +812,13 @@ pub async fn add_group_member( scope_for_tenant(group.tenant_id), ) .await?; - repo::add_group_member(&state.pool, group_id, req.entity_id).await?; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(req.entity_id)), + || repo::add_group_member(&state.pool, group_id, req.entity_id), + ) + .await?; Ok(StatusCode::NO_CONTENT) } @@ -798,7 +846,13 @@ pub async fn remove_group_member( scope_for_tenant(group.tenant_id), ) .await?; - repo::remove_group_member(&state.pool, group_id, entity_id).await?; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || repo::remove_group_member(&state.pool, group_id, entity_id), + ) + .await?; Ok(StatusCode::NO_CONTENT) } diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 24a9dfb..8fda210 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -695,43 +695,129 @@ fn entity_kind_as_str(kind: &EntityKind) -> &'static str { } } -/// Soft-delete an entity: mark it inactive, set the tombstone, and immediately -/// cut off access by revoking its credentials and active sessions. Physical -/// removal is deferred to the purge cron. Hard delete (the old behavior) relied -/// on FK cascade for the credential/session cleanup, so the revocations are now -/// explicit. -pub async fn delete_entity_with_audit( +/// The exact set of active session ids `delete_entity` is about to revoke — +/// mirrors that function's `UPDATE sessions` `WHERE` clause precisely, so +/// callers can invalidate `atom:v1:session:*` cache entries for them *before* +/// the delete runs (afterward, `revoked_at IS NULL` no longer matches these +/// rows). See `src/cache/mod.rs`'s consistency model. +pub async fn entity_active_session_ids( pool: &PgPool, - events_enabled: bool, - actor_id: Option, - id: Uuid, - deleted_by: Option, -) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; + entity_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar("SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL") + .bind(entity_id) + .fetch_all(pool) + .await + .map_err(db_err) +} - let tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1 AND deleted_at IS NULL") +/// The exact set of active access-token credential ids `delete_entity` is +/// about to revoke — restricted to the one credential kind this codebase +/// caches under `CacheCategory::Credential` (certificates are tracked via the +/// CRL instead, not this cache). Callers invalidate `atom:v1:credential:*` +/// entries for these *before* the delete runs. See `src/cache/mod.rs`'s +/// consistency model. +pub async fn entity_active_access_token_ids( + pool: &PgPool, + entity_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar( + "SELECT id FROM credentials WHERE entity_id = $1 AND status = 'active' AND kind = $2", + ) + .bind(entity_id) + .bind(crate::models::enums::CredentialKind::AccessToken) + .fetch_all(pool) + .await + .map_err(db_err) +} + +/// Locks the entity row and, in the *same* transaction, enumerates the exact +/// session and access-token credential ids a subsequent delete is about to +/// revoke — *before* the entity is touched. Callers establish the cache +/// barrier on these ids (plus the entity's own `entity_status` key) before +/// calling [`deactivate_and_finish_entity_deletion_in_tx`]: starting the +/// barrier only after the status flip would leave a window where a +/// concurrent request can still take a full cache hit on the pre-delete +/// entity/session/credential entries and keep running past the point the +/// delete commits. +/// +/// The `SELECT ... FOR UPDATE` below takes the same exclusive row lock on the +/// entity that `create_session`/`create_access_token` take (via +/// `lock_active_entity`) before inserting. So a session or access token +/// created concurrently for this entity either committed before this lock was +/// acquired (and is therefore visible to the enumeration below, which runs +/// after it, in the same transaction) or is blocked until this transaction +/// commits (and then fails, since the entity is no longer active by then). +/// Enumerating via a plain pre-transaction pool query — the previous shape of +/// this code — could miss a session/credential created in that window, +/// leaving its cache entry uninvalidated indefinitely. See +/// `src/cache/mod.rs`'s consistency model. +pub async fn lock_entity_and_collect_revocation_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, + id: Uuid, +) -> Result<(Vec, Vec), AppError> { + let locked = + sqlx::query("SELECT id FROM entities WHERE id = $1 AND deleted_at IS NULL FOR UPDATE") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; - let Some(tenant_id) = tenant_id else { + if locked.is_none() { return Err(AppError::not_found(format!("entity {id} not found"))); - }; + } - let result = sqlx::query( + let session_ids: Vec = + sqlx::query_scalar("SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL") + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + // Restricted to the one credential kind this codebase caches under + // `CacheCategory::Credential` (certificates are tracked via the CRL + // instead, not this cache). + let credential_ids: Vec = sqlx::query_scalar( + "SELECT id FROM credentials WHERE entity_id = $1 AND status = 'active' AND kind = $2", + ) + .bind(id) + .bind(crate::models::enums::CredentialKind::AccessToken) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok((session_ids, credential_ids)) +} + +/// Finishes the entity soft-delete started by +/// [`lock_entity_and_collect_revocation_ids_in_tx`] in the same transaction: +/// flips the entity to inactive/tombstoned, revokes active credentials (all +/// kinds) and sessions, and tombstones the email. Does not commit — the +/// caller commits after this succeeds, once the cache barrier established on +/// the enumerated ids covers the whole transaction. +/// +/// Returns the entity's `tenant_id`, captured here (before the tombstone) +/// rather than left for the caller to re-derive post-commit: `get_entity` +/// filters `deleted_at IS NULL`, so a re-read after this commits always +/// misses the row and silently drops the tenant from the delete's audit +/// trail. +pub async fn deactivate_and_finish_entity_deletion_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + deleted_by: Option, + id: Uuid, +) -> Result, AppError> { + let tenant_id: Option = sqlx::query_scalar( "UPDATE entities SET status = 'inactive', deleted_at = now(), deleted_by = $2, updated_at = now() - WHERE id = $1 AND deleted_at IS NULL", + WHERE id = $1 AND deleted_at IS NULL + RETURNING tenant_id", ) .bind(id) .bind(deleted_by) - .execute(&mut *tx) + .fetch_optional(&mut **tx) .await - .map_err(db_err)?; - if result.rows_affected() == 0 { - return Err(AppError::not_found(format!("entity {id} not found"))); - } + .map_err(db_err)? + .ok_or_else(|| AppError::not_found(format!("entity {id} not found")))?; let revoked_certificates: i64 = sqlx::query_scalar( r#"WITH revoked AS ( @@ -751,17 +837,17 @@ pub async fn delete_entity_with_audit( SELECT COUNT(*) FILTER (WHERE kind = 'certificate') FROM revoked"#, ) .bind(id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if revoked_certificates > 0 { - crate::certs::repo::mark_crl_dirty_tx(&mut tx).await?; + crate::certs::repo::mark_crl_dirty_tx(tx).await?; } sqlx::query( "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -770,21 +856,19 @@ pub async fn delete_entity_with_audit( WHERE entity_id = $1 AND deleted_at IS NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("entity"), + target_kind: "entity", target_id: Some(id), event: "entity.delete", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; - Ok(()) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; + Ok(tenant_id) } pub async fn delete_entity( @@ -795,6 +879,58 @@ pub async fn delete_entity( delete_entity_with_audit(pool, false, None, id, deleted_by).await } +/// Soft-delete an entity: mark it inactive, set the tombstone, and immediately +/// cut off access by revoking its credentials and active sessions. Physical +/// removal is deferred to the purge cron. Hard delete (the old behavior) relied +/// on FK cascade for the credential/session cleanup, so the revocations are now +/// explicit. +/// +/// Used directly only when no cache is configured; the cache-aware path +/// (`identity::service::delete_entity`) calls +/// [`lock_entity_and_collect_revocation_ids_in_tx`] and +/// [`deactivate_and_finish_entity_deletion_in_tx`] itself so it can establish +/// the cache barrier between them. +pub async fn delete_entity_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + deleted_by: Option, +) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + lock_entity_and_collect_revocation_ids_in_tx(&mut tx, id).await?; + let tenant_id = deactivate_and_finish_entity_deletion_in_tx( + &mut tx, + events_enabled, + actor_id, + deleted_by, + id, + ) + .await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid delete) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation above via `observe_in_tx`. `tenant_id` is captured inside the + // transaction above (before the tombstone) rather than re-read here — a + // post-commit `get_entity` would always miss the now-deleted row. + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("entity"), + target_id: Some(id), + event: "entity.delete", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} + pub async fn restore_entity_with_audit( pool: &PgPool, events_enabled: bool, @@ -993,13 +1129,6 @@ pub async fn get_session(pool: &PgPool, id: Uuid) -> Result { }) } -pub async fn revoke_session(pool: &PgPool, id: Uuid) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; - revoke_session_in_tx(&mut tx, id).await?; - tx.commit().await.map_err(db_err)?; - Ok(()) -} - /// The caller owns the commit, so a logout can bind the session revocation and /// its `auth.logout` event into one transaction. pub async fn revoke_session_in_tx( @@ -1209,6 +1338,133 @@ pub async fn list_groups(pool: &PgPool, params: ListGroups) -> Result, + events_enabled: bool, + actor_id: Option, + id: Uuid, + req: UpdateGroup, + event_name: &str, + audit_details: Value, +) -> Result { + let attributes = req.attributes.map(normalize_attributes); + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let Some(tenant_id) = tenant_id else { + return Err(AppError::not_found(format!("group {id} not found"))); + }; + crate::tenants::repo::lock_optional_active_tenant(tx, tenant_id).await?; + let group = sqlx::query_as::<_, Group>( + r#"WITH p AS ( + UPDATE principal_groups + SET name = COALESCE($2, name), + description = COALESCE($3, description), + status = COALESCE($4, status), + attributes = COALESCE($5, attributes), + updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING id, name, tenant_id, 'principal'::text AS group_type, description, + (SELECT parent_id FROM principal_group_hierarchy WHERE child_id = principal_groups.id) AS parent_id, + status, attributes, deleted_at, deleted_by, created_at, updated_at + ), + o AS ( + UPDATE object_groups + SET name = COALESCE($2, name), + description = COALESCE($3, description), + status = COALESCE($4, status), + attributes = COALESCE($5, attributes), + updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING id, name, tenant_id, 'object'::text AS group_type, description, + (SELECT parent_id FROM object_group_hierarchy WHERE child_id = object_groups.id) AS parent_id, + status, attributes, deleted_at, deleted_by, created_at, updated_at + ) + SELECT * FROM p + UNION ALL + SELECT * FROM o"#, + ) + .bind(id) + .bind(req.name) + .bind(req.description) + .bind(req.status) + .bind(attributes) + .fetch_one(&mut **tx) + .await + .map_err(|e| match e { + sqlx::Error::RowNotFound => AppError::not_found(format!("group {id} not found")), + other => AppError::Database(other), + })?; + + let meta = crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: event_name, + }; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &audit_details).await?; + Ok(group) +} + +pub async fn update_group(pool: &PgPool, id: Uuid, req: UpdateGroup) -> Result { + update_group_with_audit( + pool, + false, + None, + id, + req, + "group.update", + serde_json::json!({}), + ) + .await +} + +pub async fn update_group_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + req: UpdateGroup, + event_name: &str, + audit_details: Value, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let group = update_group_in_tx( + &mut tx, + events_enabled, + actor_id, + id, + req, + event_name, + audit_details.clone(), + ) + .await?; + tx.commit().await.map_err(db_err)?; + // `update_group_in_tx` already enqueued the outbox row via `observe_in_tx` + // before returning — this is the post-commit stdout observability log + // `commit_with_observation` would otherwise provide; calling that helper + // instead would enqueue the outbox row a second time. + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: event_name, + }, + &audit_details, + ); + Ok(group) +} + pub async fn set_group_parent( pool: &PgPool, child_id: Uuid, @@ -1224,12 +1480,42 @@ pub async fn set_group_parent_with_audit( child_id: Uuid, parent_id: Uuid, ) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + set_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id, parent_id).await?; + tx.commit().await.map_err(db_err)?; + let group = get_group(pool, child_id).await?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(child_id), + event: "group.parent.set", + }, + &serde_json::json!({ "parent_id": parent_id }), + ); + Ok(group) +} + +/// Body of [`set_group_parent`] (minus its post-commit `get_group` read); +/// caller contract per `authz::repo::create_role_assignment_in_tx`. The +/// resolver must already hold `child_id`'s closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx` — +/// reparenting only changes what `child_id` and its descendants inherit from +/// above, never `group_hierarchy` rows below it, so that closure is exactly +/// what this mutation can affect. +pub(crate) async fn set_group_parent_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + child_id: Uuid, + parent_id: Uuid, +) -> Result<(), AppError> { if child_id == parent_id { return Err(AppError::bad_request("group cannot be its own parent")); } use sqlx::Row; - let mut tx = pool.begin().await.map_err(db_err)?; let child = sqlx::query( r#"SELECT tenant_id, group_type FROM groups @@ -1238,7 +1524,7 @@ pub async fn set_group_parent_with_audit( LIMIT 1"#, ) .bind(child_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("group {child_id} not found")), @@ -1252,7 +1538,7 @@ pub async fn set_group_parent_with_audit( LIMIT 1"#, ) .bind(parent_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => { @@ -1288,7 +1574,7 @@ pub async fn set_group_parent_with_audit( } else { "object_groups" }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, child_tenant_id).await?; + crate::tenants::repo::lock_optional_active_tenant(tx, child_tenant_id).await?; let lock_sql = format!( "SELECT id FROM {group_table} WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL @@ -1296,7 +1582,7 @@ pub async fn set_group_parent_with_audit( ); let locked_ids: Vec = sqlx::query_scalar(&lock_sql) .bind(vec![child_id, parent_id]) - .fetch_all(&mut *tx) + .fetch_all(&mut **tx) .await .map_err(db_err)?; if locked_ids.len() != 2 { @@ -1316,7 +1602,7 @@ pub async fn set_group_parent_with_audit( let creates_cycle: bool = sqlx::query_scalar(&creates_cycle_sql) .bind(parent_id) .bind(child_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if creates_cycle { @@ -1335,7 +1621,7 @@ pub async fn set_group_parent_with_audit( .bind(parent_id) .bind(child_id) .bind(child_tenant_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -1346,7 +1632,7 @@ pub async fn set_group_parent_with_audit( ORDER BY id FOR UPDATE"#, ) .bind(vec![child_id, parent_id]) - .fetch_all(&mut *tx) + .fetch_all(&mut **tx) .await .map_err(db_err)?; if principal_ids.len() == 2 { @@ -1361,13 +1647,12 @@ pub async fn set_group_parent_with_audit( .bind(parent_id) .bind(child_id) .bind(child_tenant_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } } - let group = fetch_group(&mut *tx, child_id).await?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id: child_tenant_id, @@ -1376,8 +1661,8 @@ pub async fn set_group_parent_with_audit( event: "group.parent.set", }; let details = serde_json::json!({ "parent_id": parent_id }); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(group) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(()) } pub async fn remove_group_parent(pool: &PgPool, child_id: Uuid) -> Result<(), AppError> { @@ -1391,28 +1676,52 @@ pub async fn remove_group_parent_with_audit( child_id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + let tenant_id = remove_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "group", + target_id: Some(child_id), + event: "group.parent.remove", + }, + &serde_json::json!({}), + ); + Ok(()) +} + +/// Body of [`remove_group_parent`]; caller contract per +/// [`set_group_parent_in_tx`]. Returns the group's `tenant_id` for the +/// caller's post-commit observability log. +pub(crate) async fn remove_group_parent_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + child_id: Uuid, +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { return Err(AppError::not_found(format!("group {child_id} not found"))); }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, tenant_id).await?; + crate::tenants::repo::lock_optional_active_tenant(tx, tenant_id).await?; let object_locked: Option = sqlx::query_scalar( "SELECT id FROM object_groups WHERE id = $1 AND deleted_at IS NULL FOR UPDATE", ) .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let principal_locked: Option = sqlx::query_scalar( "SELECT id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL FOR UPDATE", ) .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if object_locked.is_none() && principal_locked.is_none() { @@ -1425,7 +1734,7 @@ pub async fn remove_group_parent_with_audit( DELETE FROM object_group_hierarchy WHERE child_id = $1"#, ) .bind(child_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; let meta = crate::audit::AuditMeta { @@ -1436,8 +1745,8 @@ pub async fn remove_group_parent_with_audit( event: "group.parent.remove", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(tenant_id) } pub async fn list_child_groups( @@ -1462,105 +1771,48 @@ pub async fn list_child_groups( .await } -pub async fn update_group_with_audit( +pub async fn delete_group_with_audit( pool: &PgPool, events_enabled: bool, actor_id: Option, id: Uuid, - req: UpdateGroup, - event_name: &str, - audit_details: Value, -) -> Result { - let attributes = req.attributes.clone().map(normalize_attributes); + deleted_by: Option, +) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - let tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") - .bind(id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - let Some(tenant_id) = tenant_id else { - return Err(AppError::not_found(format!("group {id} not found"))); - }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, tenant_id).await?; - let group = sqlx::query_as::<_, Group>( - r#"WITH p AS ( - UPDATE principal_groups - SET name = COALESCE($2, name), - description = COALESCE($3, description), - status = COALESCE($4, status), - attributes = COALESCE($5, attributes), - updated_at = now() - WHERE id = $1 AND deleted_at IS NULL - RETURNING id, name, tenant_id, 'principal'::text AS group_type, description, - (SELECT parent_id FROM principal_group_hierarchy WHERE child_id = principal_groups.id) AS parent_id, - status, attributes, deleted_at, deleted_by, created_at, updated_at - ), - o AS ( - UPDATE object_groups - SET name = COALESCE($2, name), - description = COALESCE($3, description), - status = COALESCE($4, status), - attributes = COALESCE($5, attributes), - updated_at = now() - WHERE id = $1 AND deleted_at IS NULL - RETURNING id, name, tenant_id, 'object'::text AS group_type, description, - (SELECT parent_id FROM object_group_hierarchy WHERE child_id = object_groups.id) AS parent_id, - status, attributes, deleted_at, deleted_by, created_at, updated_at - ) - SELECT * FROM p - UNION ALL - SELECT * FROM o"#, - ) - .bind(id) - .bind(req.name) - .bind(req.description) - .bind(req.status) - .bind(attributes) - .fetch_one(&mut *tx) - .await - .map_err(|e| match e { - sqlx::Error::RowNotFound => AppError::not_found(format!("group {id} not found")), - other => AppError::Database(other), - })?; - - let meta = crate::audit::AuditMeta { - actor_entity_id: actor_id, - tenant_id: group.tenant_id, - target_kind: "group", - target_id: Some(id), - event: event_name, - }; - let details = audit_details; - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(group) -} - -pub async fn update_group(pool: &PgPool, id: Uuid, req: UpdateGroup) -> Result { - update_group_with_audit( - pool, - false, - None, - id, - req, - "group.update", - serde_json::json!({}), - ) - .await + let tenant_id = delete_group_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.delete", + }, + &serde_json::json!({}), + ); + Ok(()) } -pub async fn delete_group_with_audit( - pool: &PgPool, +/// Body of [`delete_group`]; caller contract per +/// `authz::repo::create_role_assignment_in_tx`. The resolver must already +/// hold this group's closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx` — +/// a soft delete only sets `deleted_at`, leaving `group_hierarchy` untouched, +/// so the closure is unaffected by this mutation's own effect. Returns the +/// group's `tenant_id` for the caller's post-commit observability log — a +/// re-read after this commits would always miss the now-deleted row. +pub(crate) async fn delete_group_in_tx( + tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, deleted_by: Option, -) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { @@ -1582,7 +1834,7 @@ pub async fn delete_group_with_audit( ) .bind(id) .bind(deleted_by) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if result.is_none() { @@ -1597,8 +1849,8 @@ pub async fn delete_group_with_audit( event: "group.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(tenant_id) } pub async fn delete_group( @@ -1616,9 +1868,43 @@ pub async fn restore_group_with_audit( id: Uuid, restored_by: Option, ) -> Result<(), AppError> { - let _ = restored_by; let mut tx = pool.begin().await.map_err(db_err)?; + restore_group_in_tx(&mut tx, events_enabled, actor_id, id, restored_by).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation inside `restore_group_in_tx` via `observe_in_tx`. + let tenant_id = get_group(pool, id).await.ok().and_then(|g| g.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("group"), + target_id: Some(id), + event: "group.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} +/// Body of [`restore_group`]; caller contract per +/// `authz::repo::create_role_assignment_in_tx`. The resolver must already +/// hold this group's closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx`. +pub(crate) async fn restore_group_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result<(), AppError> { + let _ = restored_by; let tenant_info: Option<(Option, bool)> = sqlx::query_as( "SELECT g.tenant_id, (t.deleted_at IS NOT NULL) FROM groups g @@ -1626,7 +1912,7 @@ pub async fn restore_group_with_audit( WHERE g.id = $1 AND g.deleted_at IS NOT NULL", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let (tenant_id, _is_tenant_deleted) = match tenant_info { @@ -1657,20 +1943,18 @@ pub async fn restore_group_with_audit( SELECT id FROM o"#, ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(restore_conflict)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("group"), + target_kind: "group", target_id: Some(id), event: "group.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; Ok(()) } @@ -1838,6 +2122,13 @@ pub async fn add_group_member_with_audit( Ok(()) } +/// Takes the group's `principal_groups` row lock before deleting, matching +/// [`add_group_member`]. `authz::repo::lock_group_closures_and_collect_member_ids` +/// documents this lock as the reason a group-subject mutation may enumerate +/// members under its own closure lock and trust the result: without it a +/// removal is not serialized against that enumeration. Unlike `add_group_member` +/// this deliberately does not require the group to be active or live — a +/// membership must stay removable from a suspended or soft-deleted group. pub async fn remove_group_member( pool: &PgPool, group_id: Uuid, @@ -1857,13 +2148,19 @@ pub async fn remove_group_member_with_audit( // Removal stays idempotent: a missing group or a missing membership row is // not an error. But only an actual deletion is a domain event — publishing // `group_member.remove` for a no-op would lie to downstream consumers. - let tenant_id: Option> = sqlx::query_scalar( - "SELECT tenant_id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(group_id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; + // + // Deliberately no `deleted_at IS NULL` filter, unlike `add_group_member` + // — a membership must stay removable from a suspended or soft-deleted + // group. Takes the group's row lock, matching `add_group_member`'s — + // `authz::repo::lock_group_closures_and_collect_member_ids` documents + // this lock as the reason a group-subject mutation may enumerate members + // under its own closure lock and trust the result. + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM principal_groups WHERE id = $1 FOR UPDATE") + .bind(group_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; let Some(tenant_id) = tenant_id else { return Ok(()); }; diff --git a/src/identity/service.rs b/src/identity/service.rs index efd9f2c..13c9b6a 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -727,6 +727,7 @@ pub async fn request_password_reset( pub async fn reset_password( pool: &PgPool, + cache: Option<&crate::cache::CacheClient>, req: PasswordResetConfirmRequest, ) -> Result<(), AppError> { if let Some(confirm_password) = req.confirm_password.as_deref() { @@ -780,11 +781,56 @@ pub async fn reset_password( { return Err(AppError::bad_request("invalid password reset token")); } + + // Every active session is about to be bulk-revoked below — enumerated + // *inside* the transaction, right after the entity lock: `create_session` + // takes that same lock, so a concurrently created session either + // committed before we acquired it (visible here) or is blocked until we + // finish. A pre-transaction pool query could miss one, leaving its cache + // entry uninvalidated indefinitely. See `src/cache/mod.rs`'s consistency + // model. + let session_keys: Vec = sqlx::query_scalar::<_, Uuid>( + "SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL", + ) + .bind(entity_id) + .fetch_all(&mut *tx) + .await + .map_err(db_err)? + .into_iter() + .map(crate::cache::keys::session) + .collect(); + + if let Some(cache) = cache { + cache + .begin(crate::cache::CacheCategory::Session, &session_keys) + .await?; + } + let outcome = + finish_password_reset_in_tx(&mut tx, token_id, entity_id, &email, password_hash).await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(db_err), + Err(err) => Err(err), + }; + if let Some(cache) = cache { + cache + .end(crate::cache::CacheCategory::Session, &session_keys) + .await; + } + outcome +} + +async fn finish_password_reset_in_tx( + tx: &mut Transaction<'_, Postgres>, + token_id: Uuid, + entity_id: Uuid, + email: &str, + password_hash: String, +) -> Result<(), AppError> { let updated = sqlx::query( "UPDATE password_reset_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", ) .bind(token_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; if updated.rows_affected() == 0 { @@ -796,7 +842,7 @@ pub async fn reset_password( WHERE entity_id = $1 AND kind = 'password' AND status = 'active'"#, ) .bind(entity_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; sqlx::query( @@ -808,17 +854,16 @@ pub async fn reset_password( .bind(CredentialKind::Password) .bind(email) .bind(password_hash) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; sqlx::query( "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", ) .bind(entity_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; - tx.commit().await.map_err(db_err)?; Ok(()) } @@ -992,7 +1037,7 @@ pub async fn oauth_exchange( pub async fn refresh_session( pool: &PgPool, cfg: &Config, - primary_key: &LoadedKey, + signer: &crate::auth::JwtSigner, entity_id: Uuid, session_id: Uuid, ) -> Result { @@ -1004,11 +1049,11 @@ pub async fn refresh_session( let session = super::repo::refresh_session_in_tx(&mut tx, session_id, entity_id, cfg.jwt_expiry_secs) .await?; - let token = encode_jwt( + let token = crate::auth::encode_jwt_with( entity_id, session.id, tenant_id, - primary_key, + signer, cfg.jwt_expiry_secs, &cfg.jwt_issuer, &cfg.jwt_audience, @@ -2236,6 +2281,105 @@ fn make_shared_key(cred_id: Uuid) -> String { ) } +/// Soft-deletes an entity with a cache barrier spanning the whole +/// transaction. The one entry point for entity deletion on every caller +/// (REST and GraphQL alike), so no caller can under-invalidate. +/// +/// `entity_status` invalidation alone is *not* sufficient: this also revokes +/// the entity's sessions and access-token credentials, which +/// [`super::repo::restore_entity`] deliberately never reinstates. Without their own +/// invalidation, a stale cached session or credential becomes a full hit +/// again the moment a later restore repopulates `entity_status` as active — +/// despite staying revoked in Postgres. See +/// [`super::repo::lock_entity_and_collect_revocation_ids_in_tx`] for why the +/// ids are enumerated inside the same locked transaction. +pub async fn delete_entity( + pool: &PgPool, + cache: Option<&crate::cache::CacheClient>, + events_enabled: bool, + id: Uuid, + deleted_by: Option, +) -> Result<(), AppError> { + let Some(cache) = cache else { + return super::repo::delete_entity_with_audit( + pool, + events_enabled, + deleted_by, + id, + deleted_by, + ) + .await; + }; + + let mut tx = pool.begin().await.map_err(db_err)?; + // Lock and enumerate *before* establishing the barrier: the entity is + // still fully active at this point, so a concurrent cache read has + // nothing dirty to react to yet. Only once the barrier below is up does + // a concurrent read stop trusting the (about-to-be-stale) cache entries + // and fall through to Postgres — flipping status first would leave a + // window where a concurrent request keeps taking full cache hits on the + // pre-delete entity/session/credential entries and runs past the point + // this delete commits. + let (session_ids, credential_ids) = + super::repo::lock_entity_and_collect_revocation_ids_in_tx(&mut tx, id).await?; + let session_keys: Vec = session_ids + .iter() + .copied() + .map(crate::cache::keys::session) + .collect(); + let credential_keys: Vec = credential_ids + .iter() + .copied() + .map(crate::cache::keys::credential) + .collect(); + let entity_status_keys = [crate::cache::keys::entity_status(id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 3] = [ + ( + crate::cache::CacheCategory::EntityStatus, + &entity_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + (crate::cache::CacheCategory::Credential, &credential_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = super::repo::deactivate_and_finish_entity_deletion_in_tx( + &mut tx, + events_enabled, + deleted_by, + deleted_by, + id, + ) + .await; + let outcome = match outcome { + Ok(tenant_id) => tx.commit().await.map_err(db_err).map(|_| tenant_id), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + let tenant_id = outcome?; + // Mirrors `delete_entity_with_audit`'s own post-commit audit write + // (fire-and-forget, after the mutation durably commits) — see + // `audit::commit_with_audit`'s doc comment. Only needed on this locked + // path; the cache-disabled fallback above already gets it from + // `delete_entity_with_audit` itself. `tenant_id` comes from the captured + // value inside the transaction — a post-commit `get_entity` would always + // miss the now-deleted row. + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: deleted_by, + tenant_id, + target_kind: Some("entity"), + target_id: Some(id), + event: "entity.delete", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} + pub async fn revoke_credential( pool: &PgPool, entity_id: Uuid, diff --git a/src/lib.rs b/src/lib.rs index 5da2922..24fce49 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod audit; pub mod auth; pub mod authz; pub mod build_info; +pub mod cache; pub mod certs; pub mod config; pub mod crypto; diff --git a/src/main.rs b/src/main.rs index 06916f4..d71a1ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use anyhow::Context; use atom::{ - audit, certs, config, db, events, grpc, identity, keys, metrics, purge, routes, + audit, cache, certs, config, db, events, grpc, identity, keys, metrics, purge, routes, state::{self, GrpcRuntimeStatus}, }; use tracing_subscriber::EnvFilter; @@ -44,7 +44,9 @@ async fn main() -> anyhow::Result<()> { // startup instead of leaving HTTP up with a permanently failing gRPC task. let grpc_tls = grpc::load_tls_config(&cfg).await?; - let mut state = state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer); + let cache = init_cache(&cfg.cache).await?; + + let mut state = state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer, cache); if cfg.events.enabled() { let publisher = events::publisher::AmqpPublisher::connect(&cfg.events) .await @@ -97,6 +99,42 @@ async fn main() -> anyhow::Result<()> { Ok(()) } +/// Builds the Redis-backed cache when enabled. +/// +/// `None` means "caching is not configured", and every mutation guard becomes +/// a pure pass-through on that basis. So an *enabled* cache that merely can't +/// reach Redis right now must never degrade to `None`: this process would go +/// on mutating grants, sessions, and credentials without invalidating entries +/// that other replicas are still serving, and a revoke here would stay +/// authorized there. Unreachable Redis is a runtime condition, not a +/// configuration one — the client is retained either way, and its own +/// behavior covers the outage: reads fall through to Postgres as misses, +/// while `begin` fails and so refuses security-sensitive mutations until +/// Redis returns (see `src/cache/mod.rs` and `src/cache/invalidate.rs`). +/// +/// A *build* failure is fatal regardless: an unparseable URL cannot recover. +/// `ATOM_CACHE_FAIL_FAST_ON_STARTUP` then decides whether an unreachable +/// Redis should also abort startup, rather than boot into the refusing state. +async fn init_cache(cfg: &config::CacheConfig) -> anyhow::Result> { + if !cfg.enabled { + return Ok(None); + } + let client = cache::CacheClient::build(cfg).context("cache configuration is invalid")?; + match client.probe(cfg.connect_timeout_ms).await { + Ok(()) => tracing::info!("cache enabled; connected to Redis"), + Err(err) if cfg.fail_fast_on_startup => { + return Err( + err.context("cache connect failed and ATOM_CACHE_FAIL_FAST_ON_STARTUP=true") + ); + } + Err(err) => tracing::error!( + "cache enabled but Redis is unreachable: {err}. Reads fall through to Postgres; \ + security-sensitive mutations are refused until Redis recovers." + ), + } + Ok(Some(client)) +} + fn init_tracing(logging: &config::LoggingConfig) -> anyhow::Result<()> { let filter = EnvFilter::try_new(&logging.level) .context("ATOM_LOG_LEVEL/RUST_LOG must be a valid tracing filter")?; diff --git a/src/metrics.rs b/src/metrics.rs index f99137f..1e6019e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -36,6 +36,18 @@ pub const EVENT_OUTBOX_PUBLISH_FAILURES: &str = "atom_event_outbox_publish_failu pub const EVENT_OUTBOX_EXHAUSTED: &str = "atom_event_outbox_exhausted_total"; /// Gauge of DB pool connections, labelled by `state` (total|idle). pub const DB_POOL_CONNECTIONS: &str = "atom_db_pool_connections"; +/// Counter of cache reads, labelled by `category` (session|entity_status| +/// tenant_status|credential|credential_ceiling|grants) and `outcome` +/// (hit|miss|error). +pub const CACHE_LOOKUP: &str = "atom_cache_lookup_total"; +/// Counter of cache invalidation/barrier operations, labelled by `category` +/// and `outcome` (ok|error). +pub const CACHE_INVALIDATION: &str = "atom_cache_invalidation_total"; +/// Counter of post-miss cache writes, labelled by `category` and `outcome` +/// (applied|stale|error). A hit rate stuck at zero is ambiguous without this: +/// `stale` distinguishes "every write is being rejected because a barrier is +/// stuck dirty or the version keeps moving" from a merely cold cache. +pub const CACHE_POPULATE: &str = "atom_cache_populate_total"; #[cfg(feature = "metrics")] mod backend { @@ -102,6 +114,20 @@ mod backend { pub fn record_outbox_exhausted() { metrics::counter!(EVENT_OUTBOX_EXHAUSTED).increment(1); } + + pub fn record_cache_lookup(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_LOOKUP, "category" => category, "outcome" => outcome).increment(1); + } + + pub fn record_cache_invalidation(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_INVALIDATION, "category" => category, "outcome" => outcome) + .increment(1); + } + + pub fn record_cache_populate(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_POPULATE, "category" => category, "outcome" => outcome) + .increment(1); + } } #[cfg(not(feature = "metrics"))] @@ -130,9 +156,16 @@ mod backend { pub fn record_outbox_publish_failure(_rows: u64) {} #[inline] pub fn record_outbox_exhausted() {} + #[inline] + pub fn record_cache_lookup(_category: &'static str, _outcome: &'static str) {} + #[inline] + pub fn record_cache_invalidation(_category: &'static str, _outcome: &'static str) {} + #[inline] + pub fn record_cache_populate(_category: &'static str, _outcome: &'static str) {} } pub use backend::{ - enabled, init, record_audit_db_suppressed, record_audit_failure, record_decision, - record_outbox_exhausted, record_outbox_publish_failure, record_rate_limit_rejection, render, + enabled, init, record_audit_db_suppressed, record_audit_failure, record_cache_invalidation, + record_cache_lookup, record_cache_populate, record_decision, record_outbox_exhausted, + record_outbox_publish_failure, record_rate_limit_rejection, render, }; diff --git a/src/routes.rs b/src/routes.rs index 336bb3d..7fbe211 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -380,6 +380,7 @@ mod tests { standby: None, }, None, + None, ) } } diff --git a/src/state.rs b/src/state.rs index 772a4d8..c199b3b 100644 --- a/src/state.rs +++ b/src/state.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use tokio::sync::RwLock; use crate::{ - certs::service::CertificateIssuer, config::Config, events::publisher::EventPublisher, - keys::ActiveKeys, rate_limit::RateLimiter, + cache::CacheClient, certs::service::CertificateIssuer, config::Config, + events::publisher::EventPublisher, keys::ActiveKeys, rate_limit::RateLimiter, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -62,6 +62,11 @@ pub struct AppState { /// Set via [`AppState::with_event_publisher`], since connecting to a /// broker is an async operation `AppState::new` (sync) can't perform. pub event_publisher: Option>, + /// `None` when caching is disabled or unconfigured — every call site + /// checks this and falls through to Postgres, so the service runs + /// cache-free with zero behavior change (this is what keeps `cargo test` + /// and local dev working without Redis). + pub cache: Option>, grpc_status: Arc>, } @@ -71,6 +76,7 @@ impl AppState { config: Config, keys: ActiveKeys, certificate_issuer: Option, + cache: Option, ) -> Self { let grpc_status = GrpcRuntimeStatus::starting(config.grpc_addr.clone()); AppState { @@ -80,6 +86,7 @@ impl AppState { certificate_issuer: certificate_issuer.map(Arc::new), rate_limiter: Arc::new(RateLimiter::default()), event_publisher: None, + cache: cache.map(Arc::new), grpc_status: Arc::new(RwLock::new(grpc_status)), } } diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index 99816f9..2975a09 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -547,15 +547,68 @@ pub async fn update_tenant( update_tenant_with_audit(pool, false, None, id, req, updated_by).await } -pub async fn soft_delete_tenant_with_audit( - pool: &PgPool, +/// Locks the tenant row and, in the *same* transaction, enumerates the exact +/// active session ids of its member entities that a subsequent delete is +/// about to revoke — *before* the tenant is touched. Callers establish the +/// cache barrier on these ids (plus the tenant's own `tenant_status` key) +/// before calling [`deactivate_and_finish_tenant_soft_delete_in_tx`]: starting +/// the barrier only after the status flip would leave a window where a +/// concurrent request can still take a full cache hit on the pre-delete +/// tenant-status/session entries and keep running past the point the delete +/// commits. +/// +/// The `SELECT ... FOR UPDATE` below takes the same exclusive row lock on the +/// tenant that `lock_active_tenant`/`lock_optional_active_tenant` take +/// (transitively, via `lock_active_entity`) before any session or credential +/// can be created for an entity in this tenant. So a session created +/// concurrently for a member entity either committed before this lock was +/// acquired (and is therefore visible to the enumeration below, which runs +/// after it, in the same transaction) or is blocked until this transaction +/// commits (and then fails, since the tenant is no longer active by then). +/// Enumerating via a plain pre-transaction pool query — the previous shape of +/// this code — could miss a session created in that window, leaving its +/// cache entry uninvalidated indefinitely. See `src/cache/mod.rs`'s +/// consistency model. +pub async fn lock_tenant_and_collect_session_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, + id: Uuid, +) -> Result, AppError> { + let locked = + sqlx::query("SELECT id FROM tenants WHERE id = $1 AND deleted_at IS NULL FOR UPDATE") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + if locked.is_none() { + return Err(AppError::not_found(format!("tenant {id} not found"))); + } + + let session_ids: Vec = sqlx::query_scalar( + r#"SELECT id FROM sessions + WHERE revoked_at IS NULL + AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)"#, + ) + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok(session_ids) +} + +/// Finishes the tenant soft-delete started by +/// [`lock_tenant_and_collect_session_ids_in_tx`] in the same transaction: +/// flips the tenant to `deleted`, and revokes every active credential and +/// session belonging to the tenant's entities. Does not commit — the caller +/// commits after this succeeds, once the cache barrier established on the +/// enumerated session ids covers the whole transaction. +pub async fn deactivate_and_finish_tenant_soft_delete_in_tx( + tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, - id: Uuid, deleted_by: Option, + id: Uuid, ) -> Result { - let mut tx = pool.begin().await.map_err(db_err)?; - let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = 'deleted', deleted_at = now(), deleted_by = $2, @@ -565,13 +618,17 @@ pub async fn soft_delete_tenant_with_audit( )) .bind(id) .bind(deleted_by) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), other => AppError::Database(other), })?; + // Stamp the tenant-delete marker on every revoked credential (not just + // certificates) so restore_tenant can reverse exactly the revocations this + // delete caused, without disturbing credentials revoked earlier for other + // reasons. let revoked_certificates: i64 = sqlx::query_scalar( r#"WITH revoked AS ( UPDATE credentials c @@ -589,11 +646,11 @@ pub async fn soft_delete_tenant_with_audit( SELECT COUNT(*) FROM revoked WHERE kind = 'certificate'"#, ) .bind(id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if revoked_certificates > 0 { - crate::certs::repo::mark_crl_dirty_tx(&mut tx).await?; + crate::certs::repo::mark_crl_dirty_tx(tx).await?; } sqlx::query( @@ -602,7 +659,7 @@ pub async fn soft_delete_tenant_with_audit( AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -614,10 +671,14 @@ pub async fn soft_delete_tenant_with_audit( event: "tenant.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(tenant) } +/// Soft-delete a tenant: mark `status = deleted`, stamp the tombstone, and +/// immediately revoke every active credential and session of entities in the +/// tenant. Physical removal (and the entity cascade) is deferred to the purge +/// cron. pub async fn soft_delete_tenant( pool: &PgPool, id: Uuid, @@ -626,15 +687,84 @@ pub async fn soft_delete_tenant( soft_delete_tenant_with_audit(pool, false, None, id, deleted_by).await } -pub async fn restore_tenant_with_audit( +/// Used directly only when no cache is configured; the cache-aware path +/// (`graphql::tenants::delete_tenant`) calls +/// [`lock_tenant_and_collect_session_ids_in_tx`] and +/// [`deactivate_and_finish_tenant_soft_delete_in_tx`] itself so it can +/// establish the cache barrier between them. +pub async fn soft_delete_tenant_with_audit( pool: &PgPool, events_enabled: bool, actor_id: Option, id: Uuid, - restored_by: Option, + deleted_by: Option, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; + lock_tenant_and_collect_session_ids_in_tx(&mut tx, id).await?; + let tenant = deactivate_and_finish_tenant_soft_delete_in_tx( + &mut tx, + events_enabled, + actor_id, + deleted_by, + id, + ) + .await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: "tenant", + target_id: Some(id), + event: "tenant.delete", + }, + &serde_json::json!({}), + ); + Ok(tenant) +} +/// Reverse a tenant soft delete within the retention window. Reactivates the +/// tenant and clears its tombstone; its children (entities, groups, roles, +/// resources) were never individually tombstoned by `soft_delete_tenant` — they +/// were hidden only via the tenant's `deleted_at` — so they become visible again +/// automatically. +/// +/// To make the restored tenant operational, the non-certificate child +/// credentials (passwords, API keys) that *this* delete revoked — identified by +/// the `tenant_deleted` revocation marker — are reactivated, so members can log +/// in with their existing secrets. Certificates stay revoked (their revocation +/// is published via the CRL and cannot be safely undone — re-issue is required), +/// and sessions stay revoked, so a fresh login is required. Credentials revoked +/// earlier for other reasons (e.g. an individually soft-deleted child) are left +/// untouched. +/// +/// Fails with a conflict if the tenant name/alias was re-taken by a live tenant +/// during the retention window. +/// +/// Flips the tenant back to `active` and, in the *same* transaction, +/// enumerates the exact credential ids [`finish_tenant_restore_in_tx`] is +/// about to reactivate — mirrors that function's `UPDATE credentials` `WHERE` +/// clause precisely, so callers can invalidate `atom:v1:credential:*` cache +/// entries for them (see `src/cache/mod.rs`'s consistency model). A stale +/// cached "revoked" credential is a false-deny (fails closed, not a security +/// hole) but is still worth fixing: unlike a tenant or entity status flip, +/// which a later-checked fresh field can catch regardless of earlier stale +/// fields, `verify_api_key_snapshot` checks the credential's own status first +/// — a stale value there is never overridden by anything checked afterward. +/// +/// The status-flip `UPDATE` below takes an exclusive row lock on the tenant, +/// so the enumeration that follows it (in the same transaction) sees a +/// consistent snapshot with respect to any concurrent restore/purge of the +/// same tenant — mirroring the same lock-then-enumerate shape used by +/// [`lock_tenant_and_collect_session_ids_in_tx`], even though (unlike +/// that case) no *new* matching credential can appear here: only +/// `soft_delete_tenant` ever stamps the `tenant_deleted` revocation reason, +/// and it cannot run again against an already-deleted tenant. +pub async fn reactivate_tenant_and_collect_credential_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, + id: Uuid, + restored_by: Option, +) -> Result<(Tenant, Vec), AppError> { let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = 'active', deleted_at = NULL, deleted_by = NULL, @@ -644,7 +774,7 @@ pub async fn restore_tenant_with_audit( )) .bind(id) .bind(restored_by) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => { @@ -653,6 +783,35 @@ pub async fn restore_tenant_with_audit( other => restore_conflict(other), })?; + let credential_ids: Vec = sqlx::query_scalar( + r#"SELECT c.id + FROM credentials c + JOIN entities e ON c.entity_id = e.id + WHERE e.tenant_id = $1 + AND e.deleted_at IS NULL + AND c.status = 'revoked' + AND c.kind <> 'certificate' + AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, + ) + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok((tenant, credential_ids)) +} + +/// Finishes the tenant restore started by +/// [`reactivate_tenant_and_collect_credential_ids_in_tx`] in the same +/// transaction: reactivates exactly the credentials it enumerated. Does not +/// commit — the caller commits after this succeeds, once the cache barrier +/// established on those credential ids covers the whole transaction. +pub async fn finish_tenant_restore_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { sqlx::query( r#"UPDATE credentials c SET status = 'active', @@ -666,21 +825,19 @@ pub async fn restore_tenant_with_audit( AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id: Some(id), - target_kind: Some("tenant"), + target_kind: "tenant", target_id: Some(id), event: "tenant.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; - Ok(tenant) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; + Ok(()) } pub async fn restore_tenant( @@ -691,6 +848,44 @@ pub async fn restore_tenant( restore_tenant_with_audit(pool, false, None, id, restored_by).await } +/// Used directly only when no cache is configured; the cache-aware path +/// (`graphql::tenants::restore_tenant`) calls +/// [`reactivate_tenant_and_collect_credential_ids_in_tx`] and +/// [`finish_tenant_restore_in_tx`] itself so it can establish the cache +/// barrier between them. +pub async fn restore_tenant_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let (tenant, _credential_ids) = + reactivate_tenant_and_collect_credential_ids_in_tx(&mut tx, id, restored_by).await?; + finish_tenant_restore_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation above via `observe_in_tx`. + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: Some("tenant"), + target_id: Some(id), + event: "tenant.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(tenant) +} + pub(crate) async fn tenant_purge_object_ids( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, tenant_ids: &[Uuid], @@ -770,13 +965,48 @@ pub async fn change_tenant_status_with_audit( id: Uuid, status: TenantStatus, event_name: &str, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let tenant = + change_tenant_status_in_tx(&mut tx, events_enabled, actor_id, id, status, event_name) + .await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: "tenant", + target_id: Some(id), + event: event_name, + }, + &serde_json::json!({ "status": tenant.status }), + ); + Ok(tenant) +} + +/// Body of [`change_tenant_status_with_audit`]. When `status != Active`, this +/// also bulk-revokes the tenant's members' sessions — a cache-aware caller +/// must already have enumerated those session ids via +/// [`lock_tenant_and_collect_session_ids_in_tx`] and established the barrier +/// on them (alongside `tenant_status`) before calling this, the same way +/// `deleteTenant` does: a barrier put up only after this commits would leave +/// a window where a concurrent read still takes a full cache hit on a +/// session this call is about to revoke, which then survives (with +/// `revoked_at = None`) until the tenant is re-enabled and its own +/// `tenant_status` cache entry stops masking the stale session. +pub(crate) async fn change_tenant_status_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + status: TenantStatus, + event_name: &str, ) -> Result { if status == TenantStatus::Deleted { return Err(AppError::bad_request( "use delete tenant to apply the soft-delete lifecycle", )); } - let mut tx = pool.begin().await.map_err(db_err)?; let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = $2, updated_by = $3, updated_at = now() @@ -786,7 +1016,7 @@ pub async fn change_tenant_status_with_audit( .bind(id) .bind(&status) .bind(actor_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), @@ -800,7 +1030,7 @@ pub async fn change_tenant_status_with_audit( AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } @@ -813,7 +1043,7 @@ pub async fn change_tenant_status_with_audit( event: event_name, }; let details = serde_json::json!({ "status": tenant.status }); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(tenant) } @@ -1223,7 +1453,6 @@ pub async fn add_tenant_member_with_audit( let mut role_assigned = false; if let Some(role_id) = role_id { role_assigned = crate::authz::repo::create_role_assignment_if_missing_in_tx( - pool, &mut tx, &CreateRoleAssignment { tenant_id: Some(tenant_id), @@ -1328,7 +1557,7 @@ pub async fn accept_invitation( ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; let role_id = accept_invitation_row(&mut tx, tenant_id, invitee_user_id).await?; - grant_invitation_role(&mut tx, pool, tenant_id, invitee_user_id, role_id).await?; + grant_invitation_role(&mut tx, tenant_id, invitee_user_id, role_id).await?; tx.commit().await.map_err(db_err)?; Ok(()) } @@ -1399,7 +1628,7 @@ pub async fn accept_invitation_token( .await .map_err(db_err)?; - grant_invitation_role(&mut tx, pool, tenant_id, actor_id, role_id).await?; + grant_invitation_role(&mut tx, tenant_id, actor_id, role_id).await?; tx.commit().await.map_err(db_err)?; Ok(tenant_id) } @@ -1435,7 +1664,6 @@ async fn accept_invitation_row( async fn grant_invitation_role( tx: &mut Transaction<'_, Postgres>, - pool: &PgPool, tenant_id: Uuid, invitee_user_id: Uuid, role_id: Option, @@ -1460,7 +1688,6 @@ async fn grant_invitation_role( }; crate::authz::repo::create_role_assignment_if_missing_in_tx( - pool, tx, &CreateRoleAssignment { tenant_id: Some(tenant_id), diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b588ea6..2b3f245 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,9 +6,17 @@ //! ```bash //! DATABASE_URL=postgres://... cargo test -- --ignored //! ``` +//! +//! Cache-invalidation tests additionally require a reachable Redis at +//! `ATOM_TEST_REDIS_URL`, following the same `#[ignore]` convention: +//! +//! ```bash +//! DATABASE_URL=postgres://... ATOM_TEST_REDIS_URL=redis://... cargo test -- --ignored +//! ``` #![allow(dead_code)] +use atom::{cache::CacheClient, config::CacheConfig}; use sqlx::PgPool; /// Connect to the test database and run all migrations. @@ -26,6 +34,27 @@ pub async fn pool() -> PgPool { pool } +/// Connect to the test Redis with `CacheConfig::default`'s production TTLs, +/// overriding only `enabled`/`redis_url`. Deliberately *not* shortened: every +/// invalidation-correctness test asserts immediately, with no sleep, so a +/// short TTL could only mask a missing invalidation as a pass. +/// +/// Assumes Redis is flushed between test binaries (see the `run_one` helper +/// in `.github/workflows/rust.yml`) — entries keyed off fixed ids such as the +/// seeded admin's would otherwise outlive the database they describe. +pub async fn cache_client() -> CacheClient { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + let cfg = CacheConfig { + enabled: true, + redis_url: url, + ..CacheConfig::default() + }; + CacheClient::connect(&cfg) + .await + .expect("connect to test redis") +} + /// Well-known seeded admin entity. pub fn admin_id() -> uuid::Uuid { "00000000-0000-0000-0000-000000000001".parse().unwrap() diff --git a/tests/m10_graphql_profiles.rs b/tests/m10_graphql_profiles.rs index a5d6ca1..969a034 100644 --- a/tests/m10_graphql_profiles.rs +++ b/tests/m10_graphql_profiles.rs @@ -80,6 +80,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } diff --git a/tests/m11_graphql_primitives.rs b/tests/m11_graphql_primitives.rs index 42210b9..534ee23 100644 --- a/tests/m11_graphql_primitives.rs +++ b/tests/m11_graphql_primitives.rs @@ -32,7 +32,7 @@ async fn state(pool: PgPool) -> AppState { let active_keys = keys::load_active_keys(&pool, &config.signing_keys) .await .expect("load signing keys"); - AppState::new(pool, config, active_keys, None) + AppState::new(pool, config, active_keys, None, None) } fn authed(query: impl Into) -> Request { diff --git a/tests/m12_graphql_identity.rs b/tests/m12_graphql_identity.rs index 2b0ee30..ccbb6fc 100644 --- a/tests/m12_graphql_identity.rs +++ b/tests/m12_graphql_identity.rs @@ -37,6 +37,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } @@ -81,6 +82,7 @@ fn authed_scoped_with_credential( credential_id: Some(credential_id), scoped: true, ceiling: Some(std::sync::Arc::new(ceiling)), + cache: None, }) } diff --git a/tests/m13_graphql_authz_admin.rs b/tests/m13_graphql_authz_admin.rs index 8c852d3..5e7a946 100644 --- a/tests/m13_graphql_authz_admin.rs +++ b/tests/m13_graphql_authz_admin.rs @@ -36,6 +36,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } diff --git a/tests/m14_api_endpoints.rs b/tests/m14_api_endpoints.rs index 83841b5..cf8b50e 100644 --- a/tests/m14_api_endpoints.rs +++ b/tests/m14_api_endpoints.rs @@ -30,7 +30,7 @@ use uuid::Uuid; fn state(pool: PgPool, keys: ActiveKeys) -> AppState { let config = Config::for_tests(); - AppState::new(pool, config, keys, None) + AppState::new(pool, config, keys, None, None) } async fn active_keys(pool: &PgPool) -> ActiveKeys { diff --git a/tests/m17_certificates.rs b/tests/m17_certificates.rs index ae18277..0fb4efb 100644 --- a/tests/m17_certificates.rs +++ b/tests/m17_certificates.rs @@ -126,6 +126,7 @@ fn state( standby: None, }, certificate_issuer, + None, ) } diff --git a/tests/m21_soft_delete.rs b/tests/m21_soft_delete.rs index dea6cc1..b475c7e 100644 --- a/tests/m21_soft_delete.rs +++ b/tests/m21_soft_delete.rs @@ -230,6 +230,7 @@ async fn deleted_entity_cannot_consume_existing_password_reset_token() { assert!( service::reset_password( &pool, + None, PasswordResetConfirmRequest { token, password: "replacement-password".to_string(), diff --git a/tests/m23_authenticate_credential.rs b/tests/m23_authenticate_credential.rs index f08409d..a629b22 100644 --- a/tests/m23_authenticate_credential.rs +++ b/tests/m23_authenticate_credential.rs @@ -278,7 +278,7 @@ async fn credential_authentication_rejects_inactive_or_deleted_principals() { async fn grpc_authenticate_credential_requires_service_auth_and_returns_identity() { let pool = common::pool().await; let keys = active_keys(&pool).await; - let state = AppState::new(pool.clone(), Config::for_tests(), keys.clone(), None); + let state = AppState::new(pool.clone(), Config::for_tests(), keys.clone(), None, None); let listener = grpc::bind_listener("127.0.0.1:0".parse().expect("addr")) .await .expect("bind grpc"); diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs new file mode 100644 index 0000000..6461fc9 --- /dev/null +++ b/tests/m25_cache_invalidation.rs @@ -0,0 +1,1978 @@ +//! Cache invalidation correctness tests. +//! +//! Requires both a reachable Postgres at `DATABASE_URL` and a reachable Redis +//! at `ATOM_TEST_REDIS_URL`. Run with: +//! +//! ```bash +//! DATABASE_URL=postgres://... ATOM_TEST_REDIS_URL=redis://... cargo test --test m25_cache_invalidation -- --ignored +//! ``` +//! +//! Every test here follows the same shape: warm the cache with a read, mutate +//! through the real (wired) production path — a GraphQL mutation executed +//! through the schema, exactly as a client would call it — then immediately +//! (no sleep) re-check that the mutation's effect is visible. TTL is a +//! defense-in-depth safety net in this design, never the mechanism under +//! test; a passing test here means invalidation is precise, not merely that +//! a short enough TTL happened to expire. + +mod common; + +use std::sync::Arc; + +use async_graphql::Request; +use atom::{ + auth::{self, AuthContext}, + authz::{engine, repo as authz_repo}, + cache::CacheClient, + config::Config, + graphql::build_schema, + identity::{access_tokens, repo as identity_repo}, + models::{ + enums::{Effect, SubjectKind}, + group::CreateGroup, + policy::{AuthzRequest, CreateDirectPolicy, CreatePermissionBlock, CreateRoleAssignment}, + role::CreateRole, + token::CreateAccessToken, + }, + state::AppState, + tenants::repo as tenant_repo, +}; +use common::{cache_client, pool}; +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +/// Builds an `AppState` wired to a fresh Redis-backed cache, plus an +/// `Arc` handle to the *same* instance (mirroring how +/// production shares one cache between `state.cache` and `AuthContext:: +/// cache` — see `auth_from_jwt`/`auth_from_api_key`'s `cache: +/// state.cache.clone()`). Uses a real rotated EC signing key (not an empty +/// placeholder) since several tests here actually encode/verify JWTs. +async fn state_with_cache(pool: PgPool) -> (AppState, Arc) { + let cfg = Config::for_tests(); + let active_keys = atom::keys::rotate(&pool, &cfg.signing_keys) + .await + .expect("rotate test signing key"); + let state = AppState::new(pool, cfg, active_keys, None, Some(cache_client().await)); + let cache = state.cache.clone().expect("cache configured"); + (state, cache) +} + +fn auth_context(entity_id: Uuid, cache: Arc) -> AuthContext { + AuthContext { + entity_id, + tenant_id: None, + session_id: None, + credential_id: None, + scoped: false, + ceiling: None, + cache: Some(cache), + } +} + +fn authed(entity_id: Uuid, cache: Arc, query: impl Into) -> Request { + Request::new(query).data(auth_context(entity_id, cache)) +} + +async fn active_entity(pool: &PgPool, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO entities (id, kind, name, status) VALUES ($1, $2, $3, 'active')") + .bind(id) + .bind(kind) + .bind(format!("cache-test-{kind}-{id}")) + .execute(pool) + .await + .expect("insert entity"); + id +} + +async fn resource(pool: &PgPool, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO resources (id, kind, name) VALUES ($1, $2, $3)") + .bind(id) + .bind(kind) + .bind(format!("cache-test-res-{id}")) + .execute(pool) + .await + .expect("insert resource"); + id +} + +async fn read_action_id(pool: &PgPool) -> Uuid { + sqlx::query_scalar("SELECT id FROM actions WHERE name = 'read' LIMIT 1") + .fetch_one(pool) + .await + .expect("read action") +} + +/// A platform-scope allow block granting `read`, ready to attach via a +/// direct policy or a role. +async fn make_read_block(pool: &PgPool) -> Uuid { + let action_id = read_action_id(pool).await; + authz_repo::create_permission_block( + pool, + CreatePermissionBlock { + tenant_id: None, + scope_mode: "platform".into(), + object_kind: None, + object_type: None, + object_id: None, + group_id: None, + effect: Effect::Allow, + conditions: json!({}), + action_ids: vec![action_id], + }, + ) + .await + .expect("create permission block") + .id +} + +async fn new_group(pool: &PgPool, label: &str) -> Uuid { + identity_repo::create_group( + pool, + CreateGroup { + id: None, + name: format!("cache-test-{label}-{}", Uuid::new_v4()), + tenant_id: None, + group_type: Some("principal".into()), + description: None, + attributes: json!({}), + }, + ) + .await + .expect("create group") + .id +} + +/// A standalone Redis connection for tests that need to inspect cache state +/// directly (e.g. asserting a key was *not* touched), independent of +/// `CacheClient`'s own (private) connection pool. +async fn raw_redis_conn() -> redis::aio::MultiplexedConnection { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + redis::Client::open(url) + .expect("valid redis url") + .get_multiplexed_async_connection() + .await + .expect("connect to test redis") +} + +async fn evaluate_read( + pool: &PgPool, + subject_id: Uuid, + resource_id: Uuid, + cache: Arc, +) -> bool { + let req = AuthzRequest { + subject_id, + action: "read".into(), + resource_id: Some(resource_id), + object_kind: None, + object_id: None, + context: json!({}), + }; + let auth = auth_context(subject_id, cache); + engine::evaluate(pool, &req, &auth) + .await + .expect("evaluate") + .allowed +} + +// ─── Direct policy ────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn direct_policy_revoke_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let subject = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let policy = authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: subject, + permission_block_id: block_id, + }, + ) + .await + .expect("create direct policy"); + + assert!( + evaluate_read(&p, subject, res, cache.clone()).await, + "direct policy should grant read" + ); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteDirectPolicy(id: "{}") }}"#, policy.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, subject, res, cache.clone()).await, + "revoked direct policy must deny immediately, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn direct_policy_revoke_for_group_subject_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let group = new_group(&p, "direct-policy-group").await; + identity_repo::add_group_member(&p, group, member) + .await + .expect("add member"); + + let policy = authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: group, + permission_block_id: block_id, + }, + ) + .await + .expect("create group direct policy"); + + assert!(evaluate_read(&p, member, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteDirectPolicy(id: "{}") }}"#, policy.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "group-subject revoke must immediately deny the group's member" + ); +} + +// ─── Role assignment / role delete ───────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn role_assignment_revoke_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let subject = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + + let assignment = authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: subject, + role_id: role.id, + }, + ) + .await + .expect("create role assignment"); + + assert!(evaluate_read(&p, subject, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ deleteRoleAssignment(id: "{}") }}"#, + assignment.id + ), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, subject, res, cache.clone()).await, + "revoked role assignment must deny immediately" + ); +} + +/// The fan-out case: a role assigned to a group three levels of nesting away +/// from the actual member must invalidate that member's cached grants too. +#[tokio::test] +#[ignore] +async fn role_delete_invalidates_three_level_nested_group_members() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + // grandparent -> parent -> child, member belongs to `child`, role is + // assigned to `grandparent`. + let grandparent = new_group(&p, "gp").await; + let parent = new_group(&p, "parent").await; + let child = new_group(&p, "child").await; + + identity_repo::set_group_parent(&p, parent, grandparent) + .await + .expect("parent under grandparent"); + identity_repo::set_group_parent(&p, child, parent) + .await + .expect("child under parent"); + identity_repo::add_group_member(&p, child, member) + .await + .expect("add member to child"); + + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-nested-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: grandparent, + role_id: role.id, + }, + ) + .await + .expect("assign role to grandparent"); + + assert!( + evaluate_read(&p, member, res, cache.clone()).await, + "member three levels deep should inherit the grandparent's role grant" + ); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteRole(id: "{}") }}"#, role.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "deleting the role must immediately deny the deeply-nested member, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn group_membership_removal_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let group = new_group(&p, "membership-remove").await; + identity_repo::add_group_member(&p, group, member) + .await + .expect("add member"); + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-membership-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: group, + role_id: role.id, + }, + ) + .await + .expect("assign role to group"); + + assert!(evaluate_read(&p, member, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ removeGroupMember(groupId: "{group}", entityId: "{member}") }}"# + ), + )) + .await; + assert!(resp.errors.is_empty(), "remove failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "removing group membership must immediately deny the role grant it carried" + ); +} + +// ─── Session / JWT ────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn session_revoke_immediately_rejects_the_next_authentication() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + // `logout` reads `auth.session_id` from the request context — unlike + // `authed()`'s default (`session_id: None`, fine for every other test + // here since they don't exercise session-bearing mutations), this must + // carry the real session_id or there is nothing for `logout` to revoke. + let logout_auth = AuthContext { + session_id: Some(session_id), + ..auth_context(entity_id, cache.clone()) + }; + let schema = build_schema(state.clone()); + let resp = schema + .execute(Request::new("mutation { logout }").data(logout_auth)) + .await; + assert!(resp.errors.is_empty(), "logout failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "revoked session must be rejected on the very next authentication, not after a TTL" + ); +} + +/// Regression test for a review finding: `logout` used to clear the session's +/// cache barrier (`cache.end`) before `audit::commit_with_audit`'s internal +/// `tx.commit()` actually landed the revoke. In that window a concurrent +/// authentication would see a clean (non-dirty) barrier and a Postgres row +/// that (via MVCC, on a separate connection) still read as not revoked, and +/// would repopulate the cache with a "still valid" entry that then survived +/// for the full session TTL — defeating immediate revocation for exactly the +/// operation most likely to be relied on for it. +/// +/// A black-box race against real `authenticate_token` calls turned out not to +/// reliably land in the gap — it's sub-millisecond, and a full auth call has +/// too much of its own latency to consistently hit it. Instead this polls the +/// barrier's raw `dirty` flag and a direct `revoked_at` read, on a real OS +/// thread running in parallel with `logout` (`flavor = "multi_thread"`, not +/// cooperative async interleaving), so it observes the transition rather than +/// racing to beat it. The invariant: once `dirty` is observed set for this +/// key, it must never be observed clear again while `revoked_at` is still +/// null — that combination is only reachable if the barrier was released +/// before the commit landed. Confirmed this fails against the pre-fix +/// ordering (reverted locally) and passes against the fix. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] +async fn logout_cannot_leave_a_stale_valid_session_cached_during_the_revoke() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let logout_auth = AuthContext { + session_id: Some(session_id), + ..auth_context(entity_id, cache.clone()) + }; + let schema = build_schema(state.clone()); + let session_key = atom::cache::keys::session(session_id); + + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop2 = stop.clone(); + let p_poll = p.clone(); + let key_poll = session_key.clone(); + let poller = tokio::spawn(async move { + let mut conn = raw_redis_conn().await; + let mut saw_dirty = false; + loop { + let dirty: Option = redis::cmd("HGET") + .arg(&key_poll) + .arg("dirty") + .query_async(&mut conn) + .await + .unwrap_or(None); + let is_dirty = dirty.as_deref() == Some("1"); + if is_dirty { + saw_dirty = true; + } + if saw_dirty && !is_dirty { + let revoked_at: Option> = + sqlx::query_scalar("SELECT revoked_at FROM sessions WHERE id = $1") + .bind(session_id) + .fetch_one(&p_poll) + .await + .expect("read revoked_at"); + // Barrier just went clean; if the row isn't committed as + // revoked yet, that's the bug. If it's already revoked, the + // ordering was correct — nothing more to catch. + return revoked_at.is_none(); + } + if stop2.load(std::sync::atomic::Ordering::Relaxed) { + return false; + } + } + }); + + let logout_result = schema + .execute(Request::new("mutation { logout }").data(logout_auth)) + .await; + assert!( + logout_result.errors.is_empty(), + "logout failed: {:?}", + logout_result.errors + ); + + stop.store(true, std::sync::atomic::Ordering::Relaxed); + let violation = poller.await.expect("join poller"); + assert!( + !violation, + "observed the session cache barrier clear (dirty -> clean) while the session row still \ + read as un-revoked — a concurrent reader landing at that exact moment would repopulate \ + the cache with a stale \"still valid\" entry, exactly the bug this regresses" + ); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "revoked session must be rejected on the very next authentication, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn entity_deactivation_immediately_rejects_an_existing_valid_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache for this session. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + // Deactivate through the real, wired GraphQL path. + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "deactivating the entity must immediately reject its existing session, not after a TTL" + ); +} + +// ─── Credential ───────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn credential_revoke_immediately_rejects_the_next_authentication() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + // Warm the credential cache with a successful authentication. + auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ revokeAccessToken(credentialId: "{}") }}"#, + minted.credential_id + ), + )) + .await; + assert!(resp.errors.is_empty(), "revoke failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &minted.token).await; + assert!( + result.is_err(), + "revoked credential must be rejected on the very next authentication, not after a TTL" + ); +} + +// ─── Tenant delete / restore ──────────────────────────────────────────────── + +async fn tenant(pool: &PgPool) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO tenants (id, name, status) VALUES ($1, $2, 'active')") + .bind(id) + .bind(format!("cache-test-tenant-{id}")) + .execute(pool) + .await + .expect("insert tenant"); + id +} + +async fn active_entity_in_tenant(pool: &PgPool, tenant_id: Uuid, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO entities (id, kind, name, tenant_id, status) VALUES ($1, $2, $3, $4, 'active')", + ) + .bind(id) + .bind(kind) + .bind(format!("cache-test-{kind}-{id}")) + .bind(tenant_id) + .execute(pool) + .await + .expect("insert entity"); + id +} + +/// Regression test for a review finding: `soft_delete_tenant` (behind +/// `deleteTenant`) is a separate function from `change_tenant_status` and +/// was found to be completely unwired — a stale `tenant_status` cache entry +/// let an existing JWT session for a member of the deleted tenant keep +/// authenticating. Both the session cache *and* the tenant_status cache are +/// warmed stale-valid here before the delete, so this only passes if +/// `tenant_status` is genuinely invalidated (a stale-but-unrevoked session +/// cache entry alone would otherwise still let auth through). +#[tokio::test] +#[ignore] +async fn tenant_delete_immediately_rejects_an_existing_session_of_a_member() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status/tenant_status caches, all showing valid. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "deleting the tenant must immediately reject an existing member session, not after a TTL \ + (this only fails if tenant_status wasn't actually invalidated, since the session cache \ + entry alone is stale-valid and would otherwise let auth through)" + ); +} + +/// Regression test for a review finding: `change_tenant_status`'s +/// `disableTenant`/`freezeTenant` mutations bulk-revoke the tenant's +/// members' sessions in the same transaction as the status flip, but the +/// resolver only ever established a cache barrier on `tenant_status` — never +/// on the sessions it was about to revoke. +/// +/// As with `tenant_restore_clears_reactivated_credential_cache_entries` +/// below, an end-to-end auth-behavior test can't actually isolate this: the +/// `tenant_status` barrier's own `end` always clears that entry's payload +/// too, so the very next authentication after `disableTenant` (or after a +/// subsequent `enableTenant`) is forced through a fresh-Postgres-reload path +/// regardless of whether the session was ever separately invalidated — that +/// fresh reload sees the correctly-revoked row and denies either way, and +/// self-heals the session cache entry in the process, masking the bug on +/// every subsequent check too. Confirmed by trying exactly that shape of +/// test and finding it passed even with the session invalidation reverted. +/// So this checks the one thing that actually isolates it: that +/// `disableTenant` clears the session's own cache entry directly, verified +/// against Redis, independent of what any later auth attempt would do. +#[tokio::test] +#[ignore] +async fn disabling_a_tenant_invalidates_its_members_session_cache_entries() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session cache as a valid, stale-able hit. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + let session_key = atom::cache::keys::session(session_id); + let (_, _, payload_before) = hmget_raw(&session_key).await; + assert!( + payload_before.is_some(), + "session cache should be warm before disabling the tenant" + ); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ disableTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "disable failed: {:?}", resp.errors); + + // `end`'s barrier always clears the payload of whatever it covers (see + // `src/cache/mod.rs`), so a lingering payload here means this session's + // key was never part of the barrier at all -- the exact gap the review + // found. + let (_, _, payload_after) = hmget_raw(&session_key).await; + assert!( + payload_after.is_none(), + "disabling a tenant must invalidate its members' session cache entries, not just \ + tenant_status -- a lingering \"still valid\" session cache entry here would keep \ + serving hits (masked only by tenant_status denying auth while the tenant stays \ + disabled) and would resurface the instant the tenant is re-enabled" + ); +} + +/// Regression test for a review finding: `restore_tenant` reactivates +/// credentials but the resolver originally only invalidated `tenant_status`, +/// not the credential entries it reactivates. +/// +/// Important nuance found while developing this test: an *end-to-end* +/// authentication test for this can't actually distinguish "the credential +/// invalidation ran" from "it didn't", because `restore_tenant`'s own (also +/// necessary) `tenant_status` invalidation forces `auth_from_api_key` through +/// its fresh-Postgres-reload branch regardless — that branch bypasses +/// whatever is in the credential cache entirely, masking the credential fix +/// either way. Both an "authenticate while deleted" attempt (to naturally +/// populate a stale entry) and a full auth-behavior assertion after restore +/// were tried and found to pass even with the credential invalidation +/// stripped out, for exactly this reason — confirmed by reverting the fix +/// and rerunning. So this test checks the one thing that actually isolates +/// it: that `restoreTenant` clears the credential's cache entry directly, +/// verified against Redis, independent of which auth code path would +/// subsequently be taken. +/// +/// The fix is kept as a matter of defensive correctness even though the +/// review's literal "cache captures revoked, stays stuck after restore" +/// scenario isn't reachable via the normal auth flow today (the tenant's own +/// `deleted_at` join filter means nothing can populate a credential's cache +/// entry while its tenant is genuinely deleted, so there is no live window +/// in which the entry could be wrong when restore runs) — relying on that +/// incidental protection instead of an explicit invalidation would be +/// fragile against future changes to either the join or the fast-path +/// gating in `auth_from_api_key`. +#[tokio::test] +#[ignore] +async fn tenant_restore_clears_reactivated_credential_cache_entries() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-tenant-restore-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let cred_key = atom::cache::keys::credential(minted.credential_id); + seed_hit( + &cred_key, + &atom::cache::entries::CredentialCacheEntry { + entity_id, + status: atom::models::enums::CredentialStatus::Revoked, + secret_hash: None, + secret_lookup_hash: Some(vec![0u8; 32]), + expires_at: None, + scoped: false, + }, + ) + .await; + let before = hmget_raw(&cred_key).await; + assert!( + before.2.is_some(), + "seeded entry should have a payload before restore: {before:?}" + ); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + let after = hmget_raw(&cred_key).await; + assert!( + after.2.is_none(), + "restoreTenant must clear the reactivated credential's cache payload \ + (begin() deletes it, end() never restores it) — got {after:?}, expected the \ + payload field to be absent" + ); + assert_ne!( + after.0, before.0, + "the credential key's version must have been bumped by restoreTenant's barrier" + ); +} + +/// Regression test for a review finding: `soft_delete_tenant` revokes +/// members' sessions in Postgres, but the delete resolver only invalidated +/// `tenant_status`, never the sessions themselves. During the deleted window +/// this is harmless — the `tenant_status` miss forces a fresh Postgres check, +/// which denies correctly — but once `restoreTenant` repopulates +/// `tenant_status` as active again, a session cached as valid *before* the +/// delete would resurface as a full cache hit and authenticate despite being +/// revoked, since `restoreTenant` deliberately never reinstates sessions. +/// +/// The first authentication attempt after restore always denies correctly +/// regardless of this fix, because restore's own `tenant_status` invalidation +/// forces that one request through a fresh Postgres reload (which, as a side +/// effect, also repopulates `tenant_status` as a hit). It's the *second* +/// attempt — once `tenant_status` is a hit again — that actually distinguishes +/// "the session was invalidated at delete time" from "it wasn't"; confirmed +/// by temporarily reverting the fix and rerunning. +#[tokio::test] +#[ignore] +async fn tenant_delete_then_restore_immediately_rejects_a_pre_existing_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status/tenant_status caches, all showing valid. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by tenant delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by tenant delete must stay rejected even once tenant_status is a \ + cache hit again post-restore — this only fails if the session cache entry wasn't \ + invalidated at delete time and survived as a stale 'valid' hit" + ); +} + +/// Regression test for a review finding: `delete_entity` revokes the +/// entity's sessions in Postgres, but the delete resolver only invalidated +/// `entity_status`. Same masking shape as the tenant case above: the first +/// post-restore attempt always denies correctly (entity_status's own +/// invalidation forces a fresh reload), and it's the second attempt that +/// actually proves the session cache entry itself was invalidated at delete +/// time, not left as a stale 'valid' hit for `restoreEntity` to unmask. +#[tokio::test] +#[ignore] +async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by entity delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by entity delete must stay rejected even once entity_status is a \ + cache hit again post-restore — this only fails if the session cache entry wasn't \ + invalidated at delete time and survived as a stale 'valid' hit" + ); +} + +/// Same shape as the session test above, for the credential side: `delete_entity` +/// also revokes the entity's access-token credentials in Postgres, and +/// `restoreEntity` intentionally never reinstates them. Requires two +/// post-restore authentication attempts for the same reason as the session +/// test — the first always denies via a forced fresh reload; the second is +/// the one that actually proves the credential cache entry itself was +/// invalidated at delete time. +#[tokio::test] +#[ignore] +async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_access_token() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-entity-restore-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &minted.token) + .await + .is_err(), + "an access token revoked by entity delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &minted.token) + .await + .is_err(), + "an access token revoked by entity delete must stay rejected even once entity_status is \ + a cache hit again post-restore — this only fails if the credential cache entry wasn't \ + invalidated at delete time and survived as a stale 'active' hit" + ); +} + +// ─── Enumerate-before-lock race (session/credential bulk invalidation) ───── + +/// Regression test for a review finding: `deleteEntity`'s session/access-token +/// cache-key enumeration used to run via a plain pool query *before* any +/// transaction or lock was taken. A session or access-token credential +/// created for the entity concurrently — after that enumeration but before +/// the delete's own revoke ran — was never included in the cache barrier and +/// could keep serving as a valid cache hit indefinitely (past the delete, and +/// even across a later `restoreEntity`), despite being revoked in Postgres. +/// This is the identical race shape already closed for group-membership +/// invalidation (see `concurrent_group_membership_change_serializes_against_ +/// the_group_subject_lock` below), just previously unfixed for +/// sessions/credentials. +/// +/// Fixed by moving the enumeration inside the same transaction as the lock +/// (`identity::repo::lock_entity_and_collect_revocation_ids_in_tx`), which +/// takes an exclusive lock on the entity row — the same lock +/// `create_session`/`create_access_token` take (via `lock_active_entity`) +/// before inserting. As with the group-membership fix, an end-to-end test +/// racing a real `create_session` against a real `deleteEntity` mutation +/// can't actually distinguish fixed from unfixed (a post-hoc read is always a +/// fresh cold-cache load reflecting final Postgres state); this test instead +/// calls the new function directly and proves there is no window in which a +/// concurrent `create_session` can slip past it uncounted: either it's +/// blocked until the transaction commits, or (once committed) it correctly +/// fails against the now-deactivated entity — there is no third outcome +/// where it silently succeeds outside the transaction's lock. +#[tokio::test] +#[ignore] +async fn concurrent_session_creation_cannot_evade_the_entity_delete_enumeration() { + let p = pool().await; + let entity = active_entity(&p, "service").await; + + let mut tx = p.begin().await.expect("begin tx"); + let (session_ids, credential_ids) = + identity_repo::lock_entity_and_collect_revocation_ids_in_tx(&mut tx, entity) + .await + .expect("lock and enumerate"); + assert!( + session_ids.is_empty() && credential_ids.is_empty(), + "no sessions/credentials exist yet for this fresh entity" + ); + + // `create_session` takes the same entity-row lock before inserting, so it + // must block until the transaction above commits — not race ahead of the + // still-open delete transaction and create a session the enumeration has + // already missed. + let p2 = p.clone(); + let handle = + tokio::spawn(async move { identity_repo::create_session(&p2, entity, 3600).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "create_session must block on the same entity-row lock the delete's enumeration holds, \ + not commit a session the enumeration has already missed" + ); + + // Mirrors the production call site's order: the status-flip runs after + // the lock-and-enumerate step (which the barrier, in production, is + // established between), still inside the same transaction as the lock. + identity_repo::deactivate_and_finish_entity_deletion_in_tx(&mut tx, false, None, None, entity) + .await + .expect("deactivate and finish"); + tx.commit().await.expect("commit lock-holding tx"); + + // Once committed, the entity is deactivated (the same effect + // `deleteEntity`'s cache-aware path has by this point). The + // previously-blocked `create_session` must now correctly fail rather than + // slip through and create a session outside of any cache barrier the + // delete established. + let outcome = handle.await.expect("join create_session task"); + assert!( + outcome.is_err(), + "create_session must fail once the entity has been deactivated by the transaction it \ + was blocked on — succeeding here would mean a session was created in a window the \ + delete's enumeration could never see, exactly the race this fix closes" + ); +} + +/// Tenant-level counterpart to the test above: `deleteTenant`'s session +/// cache-key enumeration used to run via a plain pool query before any +/// transaction/lock was taken, so a session created for a member entity in +/// the window between enumeration and the tenant's own bulk revoke was never +/// included in the cache barrier. Fixed by moving the enumeration inside the +/// same transaction as the lock +/// (`tenants::repo::lock_tenant_and_collect_session_ids_in_tx`), which takes +/// an exclusive lock on the tenant row — the same lock `lock_active_entity` +/// takes (via `lock_optional_active_tenant`) before any session/credential +/// can be created for *any* entity in the tenant. +#[tokio::test] +#[ignore] +async fn concurrent_session_creation_cannot_evade_the_tenant_delete_enumeration() { + let p = pool().await; + let tenant_id = tenant(&p).await; + let entity = active_entity_in_tenant(&p, tenant_id, "service").await; + + let mut tx = p.begin().await.expect("begin tx"); + let session_ids = tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id) + .await + .expect("lock and enumerate"); + assert!( + session_ids.is_empty(), + "no sessions exist yet for this fresh tenant's member" + ); + + let p2 = p.clone(); + let handle = + tokio::spawn(async move { identity_repo::create_session(&p2, entity, 3600).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "create_session for an entity in this tenant must block on the tenant-row lock the \ + delete's enumeration holds, not commit a session the enumeration has already missed" + ); + + // Mirrors the production call site's order: the status-flip runs after + // the lock-and-enumerate step (which the barrier, in production, is + // established between), still inside the same transaction as the lock. + tenant_repo::deactivate_and_finish_tenant_soft_delete_in_tx( + &mut tx, false, None, None, tenant_id, + ) + .await + .expect("deactivate and finish"); + tx.commit().await.expect("commit lock-holding tx"); + + // Once committed, the tenant is deleted (so `lock_active_entity`'s own + // `lock_optional_active_tenant` check fails) — the previously-blocked + // `create_session` must now correctly fail rather than slip through. + let outcome = handle.await.expect("join create_session task"); + assert!( + outcome.is_err(), + "create_session must fail once the tenant has been deactivated by the transaction it \ + was blocked on — succeeding here would mean a session was created in a window the \ + delete's enumeration could never see, exactly the race this fix closes" + ); +} + +// ─── API-key tenant context after an entity moves tenants ────────────────── + +/// Regression test for a review finding: the API-key fast path derived +/// `AuthContext.tenant_id` from the credential cache entry's own duplicated +/// `tenant_id` field, which only got invalidated when the credential itself +/// changed — not when the owning entity moved to a different tenant (only +/// `entity_status` is invalidated on that mutation). The fix removed the +/// duplicated field entirely and derives tenant context from the entity's +/// own (correctly invalidated) cache entry. +/// +/// As with the delete/restore tests above, the first authentication attempt +/// after the move always reflects the new tenant correctly regardless of the +/// fix, because the move's `entity_status` invalidation forces that one +/// request through a fresh Postgres reload. It's the *second* attempt — once +/// `entity_status` is a hit again — that actually distinguishes "tenant +/// context comes from the fresh entity entry" from "it comes from the stale +/// credential copy"; confirmed by temporarily reverting the fix and +/// rerunning. +#[tokio::test] +#[ignore] +async fn api_key_auth_reflects_the_current_tenant_after_an_entity_moves_tenants() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let old_tenant = tenant(&p).await; + let new_tenant = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, old_tenant, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-tenant-move-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + assert_eq!(ctx.tenant_id, Some(old_tenant)); + + let schema = build_schema(state.clone()); + let mv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ updateEntity(id: "{entity_id}", input: {{ tenantId: "{new_tenant}" }}) {{ id }} }}"# + ), + )) + .await; + assert!(mv.errors.is_empty(), "tenant move failed: {:?}", mv.errors); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("authentication right after the move should still succeed"); + assert_eq!( + ctx.tenant_id, + Some(new_tenant), + "not the interesting assertion by itself" + ); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("authentication should still succeed once entity_status is a hit again"); + assert_eq!( + ctx.tenant_id, + Some(new_tenant), + "AuthContext.tenant_id must reflect the entity's current tenant, not a stale copy that \ + would have been cached in the credential entry from before the move" + ); +} + +/// Mints a real session row plus a signed JWT carrying `tenant_id` as its +/// `tid` claim, so tests can exercise `auth_from_jwt` end to end. +async fn session_jwt( + state: &AppState, + pool: &PgPool, + entity_id: Uuid, + tenant_id: Option, +) -> String { + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(pool) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + auth::encode_jwt( + entity_id, + session_id, + tenant_id, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt") +} + +/// JWT counterpart to +/// `api_key_auth_reflects_the_current_tenant_after_an_entity_moves_tenants`, +/// and a regression test for a review finding the API-key path did not have: +/// `auth_from_jwt` built the `tenant_status` cache key from the token's `tid` +/// claim but populated it with the payload of the *entity's current* tenant, +/// because the miss loader joins tenants through `entities.tenant_id`. Once a +/// token outlives a tenant move the two differ, so authenticating with the +/// stale token wrote the new tenant's status under the old tenant's key — and +/// since the populate ran *before* `check_session_entity_tenant`, even the +/// request that got rejected for the tid mismatch poisoned it on the way out. +/// A frozen tenant then read back as active for every one of its members +/// until the TTL elapsed. +#[tokio::test] +#[ignore] +async fn a_stale_jwt_from_before_a_tenant_move_cannot_poison_the_old_tenants_status() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let old_tenant = tenant(&p).await; + let new_tenant = tenant(&p).await; + // `mover` leaves `old_tenant` for `new_tenant`; `stayer` remains behind + // and is the one a poisoned `tenant_status:{old_tenant}` would wrongly + // let through. + let mover = active_entity_in_tenant(&p, old_tenant, "service").await; + let stayer = active_entity_in_tenant(&p, old_tenant, "service").await; + + let mover_token = session_jwt(&state, &p, mover, Some(old_tenant)).await; + let stayer_token = session_jwt(&state, &p, stayer, Some(old_tenant)).await; + + // Warm `stayer`'s session and entity_status entries while everything is + // still valid. Freezing the tenant below invalidates `tenant_status` and + // nothing else, so those two stay hits — which is what makes a poisoned + // `tenant_status` entry decisive rather than academic: with all three + // keys hitting, `auth_from_jwt` never consults Postgres at all. + auth::authenticate_token(&state, &stayer_token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let mv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ updateEntity(id: "{mover}", input: {{ tenantId: "{new_tenant}" }}) {{ id }} }}"# + ), + )) + .await; + assert!(mv.errors.is_empty(), "tenant move failed: {:?}", mv.errors); + + let sv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ freezeTenant(id: "{old_tenant}") {{ id }} }}"#), + )) + .await; + assert!( + sv.errors.is_empty(), + "tenant freeze failed: {:?}", + sv.errors + ); + + // `mover`'s token still claims `old_tenant` while the entity now lives in + // `new_tenant`, so this must be rejected on the tid mismatch. + let result = auth::authenticate_token(&state, &mover_token).await; + assert!( + result.is_err(), + "a JWT whose tid no longer matches the entity's tenant must be rejected" + ); + + // The interesting assertion: that rejected authentication must not have + // left `new_tenant`'s active status cached under `old_tenant`'s key. + let result = auth::authenticate_token(&state, &stayer_token).await; + assert!( + result.is_err(), + "a member of the frozen tenant must still be rejected — the rejected \ + authentication above must not have populated tenant_status for the \ + frozen tenant with the moved entity's new tenant's status" + ); +} + +/// A platform-scope allow block granting `create`, so a non-admin subject can +/// be given exactly the capability `createTenant`'s gate requires. +async fn make_platform_create_block(pool: &PgPool) -> Uuid { + let action_id: Uuid = + sqlx::query_scalar("SELECT id FROM actions WHERE name = 'create' LIMIT 1") + .fetch_one(pool) + .await + .expect("create action"); + authz_repo::create_permission_block( + pool, + CreatePermissionBlock { + tenant_id: None, + scope_mode: "platform".into(), + object_kind: None, + object_type: None, + object_id: None, + group_id: None, + effect: Effect::Allow, + conditions: json!({}), + action_ids: vec![action_id], + }, + ) + .await + .expect("create permission block") + .id +} + +/// Regression test for a review finding: `createTenant` bootstraps a +/// tenant-admin role, role assignment and membership for the creator in the +/// same transaction — growing the creator's own grant set — but ran with no +/// `grants` invalidation. The capability gate immediately above it warms that +/// exact key, so a creator without platform-wide `manage` could not administer +/// the tenant they had just created until the grants TTL lapsed. +#[tokio::test] +#[ignore] +async fn tenant_creation_immediately_grants_the_creator_tenant_admin() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + // Deliberately *not* a platform admin: with platform-wide `manage` the + // creator would pass the post-creation check no matter what the cache + // held, and the test would prove nothing. + let creator = active_entity(&p, "service").await; + let block_id = make_platform_create_block(&p).await; + authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: creator, + permission_block_id: block_id, + }, + ) + .await + .expect("create direct policy"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + creator, + cache.clone(), + r#"mutation { createTenant(input: { name: "cache-test-bootstrap" }) { id } }"#, + )) + .await; + assert!( + resp.errors.is_empty(), + "createTenant failed: {:?}", + resp.errors + ); + let tenant_id: Uuid = resp + .data + .into_json() + .expect("response is json") + .pointer("/createTenant/id") + .and_then(|id| id.as_str()) + .expect("createTenant returned an id") + .parse() + .expect("parse created tenant id"); + + // The creator's grants were warmed by `createTenant`'s own capability + // gate, moments before the bootstrap added the tenant-admin assignment. + let req = AuthzRequest { + subject_id: creator, + action: "manage".into(), + resource_id: None, + object_kind: Some("tenant".into()), + object_id: Some(tenant_id), + context: json!({}), + }; + let auth = auth_context(creator, cache.clone()); + let decision = engine::evaluate(&p, &req, &auth).await.expect("evaluate"); + assert!( + decision.allowed, + "the creator must be able to manage the tenant they just created on the very next \ + request, not after the grants TTL expires" + ); +} + +// ─── Group-subject mutation vs. concurrent membership change ─────────────── + +/// Regression test for a review finding: resolving a group-subject +/// mutation's affected `grants` keys from "current members" alone has a +/// race against a concurrent `add_group_member` for the same group — a +/// member added in between the enumeration and the mutation's own commit +/// would never be in the enumerated set, so their `grants` key would never +/// be invalidated, letting a stale cached grant (or a stale cached lack of +/// one) survive until the grants TTL. +/// +/// Fixed by locking the group's row — the same lock +/// `identity::repo::add_group_member`/`remove_group_member` already take — +/// via `authz::repo::lock_group_closures_and_collect_member_ids`, for the +/// caller's entire transaction (enumeration through commit). This test +/// proves the lock actually serializes against a concurrent membership +/// change, mirroring `m8_guardrails.rs`'s +/// `concurrent_block_link_and_role_assignment_serialize` for the analogous +/// role-lock case. +#[tokio::test] +#[ignore] +async fn concurrent_group_membership_change_serializes_against_the_group_subject_lock() { + let p = pool().await; + let group = new_group(&p, "concurrent-lock").await; + let existing_member = active_entity(&p, "service").await; + identity_repo::add_group_member(&p, group, existing_member) + .await + .expect("seed existing member"); + + // Hold the lock a group-subject mutation resolver would (e.g. + // `deleteDirectPolicy`'s) — the exact enumeration step, kept open (not + // yet committed). + let mut tx = p.begin().await.expect("begin tx"); + let member_ids = authz_repo::lock_group_closures_and_collect_member_ids(&mut tx, &[group]) + .await + .expect("lock and enumerate"); + assert_eq!( + member_ids, + vec![existing_member], + "enumeration should see only the pre-existing member while the lock is held" + ); + + // `add_group_member` locks the same group row, so it must block until + // the transaction above commits — not silently add a member our + // (already-captured) enumeration has already missed. + let p2 = p.clone(); + let new_member = active_entity(&p, "service").await; + let handle = + tokio::spawn(async move { identity_repo::add_group_member(&p2, group, new_member).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "add_group_member must block on the same group-row lock the enumeration holds, not \ + commit a membership change the enumeration has already missed" + ); + + tx.commit().await.expect("commit lock-holding tx"); + handle + .await + .expect("join add_group_member task") + .expect("add_group_member should succeed once unblocked"); +} + +/// Regression test for a review finding: `createRoleAssignment`'s +/// group-subject path locked the subject group's closure first and only +/// reached the role lock afterward (inside `create_role_assignment_in_tx`'s +/// own `lock_role` call) — the reverse of every role mutation +/// (`replaceRolePermissionBlocks`/`deleteRole`/`restoreRole`, via +/// `lock_role_and_collect_grants_keys`), which always locks the role first, +/// then any assigned groups' closures. If role R is already assigned to an +/// ancestor group and a concurrent pair of requests ran — assign R to a +/// descendant group; mutate R's blocks — they could hold the descendant +/// group's row and the role row in opposite order: a genuine wait-for cycle, +/// which Postgres detects and resolves by aborting one side with a +/// "deadlock detected" error. +/// +/// Fixed by having `createRoleAssignment`'s group-subject path lock the role +/// *before* the group closure too, via +/// `authz::repo::lock_role_then_group_closure_and_collect_grants_keys`. +/// +/// A first version of this test held only the role lock and asserted +/// `createRoleAssignment` blocked on it — but that passes under *either* +/// lock order (the old order still reaches the role lock eventually, via +/// `create_role_assignment_in_tx`'s own `lock_role` call, just later), so it +/// couldn't actually distinguish fixed from unfixed (confirmed: it passed +/// even with the fix reverted). This version instead reconstructs the +/// actual two-resource cycle: hold the role lock, let the spawned +/// `createRoleAssignment` block on it, then — *while it's still blocked* — +/// try to lock the descendant group ourselves. With the fix, the spawned +/// task hasn't touched the group yet (it's queued for the role first), so +/// our lock attempt succeeds immediately. With the old order, the spawned +/// task would already be holding the group (locked before it ever reached +/// for the role), so our attempt would itself block — the two sides +/// waiting on each other — which is exactly the cycle this fix closes. +#[tokio::test] +#[ignore] +async fn create_role_assignment_for_group_subject_locks_the_role_before_the_group() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let ancestor = new_group(&p, "lock-order-ancestor").await; + let descendant = new_group(&p, "lock-order-descendant").await; + identity_repo::set_group_parent(&p, descendant, ancestor) + .await + .expect("set parent"); + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("m25-lock-order-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + // Assigning R to the ancestor puts `descendant` in R's assigned-group + // closure — exactly what `lock_role_and_collect_grants_keys` would lock + // for a mutation on R. + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: ancestor, + role_id: role.id, + }, + ) + .await + .expect("assign role to ancestor"); + + // Hold only the role lock — exactly what a role mutation + // (`lock_role_and_collect_grants_keys`) holds at the point it's already + // past its own first step, before it reaches for the assigned groups' + // closures. + let mut tx = p.begin().await.expect("begin tx"); + sqlx::query("SELECT id FROM roles WHERE id = $1 FOR UPDATE") + .bind(role.id) + .fetch_one(&mut *tx) + .await + .expect("lock role"); + + // Concurrently: assign R to the descendant group, through the real, + // wired GraphQL path. + let schema = build_schema(state.clone()); + let cache2 = cache.clone(); + let role_id = role.id; + let handle = tokio::spawn(async move { + schema + .execute(authed( + common::admin_id(), + cache2, + format!( + r#"mutation {{ + createRoleAssignment(input: {{ + subjectKind: group, + subjectId: "{descendant}", + roleId: "{role_id}" + }}) {{ id }} + }}"# + ), + )) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "createRoleAssignment must block on the role lock" + ); + + // The decisive check: while the spawned task is blocked waiting for the + // role lock, we must still be able to lock the descendant group + // immediately — proving the spawned task has not already acquired it + // (which the old, reversed order would have done before ever touching + // the role). + tokio::time::timeout( + std::time::Duration::from_millis(500), + sqlx::query("SELECT id FROM principal_groups WHERE id = $1 FOR UPDATE") + .bind(descendant) + .fetch_one(&mut *tx), + ) + .await + .expect( + "locking the descendant group must not block while createRoleAssignment is blocked on \ + the role lock — a block here means it's holding the group while waiting on the role, \ + i.e. the exact deadlock cycle this fix closes", + ) + .expect("lock descendant group"); + + tx.commit().await.expect("commit tx"); + let resp = handle.await.expect("join createRoleAssignment task"); + assert!( + resp.errors.is_empty(), + "createRoleAssignment failed: {:?}", + resp.errors + ); +} + +// An end-to-end complement to the test above — racing a real +// `add_group_member` against the real `deleteDirectPolicy` mutation, then +// asserting the new member's grants are correctly denied — was tried and +// dropped. It passed even with the group-row lock reverted (confirmed by +// reverting and rerunning, 5/5), because it only calls `evaluate_read` once, +// *after* both operations have fully committed: that read is always a fresh +// cold-cache load reflecting final Postgres state, so it can never observe +// the stale intermediate value the race would have produced. Reproducing the +// actual staleness would require a read to land in the narrow window between +// the add's commit and the delete's commit — not achievable through ordinary +// concurrent scheduling without injecting a hook into the read path. The +// mechanism test above (proving the lock itself blocks a concurrent +// `add_group_member`) is the one that actually distinguishes fixed from +// unfixed here, the same lesson as the credential-restore investigation +// documented on `tenant_restore_clears_reactivated_credential_cache_entries`. + +async fn hmget_raw(key: &str) -> (Option, Option, Option>) { + let mut conn = raw_redis_conn().await; + redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut conn) + .await + .expect("hmget") +} + +/// Directly writes a clean (non-dirty, version 1), already-populated cache +/// entry — bypassing `try_populate`'s version check entirely, since this is +/// for hand-seeding test fixtures, not exercising the barrier itself. +async fn seed_hit(key: &str, value: &T) { + let payload = serde_json::to_vec(value).expect("serialize seed value"); + let mut conn = raw_redis_conn().await; + let _: () = redis::cmd("HSET") + .arg(key) + .arg("v") + .arg(1) + .arg("dirty") + .arg("0") + .arg("p") + .arg(payload) + .query_async(&mut conn) + .await + .expect("seed cache entry"); +} + +// ─── Negative control ─────────────────────────────────────────────────────── + +/// Proves the "tenant status short-circuits before grants" design decision is +/// actually followed, not just accidentally correct: a tenant status change +/// must invalidate `tenant_status`, but must NOT touch the `grants` key. +#[tokio::test] +#[ignore] +async fn tenant_status_change_does_not_touch_the_grants_cache_key() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + + let tenant_id = Uuid::new_v4(); + sqlx::query("INSERT INTO tenants (id, name, status) VALUES ($1, $2, 'active')") + .bind(tenant_id) + .bind(format!("cache-test-tenant-{tenant_id}")) + .execute(&p) + .await + .expect("insert tenant"); + + // Warm the member's grants cache (an empty grant set is still a cached + // entry — what matters is whether the key gets touched, not its value). + let req = AuthzRequest { + subject_id: member, + action: "read".into(), + resource_id: None, + object_kind: Some("tenant".into()), + object_id: Some(tenant_id), + context: json!({}), + }; + let auth = auth_context(member, cache.clone()); + let _ = engine::evaluate(&p, &req, &auth).await.expect("evaluate"); + + let grants_key = atom::cache::keys::grants(member); + let mut conn = raw_redis_conn().await; + let before_exists: bool = redis::cmd("EXISTS") + .arg(&grants_key) + .query_async(&mut conn) + .await + .expect("exists check"); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ disableTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "disable failed: {:?}", resp.errors); + + let after_exists: bool = redis::cmd("EXISTS") + .arg(&grants_key) + .query_async(&mut conn) + .await + .expect("exists check"); + + assert_eq!( + before_exists, after_exists, + "tenant status change must not invalidate the grants cache key — the PDP's \ + tenant-lifecycle deny check runs before grant matching, so grants invalidation \ + is unnecessary here by design (see authz::engine::load_decision_context)" + ); +} diff --git a/tests/m26_audit_event_publishing.rs b/tests/m26_audit_event_publishing.rs index 1af2827..799521b 100644 --- a/tests/m26_audit_event_publishing.rs +++ b/tests/m26_audit_event_publishing.rs @@ -50,7 +50,7 @@ async fn build_state(pool: PgPool, config: Config) -> AppState { let active_keys = keys::load_active_keys(&pool, &config.signing_keys) .await .expect("load signing keys"); - AppState::new(pool, config, active_keys, None) + AppState::new(pool, config, active_keys, None, None) } fn authed(query: impl Into) -> Request {