From 4ce5cde74a923ab888cadce2c013453a5ee4c2de Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 15:12:47 +0300 Subject: [PATCH 01/19] Add Redis-backed cache client and wire it through AppState Introduces a race-safe caching layer for authentication/authorization decision inputs: a Redis hash per key with version/dirty/payload fields and three atomic Lua primitives (begin/end/try_populate) that prevent a reader from repopulating stale data around a concurrent mutation. Caching is off by default and fully optional - disabled, this is a no-op passthrough to Postgres. This commit adds the client, config, health/metrics scaffolding, and AppState/main wiring; nothing consumes it yet. --- .env.example | 40 ++ Cargo.lock | 88 +++ Cargo.toml | 2 + docker-compose.yml | 40 ++ src/cache/entries.rs | 56 ++ src/cache/invalidate.rs | 84 +++ src/cache/keys.rs | 30 + src/cache/mod.rs | 881 +++++++++++++++++++++++++++ src/config.rs | 208 +++++++ src/error.rs | 7 + src/graphql/schema.rs | 1 + src/grpc.rs | 1 + src/health.rs | 32 + src/lib.rs | 1 + src/main.rs | 30 +- src/metrics.rs | 25 +- src/routes.rs | 1 + src/state.rs | 11 +- tests/common/mod.rs | 24 + tests/m10_graphql_profiles.rs | 1 + tests/m11_graphql_primitives.rs | 2 +- tests/m12_graphql_identity.rs | 1 + tests/m13_graphql_authz_admin.rs | 1 + tests/m14_api_endpoints.rs | 2 +- tests/m17_certificates.rs | 1 + tests/m23_authenticate_credential.rs | 2 +- 26 files changed, 1563 insertions(+), 9 deletions(-) create mode 100644 src/cache/entries.rs create mode 100644 src/cache/invalidate.rs create mode 100644 src/cache/keys.rs create mode 100644 src/cache/mod.rs diff --git a/.env.example b/.env.example index 6a1a008..598fccb 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,46 @@ ATOM_DB_CONNECT_TIMEOUT_SECS=10 ATOM_DB_IDLE_TIMEOUT_SECS=600 ATOM_DB_MAX_LIFETIME_SECS=1800 +# --- Cache (Redis) -------------------------------------------------------- +# Redis-backed cache for AuthN/AuthZ decision *inputs* (session validity, +# entity/tenant status, credential verification, the effective-grants +# expansion) — never the allow/deny decision itself, which is always +# recomputed. Postgres remains authoritative: this is a pure performance +# optimization, off by default, and every check works correctly with it +# disabled. See src/cache/mod.rs for the full consistency model. +# +# Invalidation is precise and synchronous on every relevant mutation (not +# TTL-based) — the TTLs below are a defense-in-depth safety net only, for the +# rare case an invalidation call itself is lost (e.g. a crash mid-mutation). +# While enabled, an unreachable Redis refuses security-sensitive mutations +# (grants/session/credential changes) rather than committing a change the +# cache can't be told about; reads always fall back to Postgres regardless. +# +# Off by default. Uncomment and point at a reachable Redis (e.g. the `redis` +# service in docker-compose.yml) to enable: +ATOM_CACHE_ENABLED=false +# ATOM_CACHE_REDIS_URL=redis://redis:6379/0 +# ATOM_CACHE_POOL_MAX_SIZE=20 +# ATOM_CACHE_CONNECT_TIMEOUT_MS=2000 +# A slow Redis must never make auth slower than a Postgres-only path — keep +# this small; a request that misses this timeout just falls through to Postgres. +# ATOM_CACHE_OP_TIMEOUT_MS=50 +# If true, an unreachable Redis at startup aborts boot (like an unreachable +# Postgres does). If false (default), Atom logs an error and starts anyway +# with caching disabled — recommended, since Redis is a performance +# optimization for reads, not a correctness dependency. +# ATOM_CACHE_FAIL_FAST_ON_STARTUP=false +# Per-category entry TTLs (seconds) — defense-in-depth only, see above. +# ATOM_CACHE_TTL_SESSION_SECS=60 +# ATOM_CACHE_TTL_ENTITY_STATUS_SECS=60 +# ATOM_CACHE_TTL_TENANT_STATUS_SECS=60 +# ATOM_CACHE_TTL_CREDENTIAL_SECS=60 +# ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS=60 +# ATOM_CACHE_TTL_GRANTS_SECS=60 +# Redis should be network-private (not publicly reachable) and, in any real +# deployment, protected with auth/TLS appropriate to your network — the +# docker-compose `redis` service here is dev-only (loopback-bound, no auth). + # --- Audit retention and abuse controls -------------------------------- ATOM_AUDIT_RETENTION_DAYS=365 ATOM_AUDIT_RETENTION_ENABLED=true diff --git a/Cargo.lock b/Cargo.lock index d22a1a4..f9068ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,6 +113,12 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "argon2" version = "0.5.3" @@ -426,6 +432,7 @@ dependencies = [ "axum", "base64 0.22.1", "chrono", + "deadpool-redis", "dotenvy", "hex", "insta", @@ -443,6 +450,7 @@ dependencies = [ "prost", "rand 0.8.6", "rcgen", + "redis", "ring", "serde", "serde_json", @@ -806,7 +814,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -1018,6 +1030,36 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafa30c49dafe086d10116074e422ad7fc1c3cf554697e744a3ab112599ebd09" +dependencies = [ + "deadpool", + "redis", +] + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + [[package]] name = "der" version = "0.7.10" @@ -2594,6 +2636,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oauth2" version = "5.0.0" @@ -3298,6 +3350,30 @@ dependencies = [ "yasna", ] +[[package]] +name = "redis" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3257df217f7eab0044627a268c9cc6cdb60c0c421c88f83ac41c4e31520b6b84" +dependencies = [ + "arcstr", + "async-lock", + "bytes", + "cfg-if", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2 0.6.3", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3769,6 +3845,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5483,6 +5565,12 @@ dependencies = [ "time", ] +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yasna" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index f3115ba..5124b94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,8 @@ lapin = { version = "4.10.0", default-features = false, features = [ "rustls--ring", "rustls-webpki-roots-certs", ] } +redis = { version = "1", default-features = false, features = ["script"] } +deadpool-redis = { version = "0.23", default-features = false, features = ["rt_tokio_1"] } [features] # Metrics are on by default. Disable at compile time for maximum-performance diff --git a/docker-compose.yml b/docker-compose.yml index e4eb3ae..c47e5a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,20 @@ services: timeout: 5s retries: 5 + # Redis-backed AuthN/AuthZ cache (see src/cache/mod.rs). Off by default + # (ATOM_CACHE_ENABLED=false) — a pure performance optimization, Postgres + # remains authoritative. Pure cache, no persistence needed. + redis: + image: redis:7-alpine + command: ["redis-server", "--save", "", "--appendonly", "no"] + ports: + - "127.0.0.1:${ATOM_CACHE_REDIS_PORT:-6379}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + atom: image: ${ATOM_IMAGE:-ghcr.io/absmach/atom:latest} build: @@ -88,6 +102,20 @@ services: ATOM_LOG_LEVEL: ${ATOM_LOG_LEVEL:-${RUST_LOG:-info}} ATOM_LOG_FORMAT: ${ATOM_LOG_FORMAT:-text} ATOM_EMAIL_TEMPLATES_DIR: ${ATOM_EMAIL_TEMPLATES_DIR:-/email-templates} + # Off by default. Set ATOM_CACHE_ENABLED=true (and add `redis` to + # depends_on below) once Redis is provisioned — see .env.example. + ATOM_CACHE_ENABLED: ${ATOM_CACHE_ENABLED:-false} + ATOM_CACHE_REDIS_URL: ${ATOM_CACHE_REDIS_URL:-redis://redis:6379/0} + ATOM_CACHE_POOL_MAX_SIZE: ${ATOM_CACHE_POOL_MAX_SIZE:-20} + ATOM_CACHE_CONNECT_TIMEOUT_MS: ${ATOM_CACHE_CONNECT_TIMEOUT_MS:-2000} + ATOM_CACHE_OP_TIMEOUT_MS: ${ATOM_CACHE_OP_TIMEOUT_MS:-50} + ATOM_CACHE_FAIL_FAST_ON_STARTUP: ${ATOM_CACHE_FAIL_FAST_ON_STARTUP:-false} + ATOM_CACHE_TTL_SESSION_SECS: ${ATOM_CACHE_TTL_SESSION_SECS:-60} + ATOM_CACHE_TTL_ENTITY_STATUS_SECS: ${ATOM_CACHE_TTL_ENTITY_STATUS_SECS:-60} + ATOM_CACHE_TTL_TENANT_STATUS_SECS: ${ATOM_CACHE_TTL_TENANT_STATUS_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS:-60} + ATOM_CACHE_TTL_GRANTS_SECS: ${ATOM_CACHE_TTL_GRANTS_SECS:-60} volumes: - ${ATOM_CERTS_CA_DIR:-./certs}:/certs:ro - ${ATOM_EMAIL_TEMPLATES_HOST_DIR:-./email-templates}:/email-templates:ro @@ -175,6 +203,18 @@ services: ATOM_LOG_LEVEL: ${ATOM_LOG_LEVEL:-${RUST_LOG:-info}} ATOM_LOG_FORMAT: ${ATOM_LOG_FORMAT:-text} ATOM_EMAIL_TEMPLATES_DIR: ${ATOM_EMAIL_TEMPLATES_DIR:-/email-templates} + ATOM_CACHE_ENABLED: ${ATOM_CACHE_ENABLED:-false} + ATOM_CACHE_REDIS_URL: ${ATOM_CACHE_REDIS_URL:-redis://redis:6379/0} + ATOM_CACHE_POOL_MAX_SIZE: ${ATOM_CACHE_POOL_MAX_SIZE:-20} + ATOM_CACHE_CONNECT_TIMEOUT_MS: ${ATOM_CACHE_CONNECT_TIMEOUT_MS:-2000} + ATOM_CACHE_OP_TIMEOUT_MS: ${ATOM_CACHE_OP_TIMEOUT_MS:-50} + ATOM_CACHE_FAIL_FAST_ON_STARTUP: ${ATOM_CACHE_FAIL_FAST_ON_STARTUP:-false} + ATOM_CACHE_TTL_SESSION_SECS: ${ATOM_CACHE_TTL_SESSION_SECS:-60} + ATOM_CACHE_TTL_ENTITY_STATUS_SECS: ${ATOM_CACHE_TTL_ENTITY_STATUS_SECS:-60} + ATOM_CACHE_TTL_TENANT_STATUS_SECS: ${ATOM_CACHE_TTL_TENANT_STATUS_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_SECS:-60} + ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS: ${ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS:-60} + ATOM_CACHE_TTL_GRANTS_SECS: ${ATOM_CACHE_TTL_GRANTS_SECS:-60} volumes: - ${ATOM_CERTS_CA_DIR:-./certs}:/certs:ro - ${ATOM_EMAIL_TEMPLATES_HOST_DIR:-./email-templates}:/email-templates:ro diff --git a/src/cache/entries.rs b/src/cache/entries.rs new file mode 100644 index 0000000..0860ec1 --- /dev/null +++ b/src/cache/entries.rs @@ -0,0 +1,56 @@ +//! Cache-transport DTOs. Deliberately distinct from the DB row models in +//! `crate::models` — each holds only the fields its read path actually needs, +//! never a full row dump, and never a plaintext secret. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::models::enums::{CredentialStatus, EntityStatus, TenantStatus}; + +/// The current SQL check backing `auth_from_jwt` verifies both the session id +/// *and* that it belongs to the claimed entity (`WHERE s.id = $1 AND +/// s.entity_id = $2`). `entity_id` is carried here so a cache hit can +/// re-verify that same invariant against the JWT's `sub` claim, rather than +/// trusting the key lookup alone. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SessionCacheEntry { + pub entity_id: Uuid, + pub revoked_at: Option>, + pub expires_at: DateTime, +} + +/// Shared between JWT and API-key authentication — one entity deactivation +/// invalidates both paths' view of the entity at once. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EntityStatusCacheEntry { + pub status: EntityStatus, + pub deleted_at: Option>, + pub tenant_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TenantStatusCacheEntry { + pub status: TenantStatus, + pub deleted_at: Option>, +} + +/// Never carries the plaintext API-key secret — only what +/// `auth_from_api_key`'s existing verification step consumes. +/// +/// Deliberately has no `tenant_id` field: a credential's tenant is really the +/// owning entity's tenant, which can change (entity moved to another tenant) +/// independently of this entry ever being invalidated. A duplicated copy here +/// would go stale on that move with nothing to invalidate it, and +/// `auth_from_api_key` would have no way to know. Always read tenant context +/// from the entity's own `EntityStatusCacheEntry` instead, which *is* +/// invalidated on a tenant move. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CredentialCacheEntry { + pub entity_id: Uuid, + pub status: CredentialStatus, + pub secret_hash: Option, + pub secret_lookup_hash: Option>, + pub expires_at: Option>, + pub scoped: bool, +} diff --git a/src/cache/invalidate.rs b/src/cache/invalidate.rs new file mode 100644 index 0000000..347d623 --- /dev/null +++ b/src/cache/invalidate.rs @@ -0,0 +1,84 @@ +//! The single orchestration primitive every security-sensitive mutation goes +//! through: establish the barrier on the affected cache keys, run the +//! Postgres mutation, then clear the barrier — regardless of the mutation's +//! outcome. See `src/cache/mod.rs` for the consistency model this +//! implements. +//! +//! Callers determine *which* keys are affected (a single subject, or an +//! enumerated set — see `authz::repo::affected_subject_ids_for_role` / +//! `affected_subject_ids_for_group`) before calling this; the enumeration +//! itself is domain SQL and does not belong here. + +use std::future::Future; + +use super::{CacheCategory, CacheClient}; +use crate::error::AppError; + +/// Runs `mutate` guarded by a cache barrier on `keys`. With caching disabled +/// (`cache: None`) this is a pure passthrough to `mutate`, byte-identical to +/// not having a cache at all. +/// +/// If establishing the barrier fails while caching is enabled, the mutation +/// is refused (`mutate` is never called) rather than committing a Postgres +/// change the cache cannot be told about — see the module-level consistency +/// model in `src/cache/mod.rs`. +pub async fn guarded_mutation( + cache: Option<&CacheClient>, + category: CacheCategory, + keys: &[String], + mutate: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + let Some(cache) = cache else { + return mutate().await; + }; + + cache.begin(category, keys).await?; + let result = mutate().await; + cache.end(category, keys).await; + result +} + +/// Like [`guarded_mutation`], for a mutation whose effects span more than one +/// cache category in a single Postgres transaction — e.g. tenant restore, +/// which both flips the tenant's own status and reactivates a set of +/// credentials. Establishes a barrier on every `(category, keys)` group +/// before running `mutate`; if a later group's barrier can't be established, +/// the ones already established are cleared immediately (rather than left to +/// self-heal on their barrier TTL) before the mutation is refused. Every +/// established group is cleared after `mutate` regardless of outcome. +pub async fn guarded_multi_mutation( + cache: Option<&CacheClient>, + groups: &[(CacheCategory, &[String])], + mutate: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: Future>, +{ + let Some(cache) = cache else { + return mutate().await; + }; + + let mut established = Vec::with_capacity(groups.len()); + for &(category, keys) in groups { + match cache.begin(category, keys).await { + Ok(()) => established.push((category, keys)), + Err(err) => { + for (category, keys) in established { + cache.end(category, keys).await; + } + return Err(err); + } + } + } + + let result = mutate().await; + for (category, keys) in established { + cache.end(category, keys).await; + } + result +} diff --git a/src/cache/keys.rs b/src/cache/keys.rs new file mode 100644 index 0000000..564b562 --- /dev/null +++ b/src/cache/keys.rs @@ -0,0 +1,30 @@ +//! Pure key-naming functions. Namespaced `atom:v1:` so an incompatible future +//! payload-shape change can roll out as `atom:v2:` without needing a flush. + +use uuid::Uuid; + +const NAMESPACE: &str = "atom:v1"; + +pub fn session(session_id: Uuid) -> String { + format!("{NAMESPACE}:session:{session_id}") +} + +pub fn entity_status(entity_id: Uuid) -> String { + format!("{NAMESPACE}:entity_status:{entity_id}") +} + +pub fn tenant_status(tenant_id: Uuid) -> String { + format!("{NAMESPACE}:tenant_status:{tenant_id}") +} + +pub fn credential(credential_id: Uuid) -> String { + format!("{NAMESPACE}:credential:{credential_id}") +} + +pub fn cred_ceiling(credential_id: Uuid) -> String { + format!("{NAMESPACE}:cred_ceiling:{credential_id}") +} + +pub fn grants(subject_id: Uuid) -> String { + format!("{NAMESPACE}:grants:{subject_id}") +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs new file mode 100644 index 0000000..2c67eda --- /dev/null +++ b/src/cache/mod.rs @@ -0,0 +1,881 @@ +//! Redis-backed cache for AuthN/AuthZ decision *inputs* (never decisions +//! themselves — see `src/authz/engine.rs` and `src/auth.rs` for what actually +//! decides allow/deny). Postgres remains the sole source of truth. +//! +//! # Consistency model +//! +//! Every cached entry is a Redis hash with three fields: `v` (an integer +//! version, bumped on every mutation that can affect the entry), `dirty` +//! (`"1"` while a mutation is in flight, absent otherwise), and `p` (the +//! serialized payload, present only when the entry holds a valid value). +//! +//! Three primitives, each a small atomic Lua script, implement a per-key +//! mutation barrier that prevents three races that a plain cache-aside + +//! post-commit `DEL` cannot: a read started *before* a mutation repopulating +//! the cache with stale data after the mutation's invalidation ran, a read +//! started *during* the mutation's dirty window doing the same once the +//! mutation finishes, and a lost invalidation silently resurrecting a +//! revoked value: +//! +//! - `begin` — called before a security-sensitive Postgres mutation. Bumps the +//! version, marks the entry dirty, clears any payload, and bounds the +//! barrier itself with an expiry so a lost `end` call self-heals rather than +//! leaving the entry dirty forever. +//! - `end` — called after the mutation (success or failure). Bumps the +//! version *again* and clears `dirty`. Never restores a payload — the next +//! reader does a clean reload. The second version bump (beyond the one +//! `begin` already did) is what closes the dirty-window race below — it is +//! not merely resetting a flag. +//! - `try_populate` — called by a cache-miss read after it finishes loading +//! from Postgres. Writes the payload only if the entry is not dirty and its +//! version still matches what the reader observed before it started +//! loading; otherwise the write is silently discarded. +//! +//! The dirty-window race this second bump defeats (found by external review, +//! 2026-07-29 — the original `end` only cleared `dirty`, without +//! re-bumping the version): a reader's `lookup` can land *while* a mutation +//! is mid-flight (`dirty == "1"`), observe the *post-`begin`* version, then +//! proceed to load from Postgres — possibly reading the pre-mutation state, +//! since the mutation's own Postgres write may not have committed yet. If +//! `end` only cleared `dirty` without moving the version again, that +//! reader's later `try_populate` call, run after `end`, would find the +//! version *unchanged* since the moment it was observed and would succeed — +//! re-caching a stale value for the mutation's category for a full TTL, +//! exactly during a revoke/policy-change race. Bumping the version in `end` +//! too means any version a reader could have observed during the dirty +//! window is guaranteed stale by the time `end` finishes, so `try_populate` +//! always rejects it — whether it runs while still dirty (rejected by the +//! `dirty` check) or after `end` (rejected by the version check). +//! +//! Reads never depend on Redis being reachable: any error (timeout, +//! connection failure, corrupt payload) is treated as a miss and falls +//! through to the caller's Postgres loader. `begin` is the one exception — +//! while caching is enabled, a `begin` failure refuses the mutation rather +//! than committing a change the cache cannot be told about (see +//! `src/cache/invalidate.rs`). + +pub mod entries; +pub mod invalidate; +pub mod keys; + +use std::{future::Future, time::Duration}; + +use deadpool_redis::{Config as PoolConfig, Pool, Runtime}; +use redis::AsyncCommands; +use serde::{de::DeserializeOwned, Serialize}; +use thiserror::Error; + +use crate::{ + config::{CacheConfig, CacheTtlConfig}, + error::AppError, + metrics, +}; + +/// Keys are chunked so a single Lua invocation never touches an unbounded +/// number of hash keys in one round trip. +const BULK_CHUNK_SIZE: usize = 500; + +const BEGIN_SCRIPT_SRC: &str = r#" +local ttl_ms = ARGV[1] +for i, key in ipairs(KEYS) do + redis.call('HINCRBY', key, 'v', 1) + redis.call('HSET', key, 'dirty', '1') + redis.call('HDEL', key, 'p') + redis.call('PEXPIRE', key, ttl_ms) +end +return 1 +"#; + +const END_SCRIPT_SRC: &str = r#" +for i, key in ipairs(KEYS) do + redis.call('HINCRBY', key, 'v', 1) + redis.call('HSET', key, 'dirty', '0') +end +return 1 +"#; + +const TRY_POPULATE_SCRIPT_SRC: &str = r#" +local v = redis.call('HGET', KEYS[1], 'v') +if v == false then v = '0' end +local dirty = redis.call('HGET', KEYS[1], 'dirty') +if dirty == false then dirty = '0' end +if dirty == '1' or v ~= ARGV[1] then + return 'stale' +end +redis.call('HSET', KEYS[1], 'p', ARGV[2]) +redis.call('PEXPIRE', KEYS[1], ARGV[3]) +return 'applied' +"#; + +/// Fixed, low-cardinality label for cache metrics and log lines. Never an ID, +/// action name, or arbitrary string — see `src/metrics.rs`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CacheCategory { + Session, + EntityStatus, + TenantStatus, + Credential, + CredentialCeiling, + Grants, +} + +impl CacheCategory { + pub fn as_str(self) -> &'static str { + match self { + Self::Session => "session", + Self::EntityStatus => "entity_status", + Self::TenantStatus => "tenant_status", + Self::Credential => "credential", + Self::CredentialCeiling => "credential_ceiling", + Self::Grants => "grants", + } + } + + /// The configured entry TTL for this category — resolved here, once, so + /// callers never need to thread a `Duration` through read/write-path call + /// sites themselves. + fn ttl(self, cfg: &CacheTtlConfig) -> Duration { + let secs = match self { + Self::Session => cfg.session_secs, + Self::EntityStatus => cfg.entity_status_secs, + Self::TenantStatus => cfg.tenant_status_secs, + Self::Credential => cfg.credential_secs, + Self::CredentialCeiling => cfg.credential_ceiling_secs, + Self::Grants => cfg.grants_secs, + }; + Duration::from_secs(secs) + } +} + +/// The outcome of a cache read: a valid, non-dirty entry, or a miss carrying +/// the version a subsequent `try_populate` must present unchanged to write +/// successfully. `Unavailable` means the read itself failed (timeout, +/// connection error) — callers must fall through to Postgres and must not +/// attempt to populate afterward (the write would just fail too). +#[derive(Debug)] +pub enum Lookup { + Hit(T), + Miss { version: i64 }, + Unavailable, +} + +#[derive(Debug, Error)] +pub enum CacheError { + #[error("cache operation timed out")] + Timeout, + #[error("cache pool error: {0}")] + Pool(#[from] deadpool_redis::PoolError), + #[error("cache redis error: {0}")] + Redis(#[from] redis::RedisError), +} + +#[derive(Debug)] +pub struct CacheClient { + pool: Pool, + op_timeout: Duration, + ttl: CacheTtlConfig, + begin_script: redis::Script, + end_script: redis::Script, + try_populate_script: redis::Script, +} + +impl CacheClient { + /// Connects and verifies reachability with a single `PING`, bounded by + /// `cfg.connect_timeout`. Callers decide what to do with a connect + /// failure (fail fast vs. degrade) — see `main.rs`. + pub async fn connect(cfg: &CacheConfig) -> anyhow::Result { + let pool_cfg = PoolConfig::from_url(&cfg.redis_url); + let mut pool_builder = pool_cfg + .builder() + .map_err(|e| anyhow::anyhow!("invalid ATOM_CACHE_REDIS_URL: {e}"))?; + pool_builder = pool_builder.max_size(cfg.pool_max_size as usize); + let pool = pool_builder + .runtime(Runtime::Tokio1) + .build() + .map_err(|e| anyhow::anyhow!("failed to build cache pool: {e}"))?; + + let client = Self { + pool, + op_timeout: Duration::from_millis(cfg.op_timeout_ms), + ttl: cfg.ttl, + begin_script: redis::Script::new(BEGIN_SCRIPT_SRC), + end_script: redis::Script::new(END_SCRIPT_SRC), + try_populate_script: redis::Script::new(TRY_POPULATE_SCRIPT_SRC), + }; + + tokio::time::timeout(Duration::from_millis(cfg.connect_timeout_ms), client.ping()) + .await + .map_err(|_| anyhow::anyhow!("cache connect timed out"))? + .map_err(|e| anyhow::anyhow!("cache connect failed: {e}"))?; + + Ok(client) + } + + pub async fn ping(&self) -> Result<(), CacheError> { + let mut conn = self.get_conn().await?; + redis::cmd("PING") + .query_async::(&mut conn) + .await + .map_err(CacheError::from)?; + Ok(()) + } + + async fn get_conn(&self) -> Result { + tokio::time::timeout(self.op_timeout, self.pool.get()) + .await + .map_err(|_| CacheError::Timeout)? + .map_err(CacheError::from) + } + + /// Single-key read. Never errors outward — a failure of any kind becomes + /// `Lookup::Unavailable`. + pub async fn lookup( + &self, + category: CacheCategory, + key: &str, + ) -> Lookup { + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache lookup unavailable"); + metrics::record_cache_lookup(category.as_str(), "error"); + return Lookup::Unavailable; + } + }; + + let result = tokio::time::timeout( + self.op_timeout, + redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async::<(Option, Option, Option>)>(&mut conn), + ) + .await; + + let (version, dirty, payload) = match result { + Ok(Ok(fields)) => fields, + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache lookup failed"); + metrics::record_cache_lookup(category.as_str(), "error"); + return Lookup::Unavailable; + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache lookup timed out"); + metrics::record_cache_lookup(category.as_str(), "error"); + return Lookup::Unavailable; + } + }; + + let observed_version = version.unwrap_or(0); + let is_dirty = dirty.as_deref() == Some("1"); + + let Some(payload) = payload.filter(|_| !is_dirty) else { + metrics::record_cache_lookup(category.as_str(), "miss"); + return Lookup::Miss { + version: observed_version, + }; + }; + + match serde_json::from_slice::(&payload) { + Ok(value) => { + metrics::record_cache_lookup(category.as_str(), "hit"); + Lookup::Hit(value) + } + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache payload corrupt; discarding"); + self.best_effort_delete(key).await; + metrics::record_cache_lookup(category.as_str(), "miss"); + Lookup::Miss { + version: observed_version, + } + } + } + } + + async fn best_effort_delete(&self, key: &str) { + if let Ok(mut conn) = self.get_conn().await { + let _: Result<(), redis::RedisError> = conn.del(key).await; + } + } + + /// Best-effort conditional write following a cache-miss load. Discarded + /// silently if the entry became dirty or its version moved on since the + /// caller observed `expected_version` — see the module docs. + pub async fn try_populate( + &self, + category: CacheCategory, + key: &str, + expected_version: i64, + value: &T, + ) { + let Ok(payload) = serde_json::to_vec(value) else { + tracing::warn!( + category = category.as_str(), + "cache payload serialize failed" + ); + return; + }; + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(_) => return, + }; + let ttl = category.ttl(&self.ttl); + let outcome = tokio::time::timeout( + self.op_timeout, + self.try_populate_script + .key(key) + .arg(expected_version) + .arg(payload) + .arg(ttl.as_millis() as i64) + .invoke_async::(&mut conn), + ) + .await; + if let Err(err) = outcome { + tracing::warn!(category = category.as_str(), error = %err, "cache populate timed out"); + } else if let Ok(Err(err)) = outcome { + tracing::warn!(category = category.as_str(), error = %err, "cache populate failed"); + } + } + + /// Cache-aside read with a fallback loader: a hit returns immediately, a + /// miss loads via `loader` and best-effort populates the cache, and an + /// unavailable cache falls straight through to `loader`. + pub async fn get_or_load( + &self, + category: CacheCategory, + key: &str, + loader: F, + ) -> Result + where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: Future>, + { + match self.lookup::(category, key).await { + Lookup::Hit(value) => Ok(value), + Lookup::Miss { version } => { + let value = loader().await?; + self.try_populate(category, key, version, &value).await; + Ok(value) + } + Lookup::Unavailable => loader().await, + } + } + + /// Marks `keys` dirty before a security-sensitive Postgres mutation. + /// **Fails the caller** if the barrier cannot be established (Redis + /// unreachable/timeout) — see module docs and `src/cache/invalidate.rs`. + /// A no-op that always succeeds when `keys` is empty. + pub async fn begin(&self, category: CacheCategory, keys: &[String]) -> Result<(), AppError> { + if keys.is_empty() { + return Ok(()); + } + let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); + for chunk in keys.chunks(BULK_CHUNK_SIZE) { + let mut conn = self.get_conn().await.map_err(|err| { + tracing::warn!(category = category.as_str(), error = %err, "cache begin: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + AppError::service_unavailable( + "cache unavailable; refusing security-sensitive mutation", + ) + })?; + + let mut invocation = self.begin_script.prepare_invoke(); + for key in chunk { + invocation.key(key); + } + invocation.arg(barrier_ttl.as_millis() as i64); + + let outcome = + tokio::time::timeout(self.op_timeout, invocation.invoke_async::(&mut conn)) + .await; + match outcome { + Ok(Ok(_)) => {} + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache begin failed"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return Err(AppError::service_unavailable( + "cache unavailable; refusing security-sensitive mutation", + )); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache begin timed out"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return Err(AppError::service_unavailable( + "cache unavailable; refusing security-sensitive mutation", + )); + } + } + } + metrics::record_cache_invalidation(category.as_str(), "ok"); + Ok(()) + } + + /// Bumps the version and clears the dirty marker on `keys` after the + /// mutation (success or failure). Always best-effort — never fails the + /// caller. Left dirty entries self-heal once the barrier TTL set by + /// `begin` expires. The version bump (not just the dirty clear) is what + /// stops a reader whose `lookup` landed during the dirty window from + /// repopulating a stale value afterward — see the module docs. + pub async fn end(&self, category: CacheCategory, keys: &[String]) { + if keys.is_empty() { + return; + } + for chunk in keys.chunks(BULK_CHUNK_SIZE) { + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache end: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + continue; + } + }; + let mut invocation = self.end_script.prepare_invoke(); + for key in chunk { + invocation.key(key); + } + let outcome = + tokio::time::timeout(self.op_timeout, invocation.invoke_async::(&mut conn)) + .await; + match outcome { + Ok(Ok(_)) => metrics::record_cache_invalidation(category.as_str(), "ok"), + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache end failed"); + metrics::record_cache_invalidation(category.as_str(), "error"); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache end timed out"); + metrics::record_cache_invalidation(category.as_str(), "error"); + } + } + } + } +} + +/// The barrier key's own expiry: long enough to comfortably outlast any +/// realistic Postgres mutation + `end` call, so a lost `end` self-heals by +/// the whole entry expiring outright rather than staying dirty forever. +fn barrier_ttl(entry_ttl: Duration) -> Duration { + entry_ttl * 5 +} + +/// `get_or_load`, but tolerant of caching being disabled entirely — the +/// common entry point for read paths, so call sites don't need to branch on +/// `Option<&CacheClient>` themselves. +pub async fn cached_or_load( + cache: Option<&CacheClient>, + category: CacheCategory, + key: &str, + loader: F, +) -> Result +where + T: Serialize + DeserializeOwned, + F: FnOnce() -> Fut, + Fut: Future>, +{ + match cache { + Some(cache) => cache.get_or_load(category, key, loader).await, + None => loader().await, + } +} + +/// Redis-gated unit tests for the cache mechanism itself — key formatting, +/// serialization round trips, and the barrier primitives in isolation from +/// any AuthN/AuthZ call site. Requires `ATOM_TEST_REDIS_URL`; run with +/// `ATOM_TEST_REDIS_URL=redis://... cargo test -- --ignored`. +#[cfg(test)] +mod tests { + use super::*; + use serde::{Deserialize, Serialize}; + use uuid::Uuid; + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + struct Payload { + value: String, + } + + async fn test_client() -> CacheClient { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + let cfg = CacheConfig { + enabled: true, + redis_url: url, + ..CacheConfig::default() + }; + CacheClient::connect(&cfg) + .await + .expect("connect to test redis") + } + + fn unique_key(label: &str) -> String { + format!("atom:v1:test:{label}:{}", Uuid::new_v4()) + } + + /// Drives `main.rs`'s fail-fast-vs-degrade startup branching: `connect` + /// must return `Err` (not hang, not panic) against an unreachable Redis, + /// bounded by `connect_timeout_ms`. No live Redis needed — this + /// deliberately never reaches one — so it runs in default `cargo test`. + #[tokio::test] + async fn connect_fails_against_an_unreachable_redis() { + let cfg = CacheConfig { + enabled: true, + redis_url: "redis://127.0.0.1:1/0".into(), + connect_timeout_ms: 200, + op_timeout_ms: 50, + ..CacheConfig::default() + }; + let result = CacheClient::connect(&cfg).await; + assert!( + result.is_err(), + "connect must fail against an unreachable redis, not hang or succeed" + ); + } + + #[tokio::test] + #[ignore] + async fn ping_succeeds_against_reachable_redis() { + let client = test_client().await; + client.ping().await.expect("ping"); + } + + #[tokio::test] + #[ignore] + async fn lookup_on_absent_key_is_a_clean_miss_at_version_zero() { + let client = test_client().await; + let key = unique_key("lookup-miss"); + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => assert_eq!(version, 0), + other => panic!("expected a clean miss, got {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn try_populate_then_lookup_round_trips_the_payload() { + let client = test_client().await; + let key = unique_key("roundtrip"); + let value = Payload { + value: "hello".into(), + }; + + let version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected miss before populate, got {other:?}"), + }; + client + .try_populate(CacheCategory::Grants, &key, version, &value) + .await; + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_eq!(got, value), + other => panic!("expected a hit after populate, got {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn corrupt_payload_is_treated_as_a_miss_and_deleted() { + let client = test_client().await; + let key = unique_key("corrupt"); + + // Write a payload that isn't valid JSON for `Payload` directly into + // the hash, bypassing `try_populate`, to simulate corruption. + let mut conn = client.get_conn().await.expect("conn"); + let _: () = redis::cmd("HSET") + .arg(&key) + .arg("v") + .arg(1) + .arg("p") + .arg("not valid json") + .query_async(&mut conn) + .await + .expect("seed corrupt payload"); + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + other => panic!("expected corrupt payload to be a miss, got {other:?}"), + } + + // The corrupt entry must have been best-effort deleted so the next + // lookup doesn't repeat the same deserialize failure. + let exists: bool = redis::cmd("EXISTS") + .arg(&key) + .query_async(&mut conn) + .await + .expect("exists check"); + assert!(!exists, "corrupt entry should have been deleted"); + } + + #[tokio::test] + #[ignore] + async fn try_populate_rejects_a_stale_version() { + let client = test_client().await; + let key = unique_key("stale-version"); + let keys = vec![key.clone()]; + + // A concurrent mutation bumps the version between the reader's + // initial lookup and its populate attempt. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + client.end(CacheCategory::Grants, &keys).await; + + // The reader observed version 0 (before the mutation) and now tries + // to populate with that stale value. + let stale_value = Payload { + value: "stale".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, 0, &stale_value) + .await; + + // Must still be a miss — the stale write was discarded. + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + Lookup::Hit(got) => panic!("stale write should have been rejected, got {got:?}"), + other => panic!("unexpected lookup outcome: {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn dirty_entry_is_never_served_as_a_hit() { + let client = test_client().await; + let key = unique_key("dirty"); + let keys = vec![key.clone()]; + + let version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected miss, got {other:?}"), + }; + let value = Payload { + value: "before-mutation".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, version, &value) + .await; + // Confirm it's actually cached before dirtying it. + assert!(matches!( + client.lookup::(CacheCategory::Grants, &key).await, + Lookup::Hit(_) + )); + + // `begin` marks it dirty and clears the payload without an `end`. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { .. } => {} + Lookup::Hit(got) => panic!("dirty entry served as a hit: {got:?}"), + other => panic!("unexpected outcome: {other:?}"), + } + } + + #[tokio::test] + #[ignore] + async fn bulk_begin_and_end_cover_every_key_in_one_round_trip() { + let client = test_client().await; + let keys: Vec = (0..5).map(|i| unique_key(&format!("bulk-{i}"))).collect(); + + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + for key in &keys { + let (_, dirty, _): (Option, Option, Option>) = redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut client.get_conn().await.expect("conn")) + .await + .expect("hmget"); + assert_eq!( + dirty.as_deref(), + Some("1"), + "key {key} should be dirty after begin" + ); + } + + client.end(CacheCategory::Grants, &keys).await; + for key in &keys { + let (_, dirty, _): (Option, Option, Option>) = redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut client.get_conn().await.expect("conn")) + .await + .expect("hmget"); + assert_eq!( + dirty.as_deref(), + Some("0"), + "key {key} should be clean after end" + ); + } + } + + #[tokio::test] + #[ignore] + async fn get_or_load_hits_cache_on_second_call_without_invoking_loader() { + let client = test_client().await; + let key = unique_key("get-or-load"); + let calls = std::sync::atomic::AtomicUsize::new(0); + + let first: Payload = client + .get_or_load(CacheCategory::Grants, &key, || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async { + Ok(Payload { + value: "loaded".into(), + }) + } + }) + .await + .expect("first load"); + assert_eq!(first.value, "loaded"); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1); + + let second: Payload = client + .get_or_load(CacheCategory::Grants, &key, || { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + async { + Ok(Payload { + value: "should-not-be-called".into(), + }) + } + }) + .await + .expect("second load"); + assert_eq!( + second.value, "loaded", + "second call must be served from cache" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "loader must not run again on a cache hit" + ); + } + + /// The test that actually validates the consistency model, not just its + /// individual primitives: a reader that captured a version *before* a + /// concurrent mutation must not be able to repopulate the cache with its + /// now-stale result *after* the mutation commits and clears the barrier. + /// This is the exact race described in `src/cache/mod.rs`'s module docs. + #[tokio::test] + #[ignore] + async fn stale_reader_cannot_repopulate_after_a_concurrent_mutation_completes() { + let client = test_client().await; + let key = unique_key("race"); + let keys = vec![key.clone()]; + + // Reader observes the version before any mutation has happened. + let observed_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!("expected initial miss, got {other:?}"), + }; + + // A concurrent mutation runs to completion while the reader's + // (simulated) Postgres load is still in flight: begin bumps the + // version and marks dirty, then end clears dirty after the mutation + // commits. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + client.end(CacheCategory::Grants, &keys).await; + + // The reader's stale load now finishes and attempts to populate with + // the version it observed *before* the mutation. + let stale_value = Payload { + value: "STALE — must never be visible".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, observed_version, &stale_value) + .await; + + // The cache must not have been poisoned with the stale value — the + // next reader must see a clean miss (correctness bound: even though + // the mutation revealed no new payload of its own, the stale write + // must never have applied), never the disallowed value. + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "stale reader was able to poison the cache after a concurrent mutation" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + } + + /// Regression test for a review finding: the test above only covers a + /// reader whose `lookup` landed *before* `begin` ran (a version `begin` + /// then bumps past). It does not cover a reader whose `lookup` lands + /// *during* the dirty window itself (after `begin`, before `end`) — that + /// reader observes the *post-`begin`* version, and if `end` only cleared + /// `dirty` without bumping the version again, that same version would + /// still match after `end`, so `try_populate` would wrongly accept a + /// value the reader may have loaded from Postgres before the mutation's + /// own write committed. This is the actual scenario the review + /// described: "a reader that starts during a mutation" — not one that + /// started before it. + #[tokio::test] + #[ignore] + async fn stale_reader_cannot_repopulate_during_the_mutations_dirty_window() { + let client = test_client().await; + let key = unique_key("dirty-window-race"); + let keys = vec![key.clone()]; + + // The mutation begins — the key is now dirty. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + // A reader's `lookup` lands *while the mutation is still in + // flight* — this is the case the previous test didn't cover. + let dirty_window_version = match client.lookup::(CacheCategory::Grants, &key).await + { + Lookup::Miss { version } => version, + other => panic!("expected a miss while dirty, got {other:?}"), + }; + + // The mutation finishes — `end` clears dirty (and, with the fix, + // bumps the version again). + client.end(CacheCategory::Grants, &keys).await; + + // The reader's (simulated) Postgres load — possibly stale, since it + // may have run before the mutation's own write committed — finishes + // and attempts to populate using the version it observed *during* + // the dirty window. + let stale_value = Payload { + value: "STALE — read during the dirty window, must never be visible".into(), + }; + client + .try_populate( + CacheCategory::Grants, + &key, + dirty_window_version, + &stale_value, + ) + .await; + + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "a reader that started during the mutation's dirty window was able to poison \ + the cache once the mutation completed — this is exactly the P1 the barrier \ + model exists to prevent, and means `end` isn't invalidating a dirty-window \ + read's captured version" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + } +} diff --git a/src/config.rs b/src/config.rs index e4a46ac..dd0bdf8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -75,6 +75,7 @@ pub struct Config { pub certs_root_ca_key_path: Option, pub certs_leaf_default_ttl_secs: u64, pub certs_leaf_max_ttl_secs: u64, + pub cache: CacheConfig, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -100,6 +101,65 @@ impl Default for DbPoolConfig { } } +/// Per-category TTLs, applied to cached entries as a defense-in-depth safety +/// net (not the primary invalidation mechanism — see `src/cache/mod.rs`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheTtlConfig { + pub session_secs: u64, + pub entity_status_secs: u64, + pub tenant_status_secs: u64, + pub credential_secs: u64, + pub credential_ceiling_secs: u64, + pub grants_secs: u64, +} + +impl Default for CacheTtlConfig { + fn default() -> Self { + Self { + session_secs: 60, + entity_status_secs: 60, + tenant_status_secs: 60, + credential_secs: 60, + credential_ceiling_secs: 60, + grants_secs: 60, + } + } +} + +/// Redis-backed cache for AuthN/AuthZ decision inputs. Off by default — this +/// is a pure performance optimization; every check works correctly with it +/// disabled, since Postgres remains authoritative. See `src/cache/mod.rs` for +/// the consistency model this configures. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheConfig { + pub enabled: bool, + pub redis_url: String, + pub pool_max_size: u32, + pub connect_timeout_ms: u64, + pub op_timeout_ms: u64, + /// When `enabled` and Redis is unreachable at startup: abort like an + /// unreachable Postgres would (`true`), or log and continue with caching + /// disabled (`false`, the recommended default — Redis is not a + /// correctness dependency for reads, only, while enabled, a write-path + /// dependency for security-sensitive mutations; see `src/cache/mod.rs`). + pub fail_fast_on_startup: bool, + pub ttl: CacheTtlConfig, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + enabled: false, + redis_url: String::new(), + pool_max_size: 20, + connect_timeout_ms: 2_000, + op_timeout_ms: 50, + fail_fast_on_startup: false, + ttl: CacheTtlConfig::default(), + } + } +} + #[derive(Clone, PartialEq, Eq)] pub struct SecretBytes(Vec); @@ -540,6 +600,7 @@ impl Config { certs_root_ca_key_path: std::env::var("ATOM_CERTS_ROOT_CA_KEY_PATH").ok(), certs_leaf_default_ttl_secs: env_u64("ATOM_CERTS_LEAF_DEFAULT_TTL_SECS", 2_592_000), certs_leaf_max_ttl_secs: env_u64("ATOM_CERTS_LEAF_MAX_TTL_SECS", 2_592_000), + cache: cache_from_env()?, public_base_url, }) } @@ -606,6 +667,7 @@ impl Config { certs_root_ca_key_path: None, certs_leaf_default_ttl_secs: 2_592_000, certs_leaf_max_ttl_secs: 2_592_000, + cache: CacheConfig::default(), } } } @@ -715,6 +777,70 @@ fn db_pool_from_env() -> Result { Ok(cfg) } +fn cache_from_env() -> Result { + let default = CacheConfig::default(); + let default_ttl = CacheTtlConfig::default(); + let cfg = CacheConfig { + enabled: env_bool_default("ATOM_CACHE_ENABLED", default.enabled), + redis_url: std::env::var("ATOM_CACHE_REDIS_URL").unwrap_or_default(), + pool_max_size: env_parse("ATOM_CACHE_POOL_MAX_SIZE", default.pool_max_size)?, + connect_timeout_ms: env_parse("ATOM_CACHE_CONNECT_TIMEOUT_MS", default.connect_timeout_ms)?, + op_timeout_ms: env_parse("ATOM_CACHE_OP_TIMEOUT_MS", default.op_timeout_ms)?, + fail_fast_on_startup: env_bool_default( + "ATOM_CACHE_FAIL_FAST_ON_STARTUP", + default.fail_fast_on_startup, + ), + ttl: CacheTtlConfig { + session_secs: env_parse("ATOM_CACHE_TTL_SESSION_SECS", default_ttl.session_secs)?, + entity_status_secs: env_parse( + "ATOM_CACHE_TTL_ENTITY_STATUS_SECS", + default_ttl.entity_status_secs, + )?, + tenant_status_secs: env_parse( + "ATOM_CACHE_TTL_TENANT_STATUS_SECS", + default_ttl.tenant_status_secs, + )?, + credential_secs: env_parse( + "ATOM_CACHE_TTL_CREDENTIAL_SECS", + default_ttl.credential_secs, + )?, + credential_ceiling_secs: env_parse( + "ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS", + default_ttl.credential_ceiling_secs, + )?, + grants_secs: env_parse("ATOM_CACHE_TTL_GRANTS_SECS", default_ttl.grants_secs)?, + }, + }; + if cfg.enabled { + if cfg.redis_url.trim().is_empty() { + anyhow::bail!("ATOM_CACHE_REDIS_URL must be set when ATOM_CACHE_ENABLED=true"); + } + if cfg.pool_max_size == 0 { + anyhow::bail!("ATOM_CACHE_POOL_MAX_SIZE must be greater than zero"); + } + if cfg.connect_timeout_ms == 0 { + anyhow::bail!("ATOM_CACHE_CONNECT_TIMEOUT_MS must be greater than zero"); + } + if cfg.op_timeout_ms == 0 { + anyhow::bail!("ATOM_CACHE_OP_TIMEOUT_MS must be greater than zero"); + } + let ttl = &cfg.ttl; + if [ + ttl.session_secs, + ttl.entity_status_secs, + ttl.tenant_status_secs, + ttl.credential_secs, + ttl.credential_ceiling_secs, + ttl.grants_secs, + ] + .contains(&0) + { + anyhow::bail!("ATOM_CACHE_TTL_* values must all be greater than zero"); + } + } + Ok(cfg) +} + fn signing_keys_from_env() -> Result { let default = SigningKeyConfig::default(); Ok(SigningKeyConfig { @@ -1173,6 +1299,76 @@ mod tests { clear_hardening_env(); } + #[test] + fn cache_is_disabled_by_default() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + + let cfg = Config::from_env().expect("config"); + assert!(!cfg.cache.enabled, "caching must default off"); + assert!(!cfg.cache.fail_fast_on_startup); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_without_redis_url_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + + let err = Config::from_env().expect_err("cache enabled without a redis url"); + assert!(err.to_string().contains("ATOM_CACHE_REDIS_URL")); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_with_zero_ttl_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + std::env::set_var("ATOM_CACHE_REDIS_URL", "redis://localhost:6379/0"); + std::env::set_var("ATOM_CACHE_TTL_GRANTS_SECS", "0"); + + let err = Config::from_env().expect_err("zero grants ttl"); + assert!(err.to_string().contains("ATOM_CACHE_TTL")); + + clear_hardening_env(); + } + + #[test] + fn cache_enabled_with_zero_pool_size_fails_config() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + std::env::set_var("ATOM_CACHE_ENABLED", "true"); + std::env::set_var("ATOM_CACHE_REDIS_URL", "redis://localhost:6379/0"); + std::env::set_var("ATOM_CACHE_POOL_MAX_SIZE", "0"); + + let err = Config::from_env().expect_err("zero pool size"); + assert!(err.to_string().contains("ATOM_CACHE_POOL_MAX_SIZE")); + + clear_hardening_env(); + } + + #[test] + fn cache_disabled_ignores_missing_redis_url() { + let _guard = ENV_LOCK.lock().expect("env lock"); + clear_hardening_env(); + let _db_guard = DatabaseUrlGuard::set(); + // ATOM_CACHE_ENABLED left unset (false) — an absent/invalid redis url + // must not fail config parsing when caching is off. + let cfg = Config::from_env().expect("config"); + assert!(!cfg.cache.enabled); + assert!(cfg.cache.redis_url.is_empty()); + + clear_hardening_env(); + } + #[test] fn hot_path_allow_db_audit_opts_in_via_env() { let _guard = ENV_LOCK.lock().expect("env lock"); @@ -1302,6 +1498,18 @@ mod tests { "ATOM_GRPC_TLS_KEY_PATH", "ATOM_GRPC_TLS_CLIENT_CA_PATH", "ATOM_EMAIL_TEMPLATES_DIR", + "ATOM_CACHE_ENABLED", + "ATOM_CACHE_REDIS_URL", + "ATOM_CACHE_POOL_MAX_SIZE", + "ATOM_CACHE_CONNECT_TIMEOUT_MS", + "ATOM_CACHE_OP_TIMEOUT_MS", + "ATOM_CACHE_FAIL_FAST_ON_STARTUP", + "ATOM_CACHE_TTL_SESSION_SECS", + "ATOM_CACHE_TTL_ENTITY_STATUS_SECS", + "ATOM_CACHE_TTL_TENANT_STATUS_SECS", + "ATOM_CACHE_TTL_CREDENTIAL_SECS", + "ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS", + "ATOM_CACHE_TTL_GRANTS_SECS", ] { std::env::remove_var(name); } diff --git a/src/error.rs b/src/error.rs index 7f81c69..c8d762f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub enum AppError { message: String, retry_after_secs: u64, }, + #[error("{0}")] + ServiceUnavailable(String), #[error("database error")] Database(#[from] sqlx::Error), #[error("internal error")] @@ -65,6 +67,9 @@ impl AppError { retry_after_secs, } } + pub fn service_unavailable(msg: impl Into) -> Self { + AppError::ServiceUnavailable(msg.into()) + } } impl IntoResponse for AppError { @@ -90,6 +95,7 @@ impl IntoResponse for AppError { } return response; } + AppError::ServiceUnavailable(m) => (StatusCode::SERVICE_UNAVAILABLE, m.clone()), AppError::Database(e) => { if let sqlx::Error::Database(db) = e { match db.code().as_deref() { @@ -150,6 +156,7 @@ impl From for tonic::Status { AppError::Conflict(msg) => tonic::Status::already_exists(msg), AppError::PayloadTooLarge(msg) => tonic::Status::invalid_argument(msg), AppError::RateLimited { message, .. } => tonic::Status::resource_exhausted(message), + AppError::ServiceUnavailable(msg) => tonic::Status::unavailable(msg), AppError::Database(e) => { tracing::error!("db error: {e}"); tonic::Status::internal("database error") diff --git a/src/graphql/schema.rs b/src/graphql/schema.rs index ca49453..f7c3667 100644 --- a/src/graphql/schema.rs +++ b/src/graphql/schema.rs @@ -519,6 +519,7 @@ mod tests { standby: None, }, None, + None, ) } diff --git a/src/grpc.rs b/src/grpc.rs index 54a8750..72377dc 100644 --- a/src/grpc.rs +++ b/src/grpc.rs @@ -816,6 +816,7 @@ mod tests { standby: None, }, None, + None, ) } } diff --git a/src/health.rs b/src/health.rs index 5747aff..c9c4402 100644 --- a/src/health.rs +++ b/src/health.rs @@ -74,6 +74,7 @@ pub struct SystemStatus { pub migrations: ComponentCheck, pub signing_keys: ComponentCheck, pub certificate_issuer: ComponentCheck, + pub cache: ComponentCheck, pub db_pool: DbPoolStatus, pub signing_key_state: Option, pub audit_retention: AuditRetentionStatus, @@ -97,6 +98,7 @@ pub async fn readiness(state: &AppState) -> (StatusCode, Json) { let migrations = migrations_check(state).await; let (signing_keys, signing_key_state) = signing_keys_check(state).await; let certificate_issuer = certificate_issuer_check(state); + let cache = cache_check(state).await; let grpc_ready = grpc_check(state).await; let ready = readiness_ok( &database, @@ -124,6 +126,7 @@ pub async fn readiness(state: &AppState) -> (StatusCode, Json) { migrations, signing_keys, certificate_issuer, + cache, db_pool: db_pool_status(state), signing_key_state, audit_retention: audit_retention_status(state).await, @@ -260,6 +263,34 @@ fn certificate_issuer_check(state: &AppState) -> ComponentCheck { } } +/// Never part of the hard readiness requirement (see `readiness_ok`) — caching +/// is a performance optimization with a Postgres fallback for reads, so a down +/// Redis must not fail `/health/ready`. While enabled, an unreachable Redis +/// does refuse security-sensitive mutations (see `src/cache/mod.rs`); that +/// distinction is noted in the message rather than folded into the readiness +/// gate, which only concerns general request-serving availability. +async fn cache_check(state: &AppState) -> ComponentCheck { + let Some(cache) = &state.cache else { + return ComponentCheck { + status: ComponentStatus::Disabled, + message: "cache disabled".to_string(), + }; + }; + match cache.ping().await { + Ok(()) => ComponentCheck { + status: ComponentStatus::Ok, + message: "cache reachable".to_string(), + }, + Err(err) => ComponentCheck { + status: ComponentStatus::Degraded, + message: format!( + "cache unreachable: {err}; reads fall back to the database, but \ + security-sensitive mutations are refused until it recovers" + ), + }, + } +} + async fn grpc_check(state: &AppState) -> ComponentCheck { let status = state.grpc_status().await; match status.state { @@ -341,6 +372,7 @@ mod tests { migrations: check(ComponentStatus::Ok), signing_keys: check(ComponentStatus::Ok), certificate_issuer: check(ComponentStatus::Disabled), + cache: check(ComponentStatus::Disabled), db_pool: DbPoolStatus { max_connections: 0, min_connections: 0, diff --git a/src/lib.rs b/src/lib.rs index 5da2922..24fce49 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod audit; pub mod auth; pub mod authz; pub mod build_info; +pub mod cache; pub mod certs; pub mod config; pub mod crypto; diff --git a/src/main.rs b/src/main.rs index 06916f4..be23534 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use anyhow::Context; use atom::{ - audit, certs, config, db, events, grpc, identity, keys, metrics, purge, routes, + audit, cache, certs, config, db, events, grpc, identity, keys, metrics, purge, routes, state::{self, GrpcRuntimeStatus}, }; use tracing_subscriber::EnvFilter; @@ -44,7 +44,10 @@ async fn main() -> anyhow::Result<()> { // startup instead of leaving HTTP up with a permanently failing gRPC task. let grpc_tls = grpc::load_tls_config(&cfg).await?; - let mut state = state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer); + let cache = init_cache(&cfg.cache).await?; + + let mut state = + state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer, cache); if cfg.events.enabled() { let publisher = events::publisher::AmqpPublisher::connect(&cfg.events) .await @@ -97,6 +100,29 @@ async fn main() -> anyhow::Result<()> { Ok(()) } +/// Connects the Redis-backed cache when enabled. A connect failure honors +/// `ATOM_CACHE_FAIL_FAST_ON_STARTUP`: abort like an unreachable Postgres +/// would, or log and continue cache-free (the recommended default — caching +/// is a performance optimization, not a correctness dependency for reads). +async fn init_cache(cfg: &config::CacheConfig) -> anyhow::Result> { + if !cfg.enabled { + return Ok(None); + } + match cache::CacheClient::connect(cfg).await { + Ok(client) => { + tracing::info!("cache enabled; connected to Redis"); + Ok(Some(client)) + } + Err(err) if cfg.fail_fast_on_startup => { + Err(err.context("cache connect failed and ATOM_CACHE_FAIL_FAST_ON_STARTUP=true")) + } + Err(err) => { + tracing::error!("cache connect failed, continuing without cache: {err}"); + Ok(None) + } + } +} + fn init_tracing(logging: &config::LoggingConfig) -> anyhow::Result<()> { let filter = EnvFilter::try_new(&logging.level) .context("ATOM_LOG_LEVEL/RUST_LOG must be a valid tracing filter")?; diff --git a/src/metrics.rs b/src/metrics.rs index f99137f..539c0be 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -36,6 +36,13 @@ pub const EVENT_OUTBOX_PUBLISH_FAILURES: &str = "atom_event_outbox_publish_failu pub const EVENT_OUTBOX_EXHAUSTED: &str = "atom_event_outbox_exhausted_total"; /// Gauge of DB pool connections, labelled by `state` (total|idle). pub const DB_POOL_CONNECTIONS: &str = "atom_db_pool_connections"; +/// Counter of cache reads, labelled by `category` (session|entity_status| +/// tenant_status|credential|credential_ceiling|grants) and `outcome` +/// (hit|miss|error). +pub const CACHE_LOOKUP: &str = "atom_cache_lookup_total"; +/// Counter of cache invalidation/barrier operations, labelled by `category` +/// and `outcome` (ok|error). +pub const CACHE_INVALIDATION: &str = "atom_cache_invalidation_total"; #[cfg(feature = "metrics")] mod backend { @@ -102,6 +109,15 @@ mod backend { pub fn record_outbox_exhausted() { metrics::counter!(EVENT_OUTBOX_EXHAUSTED).increment(1); } + + pub fn record_cache_lookup(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_LOOKUP, "category" => category, "outcome" => outcome).increment(1); + } + + pub fn record_cache_invalidation(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_INVALIDATION, "category" => category, "outcome" => outcome) + .increment(1); + } } #[cfg(not(feature = "metrics"))] @@ -130,9 +146,14 @@ mod backend { pub fn record_outbox_publish_failure(_rows: u64) {} #[inline] pub fn record_outbox_exhausted() {} + #[inline] + pub fn record_cache_lookup(_category: &'static str, _outcome: &'static str) {} + #[inline] + pub fn record_cache_invalidation(_category: &'static str, _outcome: &'static str) {} } pub use backend::{ - enabled, init, record_audit_db_suppressed, record_audit_failure, record_decision, - record_outbox_exhausted, record_outbox_publish_failure, record_rate_limit_rejection, render, + enabled, init, record_audit_db_suppressed, record_audit_failure, record_cache_invalidation, + record_cache_lookup, record_decision, record_outbox_exhausted, record_outbox_publish_failure, + record_rate_limit_rejection, render, }; diff --git a/src/routes.rs b/src/routes.rs index 336bb3d..7fbe211 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -380,6 +380,7 @@ mod tests { standby: None, }, None, + None, ) } } diff --git a/src/state.rs b/src/state.rs index 772a4d8..c199b3b 100644 --- a/src/state.rs +++ b/src/state.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use tokio::sync::RwLock; use crate::{ - certs::service::CertificateIssuer, config::Config, events::publisher::EventPublisher, - keys::ActiveKeys, rate_limit::RateLimiter, + cache::CacheClient, certs::service::CertificateIssuer, config::Config, + events::publisher::EventPublisher, keys::ActiveKeys, rate_limit::RateLimiter, }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -62,6 +62,11 @@ pub struct AppState { /// Set via [`AppState::with_event_publisher`], since connecting to a /// broker is an async operation `AppState::new` (sync) can't perform. pub event_publisher: Option>, + /// `None` when caching is disabled or unconfigured — every call site + /// checks this and falls through to Postgres, so the service runs + /// cache-free with zero behavior change (this is what keeps `cargo test` + /// and local dev working without Redis). + pub cache: Option>, grpc_status: Arc>, } @@ -71,6 +76,7 @@ impl AppState { config: Config, keys: ActiveKeys, certificate_issuer: Option, + cache: Option, ) -> Self { let grpc_status = GrpcRuntimeStatus::starting(config.grpc_addr.clone()); AppState { @@ -80,6 +86,7 @@ impl AppState { certificate_issuer: certificate_issuer.map(Arc::new), rate_limiter: Arc::new(RateLimiter::default()), event_publisher: None, + cache: cache.map(Arc::new), grpc_status: Arc::new(RwLock::new(grpc_status)), } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b588ea6..1e889c6 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -6,9 +6,17 @@ //! ```bash //! DATABASE_URL=postgres://... cargo test -- --ignored //! ``` +//! +//! Cache-invalidation tests additionally require a reachable Redis at +//! `ATOM_TEST_REDIS_URL`, following the same `#[ignore]` convention: +//! +//! ```bash +//! DATABASE_URL=postgres://... ATOM_TEST_REDIS_URL=redis://... cargo test -- --ignored +//! ``` #![allow(dead_code)] +use atom::{cache::CacheClient, config::CacheConfig}; use sqlx::PgPool; /// Connect to the test database and run all migrations. @@ -26,6 +34,22 @@ pub async fn pool() -> PgPool { pool } +/// Connect to the test Redis with short TTLs so invalidation-correctness +/// tests aren't waiting on production-sized windows, and a fresh v1 +/// namespace-mate configuration otherwise identical to `CacheConfig::default`. +pub async fn cache_client() -> CacheClient { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + let cfg = CacheConfig { + enabled: true, + redis_url: url, + ..CacheConfig::default() + }; + CacheClient::connect(&cfg) + .await + .expect("connect to test redis") +} + /// Well-known seeded admin entity. pub fn admin_id() -> uuid::Uuid { "00000000-0000-0000-0000-000000000001".parse().unwrap() diff --git a/tests/m10_graphql_profiles.rs b/tests/m10_graphql_profiles.rs index a5d6ca1..969a034 100644 --- a/tests/m10_graphql_profiles.rs +++ b/tests/m10_graphql_profiles.rs @@ -80,6 +80,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } diff --git a/tests/m11_graphql_primitives.rs b/tests/m11_graphql_primitives.rs index 42210b9..534ee23 100644 --- a/tests/m11_graphql_primitives.rs +++ b/tests/m11_graphql_primitives.rs @@ -32,7 +32,7 @@ async fn state(pool: PgPool) -> AppState { let active_keys = keys::load_active_keys(&pool, &config.signing_keys) .await .expect("load signing keys"); - AppState::new(pool, config, active_keys, None) + AppState::new(pool, config, active_keys, None, None) } fn authed(query: impl Into) -> Request { diff --git a/tests/m12_graphql_identity.rs b/tests/m12_graphql_identity.rs index 2b0ee30..ea5d835 100644 --- a/tests/m12_graphql_identity.rs +++ b/tests/m12_graphql_identity.rs @@ -37,6 +37,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } diff --git a/tests/m13_graphql_authz_admin.rs b/tests/m13_graphql_authz_admin.rs index 8c852d3..5e7a946 100644 --- a/tests/m13_graphql_authz_admin.rs +++ b/tests/m13_graphql_authz_admin.rs @@ -36,6 +36,7 @@ fn state(pool: PgPool) -> AppState { standby: None, }, None, + None, ) } diff --git a/tests/m14_api_endpoints.rs b/tests/m14_api_endpoints.rs index 83841b5..cf8b50e 100644 --- a/tests/m14_api_endpoints.rs +++ b/tests/m14_api_endpoints.rs @@ -30,7 +30,7 @@ use uuid::Uuid; fn state(pool: PgPool, keys: ActiveKeys) -> AppState { let config = Config::for_tests(); - AppState::new(pool, config, keys, None) + AppState::new(pool, config, keys, None, None) } async fn active_keys(pool: &PgPool) -> ActiveKeys { diff --git a/tests/m17_certificates.rs b/tests/m17_certificates.rs index ae18277..0fb4efb 100644 --- a/tests/m17_certificates.rs +++ b/tests/m17_certificates.rs @@ -126,6 +126,7 @@ fn state( standby: None, }, certificate_issuer, + None, ) } diff --git a/tests/m23_authenticate_credential.rs b/tests/m23_authenticate_credential.rs index f08409d..a629b22 100644 --- a/tests/m23_authenticate_credential.rs +++ b/tests/m23_authenticate_credential.rs @@ -278,7 +278,7 @@ async fn credential_authentication_rejects_inactive_or_deleted_principals() { async fn grpc_authenticate_credential_requires_service_auth_and_returns_identity() { let pool = common::pool().await; let keys = active_keys(&pool).await; - let state = AppState::new(pool.clone(), Config::for_tests(), keys.clone(), None); + let state = AppState::new(pool.clone(), Config::for_tests(), keys.clone(), None, None); let listener = grpc::bind_listener("127.0.0.1:0".parse().expect("addr")) .await .expect("bind grpc"); From f92a6042abd5ba70b82a20cbf7a2ca282e42c2e1 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 15:14:06 +0300 Subject: [PATCH 02/19] Cache AuthN inputs: sessions, entity/tenant status, and credentials Routes JWT and API-key authentication through the cache client added previously, guarded by the version/dirty barrier so a revoked session, deactivated entity/tenant, or revoked credential is denied immediately even when caching is enabled - never served stale for the cache TTL. Wires invalidation into every mutation that can affect these: session revoke/refresh, entity and tenant status changes and soft-delete/ restore, and credential revoke/rotate/verifier-upgrade, including the session and credential fallout of tenant/entity soft-delete that a naive "invalidate only the status field" approach would miss once the tenant/entity is later restored. --- src/audit.rs | 4 +- src/auth.rs | 597 ++++++++++++++++++++++++++++++------- src/graphql/auth.rs | 55 +++- src/graphql/credentials.rs | 145 +++++---- src/graphql/entities.rs | 151 +++++++--- src/graphql/tenants.rs | 181 ++++++++--- src/identity/handlers.rs | 81 +++-- src/identity/repo.rs | 371 ++++++++++++++++------- src/identity/service.rs | 107 ++++--- src/tenants/repo.rs | 81 ++++- tests/m21_soft_delete.rs | 1 + 11 files changed, 1326 insertions(+), 448 deletions(-) diff --git a/src/audit.rs b/src/audit.rs index 9f16e7c..c22b1e0 100644 --- a/src/audit.rs +++ b/src/audit.rs @@ -205,7 +205,7 @@ pub async fn commit_with_audit( /// Enqueues a domain event outbox row inside an existing DB transaction for /// non-audited operations (e.g. create mutations), keeping the mutation and outbox /// event strictly atomic. -async fn observe_in_tx( +pub(crate) async fn observe_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, events_enabled: bool, meta: &AuditMeta<'_>, @@ -256,7 +256,7 @@ pub async fn commit_with_observation( /// Emits an audit log tracing line for a successful non-audited operation (observe path) /// after its database transaction has successfully committed. -fn log_observe_allow(meta: &AuditMeta<'_>, details: &Value) { +pub(crate) fn log_observe_allow(meta: &AuditMeta<'_>, details: &Value) { let event = AuditEvent { actor_entity_id: meta.actor_entity_id, tenant_id: meta.tenant_id, diff --git a/src/auth.rs b/src/auth.rs index 2948eaf..6e39a23 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -12,6 +12,12 @@ use sqlx::PgPool; use uuid::Uuid; use crate::{ + cache::{ + entries::{ + CredentialCacheEntry, EntityStatusCacheEntry, SessionCacheEntry, TenantStatusCacheEntry, + }, + keys as cache_keys, CacheCategory, Lookup, + }, error::{db_err, AppError}, keys::{ActiveKeys, LoadedKey}, models::enums::{ @@ -55,6 +61,14 @@ pub struct AuthContext { /// The loaded permission ceiling for a scoped access token; `None` for /// unscoped tokens and JWT/session auth (no cap). pub ceiling: Option>, + /// The cache backing this request's grant-expansion lookups (see + /// `src/cache/mod.rs`). Carried here — rather than threaded as an extra + /// parameter through every capability gate (`has_capability_in_scope`, + /// `require_any_capability`, ...) and through `authz::engine::evaluate`/ + /// `explain` — because those have dozens of call sites across the + /// codebase, while `AuthContext` is constructed in exactly a few places. + /// `None` reproduces pre-caching behavior exactly. + pub cache: Option>, } /// Extractor that requires the authenticated entity to hold a `manage` policy @@ -163,31 +177,28 @@ async fn auth_from_token(state: &AppState, token: &str) -> Result Result { - let keys = state.keys.read().await; - let claims = decode_jwt( - token, - &keys, - &state.config.jwt_issuer, - &state.config.jwt_audience, - )?; - drop(keys); - - let entity_id: Uuid = claims - .sub - .parse() - .map_err(|_| AppError::unauthorized("invalid entity id in token"))?; - let session_id: Uuid = claims - .sid - .parse() - .map_err(|_| AppError::unauthorized("invalid session id in token"))?; - let tenant_id: Option = claims - .tid - .as_deref() - .map(|s| s.parse()) - .transpose() - .map_err(|_| AppError::unauthorized("invalid tenant id in token"))?; +/// A snapshot of the fields `auth_from_jwt` needs, regardless of whether they +/// came from a cache hit or a fresh Postgres load — one canonical check +/// function (`check_session_entity_tenant`) validates either shape +/// identically, so the deny logic can never drift between the cached and +/// uncached paths. +struct SessionEntityTenantSnapshot { + session_entity_id: Uuid, + revoked_at: Option>, + expires_at: chrono::DateTime, + entity_tenant_id: Option, + entity_status: EntityStatus, + tenant_status: Option, +} +/// The existing session/entity/tenant join, unchanged from before caching was +/// introduced. This remains the single cache-miss loader — it is never split +/// into per-field queries. +async fn load_session_entity_tenant( + pool: &PgPool, + session_id: Uuid, + entity_id: Uuid, +) -> Result { use sqlx::Row; let row = sqlx::query( r#"SELECT s.revoked_at, @@ -204,7 +215,7 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result AppError::unauthorized("session not found"), @@ -215,48 +226,236 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result = row .try_get("expires_at") .map_err(|_| AppError::unauthorized("corrupt session"))?; + let entity_status: EntityStatus = row + .try_get("entity_status") + .map_err(|_| AppError::unauthorized("corrupt entity"))?; + let entity_tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); + let tenant_status: Option = row + .try_get::, _>("tenant_status") + .unwrap_or(None); + + Ok(SessionEntityTenantSnapshot { + session_entity_id: entity_id, + revoked_at, + expires_at, + entity_tenant_id, + entity_status, + tenant_status, + }) +} - if revoked_at.is_some() { +/// The one canonical set of deny checks for JWT/session authentication — +/// consumed identically whether `snapshot` came from a cache hit or a fresh +/// Postgres load. +fn check_session_entity_tenant( + snapshot: &SessionEntityTenantSnapshot, + expected_entity_id: Uuid, + expected_tenant_id: Option, +) -> Result<(), AppError> { + if snapshot.session_entity_id != expected_entity_id { + return Err(AppError::unauthorized("session not found")); + } + if snapshot.revoked_at.is_some() { return Err(AppError::unauthorized("session revoked")); } - if expires_at < Utc::now() { + if snapshot.expires_at < Utc::now() { return Err(AppError::unauthorized("session expired")); } - - let entity_status: EntityStatus = row - .try_get("entity_status") - .map_err(|_| AppError::unauthorized("corrupt entity"))?; - if entity_status != EntityStatus::Active { + if snapshot.entity_status != EntityStatus::Active { return Err(AppError::unauthorized("entity is not active")); } - - let entity_tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); - if tenant_id != entity_tenant_id { + if expected_tenant_id != snapshot.entity_tenant_id { return Err(AppError::unauthorized("token tenant does not match entity")); } - if let Some(tenant_status) = row - .try_get::, _>("tenant_status") - .unwrap_or(None) - { - if tenant_status != TenantStatus::Active { + if let Some(tenant_status) = &snapshot.tenant_status { + if *tenant_status != TenantStatus::Active { return Err(AppError::unauthorized("tenant is not active")); } } + Ok(()) +} - Ok(AuthContext { +fn jwt_auth_context( + state: &AppState, + entity_id: Uuid, + session_id: Uuid, + snapshot: &SessionEntityTenantSnapshot, +) -> AuthContext { + AuthContext { entity_id, - tenant_id: entity_tenant_id, + tenant_id: snapshot.entity_tenant_id, session_id: Some(session_id), + cache: state.cache.clone(), ..Default::default() - }) + } } -async fn auth_from_api_key(state: &AppState, key: &str) -> Result { - let (cred_id, secret_bytes) = - parse_api_key(key).ok_or_else(|| AppError::unauthorized("malformed api key"))?; +async fn auth_from_jwt(state: &AppState, token: &str) -> Result { + let keys = state.keys.read().await; + let claims = decode_jwt( + token, + &keys, + &state.config.jwt_issuer, + &state.config.jwt_audience, + )?; + drop(keys); - use sqlx::Row; + let entity_id: Uuid = claims + .sub + .parse() + .map_err(|_| AppError::unauthorized("invalid entity id in token"))?; + let session_id: Uuid = claims + .sid + .parse() + .map_err(|_| AppError::unauthorized("invalid session id in token"))?; + let tenant_id: Option = claims + .tid + .as_deref() + .map(|s| s.parse()) + .transpose() + .map_err(|_| AppError::unauthorized("invalid tenant id in token"))?; + let Some(cache) = &state.cache else { + let snapshot = load_session_entity_tenant(&state.pool, session_id, entity_id).await?; + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + return Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)); + }; + + let session_key = cache_keys::session(session_id); + let entity_key = cache_keys::entity_status(entity_id); + let tenant_key = tenant_id.map(cache_keys::tenant_status); + + let session_lookup = cache + .lookup::(CacheCategory::Session, &session_key) + .await; + let entity_lookup = cache + .lookup::(CacheCategory::EntityStatus, &entity_key) + .await; + let tenant_lookup = match &tenant_key { + Some(key) => Some( + cache + .lookup::(CacheCategory::TenantStatus, key) + .await, + ), + None => None, + }; + + // Full cache hit across every key this token needs: validate entirely in + // memory, no Postgres round trip. + if let (Lookup::Hit(session), Lookup::Hit(entity)) = (&session_lookup, &entity_lookup) { + let tenant_hit = match (&tenant_key, &tenant_lookup) { + (None, _) => Some(None), + (Some(_), Some(Lookup::Hit(tenant))) => Some(Some(tenant)), + _ => None, + }; + if let Some(tenant_opt) = tenant_hit { + let snapshot = SessionEntityTenantSnapshot { + session_entity_id: session.entity_id, + revoked_at: session.revoked_at, + expires_at: session.expires_at, + entity_tenant_id: entity.tenant_id, + entity_status: entity.status.clone(), + tenant_status: tenant_opt.map(|t| t.status.clone()), + }; + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + return Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)); + } + } + + // Any miss/dirty/unavailable key: fall back to the existing combined + // query, unchanged, then best-effort populate whichever entries missed. + let snapshot = load_session_entity_tenant(&state.pool, session_id, entity_id).await?; + + if let Lookup::Miss { version } = session_lookup { + let entry = SessionCacheEntry { + entity_id, + revoked_at: snapshot.revoked_at, + expires_at: snapshot.expires_at, + }; + cache + .try_populate(CacheCategory::Session, &session_key, version, &entry) + .await; + } + if let Lookup::Miss { version } = entity_lookup { + let entry = EntityStatusCacheEntry { + status: snapshot.entity_status.clone(), + deleted_at: None, + tenant_id: snapshot.entity_tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + if let (Some(tenant_key), Some(Lookup::Miss { version })) = (&tenant_key, tenant_lookup) { + if let Some(tenant_status) = &snapshot.tenant_status { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + deleted_at: None, + }; + cache + .try_populate(CacheCategory::TenantStatus, tenant_key, version, &entry) + .await; + } + } + + check_session_entity_tenant(&snapshot, entity_id, tenant_id)?; + Ok(jwt_auth_context(state, entity_id, session_id, &snapshot)) +} + +/// A snapshot of every field `auth_from_api_key` needs, regardless of whether +/// it came from cache hits or a fresh Postgres load — one canonical function +/// (`verify_api_key_snapshot`) validates either shape identically. +struct CredentialSnapshot { + entity_id: Uuid, + tenant_id: Option, + secret_hash: Option, + secret_lookup_hash: Option>, + status: CredentialStatus, + expires_at: Option>, + scoped: bool, + entity_status: EntityStatus, + tenant_status: Option, +} + +impl CredentialSnapshot { + fn from_cache_entries( + cred: &CredentialCacheEntry, + entity: &EntityStatusCacheEntry, + tenant: Option<&TenantStatusCacheEntry>, + ) -> Self { + Self { + entity_id: cred.entity_id, + // From the entity entry, not `cred` — see `CredentialCacheEntry`'s + // doc comment for why the credential entry carries no tenant_id + // of its own. + tenant_id: entity.tenant_id, + secret_hash: cred.secret_hash.clone(), + secret_lookup_hash: cred.secret_lookup_hash.clone(), + status: cred.status.clone(), + expires_at: cred.expires_at, + scoped: cred.scoped, + entity_status: entity.status.clone(), + tenant_status: tenant.map(|t| t.status.clone()), + } + } +} + +fn credential_cache_entry(snapshot: &CredentialSnapshot) -> CredentialCacheEntry { + CredentialCacheEntry { + entity_id: snapshot.entity_id, + status: snapshot.status.clone(), + secret_hash: snapshot.secret_hash.clone(), + secret_lookup_hash: snapshot.secret_lookup_hash.clone(), + expires_at: snapshot.expires_at, + scoped: snapshot.scoped, + } +} + +/// The existing credential/entity/tenant join, unchanged from before caching +/// was introduced. This remains the single cache-miss loader for all three +/// cached entities it touches — it is never split into per-field queries. +async fn load_credential_row(pool: &PgPool, cred_id: Uuid) -> Result { + use sqlx::Row; let row = sqlx::query( r#"SELECT c.entity_id, c.secret_hash, @@ -276,94 +475,125 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result AppError::unauthorized("api key not found"), other => AppError::Database(other), })?; - // Verify the secret before any state checks, so a caller holding only a - // credential ID cannot learn the token's revoked/expired status. - // - // Verifier: keyed HMAC digest when present (tokens minted with a KEK); - // argon2 hash otherwise. See create_access_token for the rationale. - let lookup_hash: Option> = row.try_get("secret_lookup_hash").unwrap_or(None); - let had_lookup_hash = lookup_hash.is_some(); - let verified = match lookup_hash { + Ok(CredentialSnapshot { + entity_id: row.try_get("entity_id").map_err(db_err)?, + tenant_id: row.try_get("tenant_id").unwrap_or(None), + secret_hash: row.try_get("secret_hash").unwrap_or(None), + secret_lookup_hash: row.try_get("secret_lookup_hash").unwrap_or(None), + status: row.try_get("status").map_err(db_err)?, + expires_at: row.try_get("expires_at").unwrap_or(None), + scoped: row.try_get("scoped").unwrap_or(false), + entity_status: row + .try_get("entity_status") + .map_err(|_| AppError::unauthorized("corrupt entity"))?, + tenant_status: row + .try_get::, _>("tenant_status") + .unwrap_or(None), + }) +} + +/// The one canonical set of deny checks for API-key authentication — +/// consumed identically whether `snapshot` came from cache hits or a fresh +/// Postgres load. Secret verification runs before any state check, so a +/// caller holding only a credential ID cannot learn the token's +/// revoked/expired status — caching the row does not change this order. +fn verify_api_key_snapshot( + snapshot: &CredentialSnapshot, + secret_bytes: &[u8], + kek: Option<&crate::config::SecretBytes>, +) -> Result<(), AppError> { + let verified = match &snapshot.secret_lookup_hash { Some(stored) => { - let kek = state - .config - .signing_keys - .key_encryption_key - .as_ref() - .ok_or_else(|| AppError::unauthorized("invalid api key"))?; - crate::crypto::hmac_sha256_verify(kek.expose(), &secret_bytes, &stored) + let kek = kek.ok_or_else(|| AppError::unauthorized("invalid api key"))?; + crate::crypto::hmac_sha256_verify(kek.expose(), secret_bytes, stored) } None => { - let hash: Option = row.try_get("secret_hash").unwrap_or(None); - let hash = hash.ok_or_else(|| AppError::unauthorized("invalid credential"))?; + let hash = snapshot + .secret_hash + .as_deref() + .ok_or_else(|| AppError::unauthorized("invalid credential"))?; use argon2::{ password_hash::{PasswordHash, PasswordVerifier}, Argon2, }; - let parsed = PasswordHash::new(&hash) + let parsed = PasswordHash::new(hash) .map_err(|_| AppError::unauthorized("invalid credential"))?; Argon2::default() - .verify_password(&secret_bytes, &parsed) + .verify_password(secret_bytes, &parsed) .is_ok() } }; if !verified { return Err(AppError::unauthorized("invalid api key")); } - - let status: CredentialStatus = row.try_get("status").map_err(db_err)?; - if status != CredentialStatus::Active { + if snapshot.status != CredentialStatus::Active { return Err(AppError::unauthorized("api key revoked")); } - - let expires_at: Option> = row.try_get("expires_at").unwrap_or(None); - if let Some(exp) = expires_at { + if let Some(exp) = snapshot.expires_at { if exp < Utc::now() { return Err(AppError::unauthorized("api key expired")); } } - - let entity_status: EntityStatus = row - .try_get("entity_status") - .map_err(|_| AppError::unauthorized("corrupt entity"))?; - if entity_status != EntityStatus::Active { + if snapshot.entity_status != EntityStatus::Active { return Err(AppError::unauthorized("entity is not active")); } - if let Some(tenant_status) = row - .try_get::, _>("tenant_status") - .unwrap_or(None) - { - if tenant_status != TenantStatus::Active { + if let Some(tenant_status) = &snapshot.tenant_status { + if *tenant_status != TenantStatus::Active { return Err(AppError::unauthorized("tenant is not active")); } } + Ok(()) +} - let entity_id: Uuid = row.try_get("entity_id").map_err(db_err)?; - - let tenant_id: Option = row.try_get("tenant_id").unwrap_or(None); +/// Runs the shared post-verification steps (opportunistic verifier upgrade, +/// usage stamp, scoped-ceiling load) and builds the `AuthContext`. Shared by +/// every `auth_from_api_key` path (cached or not) so these never drift. +async fn finish_api_key_auth( + state: &AppState, + cred_id: Uuid, + secret_bytes: &[u8], + snapshot: &CredentialSnapshot, +) -> Result { + verify_api_key_snapshot( + snapshot, + secret_bytes, + state.config.signing_keys.key_encryption_key.as_ref(), + )?; // Opportunistic verifier upgrade: a token minted without a KEK verifies via // argon2, paying the KDF on every request. Once a KEK is configured, swap to // the keyed digest on first successful use. Best-effort — a failed upgrade - // must never fail an otherwise-valid authentication. - if !had_lookup_hash { + // must never fail an otherwise-valid authentication. Guarded by the cache + // barrier (not a bare DEL) so a concurrent reader can't repopulate the + // credential cache entry with the stale verifier after this commits. + if snapshot.secret_lookup_hash.is_none() { if let Some(kek) = state.config.signing_keys.key_encryption_key.as_ref() { - let digest = crate::crypto::hmac_sha256(kek.expose(), &secret_bytes); - if let Err(err) = sqlx::query( - "UPDATE credentials SET secret_lookup_hash = $1, secret_hash = NULL WHERE id = $2", + let digest = crate::crypto::hmac_sha256(kek.expose(), secret_bytes); + let credential_key = cache_keys::credential(cred_id); + let result = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + CacheCategory::Credential, + std::slice::from_ref(&credential_key), + || async { + sqlx::query( + "UPDATE credentials SET secret_lookup_hash = $1, secret_hash = NULL WHERE id = $2", + ) + .bind(digest) + .bind(cred_id) + .execute(&state.pool) + .await + .map_err(AppError::Database) + }, ) - .bind(digest) - .bind(cred_id) - .execute(&state.pool) - .await - { + .await; + if let Err(err) = result { tracing::warn!( credential_id = %cred_id, error = %err, @@ -376,6 +606,7 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result Result Result { + let (cred_id, secret_bytes) = + parse_api_key(key).ok_or_else(|| AppError::unauthorized("malformed api key"))?; + + let Some(cache) = &state.cache else { + let row = load_credential_row(&state.pool, cred_id).await?; + return finish_api_key_auth(state, cred_id, &secret_bytes, &row).await; + }; + + let credential_key = cache_keys::credential(cred_id); + let credential_lookup = cache + .lookup::(CacheCategory::Credential, &credential_key) + .await; + + if let Lookup::Hit(cred_entry) = &credential_lookup { + let entity_key = cache_keys::entity_status(cred_entry.entity_id); + let entity_lookup = cache + .lookup::(CacheCategory::EntityStatus, &entity_key) + .await; + + // Tenant context is derived from the entity's own (freshly + // invalidated) cache entry, never from the credential entry: a + // tenant move invalidates `entity_status` but not `credential`, so a + // stale `cred_entry` would otherwise check the entity's *former* + // tenant's status and hand the request an `AuthContext.tenant_id` + // for a tenant the entity no longer belongs to. + if let Lookup::Hit(entity_entry) = &entity_lookup { + let tenant_key = entity_entry.tenant_id.map(cache_keys::tenant_status); + let tenant_lookup = match &tenant_key { + Some(k) => Some( + cache + .lookup::(CacheCategory::TenantStatus, k) + .await, + ), + None => None, + }; + let tenant_hit = match (&tenant_key, &tenant_lookup) { + (None, _) => Some(None), + (Some(_), Some(Lookup::Hit(t))) => Some(Some(t)), + _ => None, + }; + if let Some(tenant_opt) = tenant_hit { + let snapshot = + CredentialSnapshot::from_cache_entries(cred_entry, entity_entry, tenant_opt); + return finish_api_key_auth(state, cred_id, &secret_bytes, &snapshot).await; + } + } + + // Credential hit but entity/tenant status missed (or the entity's + // current tenant's status wasn't cached): reuse the same combined + // query the cold-start path uses below, and best-effort populate + // whichever entries missed — always keyed off `row`'s freshly + // joined entity_id/tenant_id, mirroring the cold-start path exactly, + // never off the credential entry's stale copy. + let row = load_credential_row(&state.pool, cred_id).await?; + if let Lookup::Miss { version } = entity_lookup { + let entry = EntityStatusCacheEntry { + status: row.entity_status.clone(), + deleted_at: None, + tenant_id: row.tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + if let (Some(tenant_id), Some(tenant_status)) = (row.tenant_id, &row.tenant_status) { + let tenant_key = cache_keys::tenant_status(tenant_id); + if let Lookup::Miss { version } = cache + .lookup::(CacheCategory::TenantStatus, &tenant_key) + .await + { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + deleted_at: None, + }; + cache + .try_populate(CacheCategory::TenantStatus, &tenant_key, version, &entry) + .await; + } + } + return finish_api_key_auth(state, cred_id, &secret_bytes, &row).await; + } + + // Credential missed (or dirty/unavailable): one combined query loads + // everything; populate the credential entry and, best-effort, the + // entity/tenant entries too (each with its own freshly-observed version). + let row = load_credential_row(&state.pool, cred_id).await?; + + if let Lookup::Miss { version } = credential_lookup { + let entry = credential_cache_entry(&row); + cache + .try_populate(CacheCategory::Credential, &credential_key, version, &entry) + .await; + } + let entity_key = cache_keys::entity_status(row.entity_id); + if let Lookup::Miss { version } = cache + .lookup::(CacheCategory::EntityStatus, &entity_key) + .await + { + let entry = EntityStatusCacheEntry { + status: row.entity_status.clone(), + deleted_at: None, + tenant_id: row.tenant_id, + }; + cache + .try_populate(CacheCategory::EntityStatus, &entity_key, version, &entry) + .await; + } + if let (Some(tenant_id), Some(tenant_status)) = (row.tenant_id, &row.tenant_status) { + let tenant_key = cache_keys::tenant_status(tenant_id); + if let Lookup::Miss { version } = cache + .lookup::(CacheCategory::TenantStatus, &tenant_key) + .await + { + let entry = TenantStatusCacheEntry { + status: tenant_status.clone(), + deleted_at: None, + }; + cache + .try_populate(CacheCategory::TenantStatus, &tenant_key, version, &entry) + .await; + } + } + + finish_api_key_auth(state, cred_id, &secret_bytes, &row).await +} + impl AuthContext { /// Load the authenticated entity's canonical grant expansion for one - /// authorization decision. This deliberately hits the DB on each gate/PDP - /// entry so policy writes take effect for the next check, even inside a - /// multi-field GraphQL request. + /// authorization decision. This deliberately hits the DB (or, when + /// caching is enabled, the cache-aside path backed by `self.cache`) on + /// each gate/PDP entry so policy writes take effect for the next check, + /// even inside a multi-field GraphQL request. This is the single + /// canonical cache-aware grants loader; `authz::engine`'s delegated + /// (non-self-check) path shares the same cache category via + /// `load_decision_context`, keyed the same way. pub async fn effective_grants( &self, pool: &PgPool, ) -> Result>, AppError> { - crate::authz::repo::effective_grants_for_subject(pool, self.entity_id) - .await - .map(std::sync::Arc::new) + crate::cache::cached_or_load( + self.cache.as_deref(), + CacheCategory::Grants, + &cache_keys::grants(self.entity_id), + || crate::authz::repo::effective_grants_for_subject(pool, self.entity_id), + ) + .await + .map(std::sync::Arc::new) } /// The permission ceiling to apply when `subject_id` is the token owner diff --git a/src/graphql/auth.rs b/src/graphql/auth.rs index 84e009b..ed1f4f2 100644 --- a/src/graphql/auth.rs +++ b/src/graphql/auth.rs @@ -94,9 +94,30 @@ impl AuthMutation { .await .map_err(|e| gql_error(crate::error::db_err(e)))?; if let Some(session_id) = auth.session_id { - repo::revoke_session_in_tx(&mut tx, session_id) - .await - .map_err(gql_error)?; + // The revoke must stay inside `tx` so it commits atomically with + // the audit event/outbox row below, but it also needs the cache + // barrier around it so a concurrent reader can't repopulate a + // stale (not-yet-revoked) session entry — see `src/cache/mod.rs`. + let session_key = crate::cache::keys::session(session_id); + if let Some(cache) = state.cache.as_deref() { + cache + .begin( + crate::cache::CacheCategory::Session, + std::slice::from_ref(&session_key), + ) + .await + .map_err(gql_error)?; + } + let result = repo::revoke_session_in_tx(&mut tx, session_id).await; + if let Some(cache) = state.cache.as_deref() { + cache + .end( + crate::cache::CacheCategory::Session, + std::slice::from_ref(&session_key), + ) + .await; + } + result.map_err(gql_error)?; } audit::commit_with_audit( &state.pool, @@ -127,13 +148,26 @@ impl AuthMutation { })?; let state = ctx.data::()?; let keys = state.keys.read().await; + let primary_key = keys.primary.clone(); + drop(keys); - service::refresh_session( - &state.pool, - &state.config, - &keys.primary, - auth.entity_id, - session_id, + // Extends `expires_at` in place for the same session_id; without + // invalidating, a stale cached (shorter) expiry could cause a + // spurious "session expired" false-deny until the entry's own TTL + // catches up — not a security issue, but worth fixing. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || { + service::refresh_session( + &state.pool, + &state.config, + &primary_key, + auth.entity_id, + session_id, + ) + }, ) .await .map(Into::into) @@ -176,7 +210,8 @@ pub(crate) fn gql_error(err: AppError) -> async_graphql::Error { | AppError::Forbidden | AppError::Conflict(_) | AppError::PayloadTooLarge(_) - | AppError::RateLimited { .. } => async_graphql::Error::new(err.to_string()), + | AppError::RateLimited { .. } + | AppError::ServiceUnavailable(_) => async_graphql::Error::new(err.to_string()), } } diff --git a/src/graphql/credentials.rs b/src/graphql/credentials.rs index dbb2178..9530775 100644 --- a/src/graphql/credentials.rs +++ b/src/graphql/credentials.rs @@ -216,32 +216,39 @@ impl CredentialMutation { .into_iter() .map(permission_input_into_model) .collect::>>()?; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::replace_access_token_permissions_in_tx( - &mut tx, - owner_id, - credential_id, - permissions, - ) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: audit_tenant_id, - target_kind: Some("credential"), - target_id: Some(credential_id), - event: "credential.update", - outcome: AuditOutcome::Allow, - details: serde_json::json!({ - "entity_id": owner_id, - "kind": "access_token", - "delegated": delegated, - "credential_id": credential_id - }), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::CredentialCeiling, + std::slice::from_ref(&crate::cache::keys::cred_ceiling(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::replace_access_token_permissions_in_tx( + &mut tx, + owner_id, + credential_id, + permissions, + ) + .await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: audit_tenant_id, + target_kind: Some("credential"), + target_id: Some(credential_id), + event: "credential.update", + outcome: AuditOutcome::Allow, + details: serde_json::json!({ + "entity_id": owner_id, + "kind": "access_token", + "delegated": delegated, + "credential_id": credential_id + }), + }, + ) + .await }, ) .await @@ -256,27 +263,33 @@ impl CredentialMutation { let credential_id = parse_id(credential_id, "credentialId")?; let (owner_id, delegated, audit_tenant_id) = resolve_token_lifecycle_target(state, &auth, credential_id).await?; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::revoke_access_token_in_tx(&mut tx, owner_id, credential_id) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: audit_tenant_id, - target_kind: Some("credential"), - target_id: Some(credential_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({ - "entity_id": owner_id, - "kind": "access_token", - "delegated": delegated, - "credential_id": credential_id - }), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::revoke_access_token_in_tx(&mut tx, owner_id, credential_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: audit_tenant_id, + target_kind: Some("credential"), + target_id: Some(credential_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({ + "entity_id": owner_id, + "kind": "access_token", + "delegated": delegated, + "credential_id": credential_id + }), + }, + ) + .await }, ) .await @@ -390,22 +403,28 @@ impl CredentialMutation { } else { require_credential_management(state, &auth, entity_id).await? }; - let mut tx = state.pool.begin().await.map_err(|e| gql_error(db_err(e)))?; - service::revoke_credential_in_tx(&mut tx, entity_id, credential_id) - .await - .map_err(gql_error)?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id, - target_kind: Some("entity"), - target_id: Some(entity_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({"credential_id": credential_id}), + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(credential_id)), + || async { + let mut tx = state.pool.begin().await.map_err(db_err)?; + service::revoke_credential_in_tx(&mut tx, entity_id, credential_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("entity"), + target_id: Some(entity_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({"credential_id": credential_id}), + }, + ) + .await }, ) .await diff --git a/src/graphql/entities.rs b/src/graphql/entities.rs index 8639955..a5af067 100644 --- a/src/graphql/entities.rs +++ b/src/graphql/entities.rs @@ -298,23 +298,33 @@ impl EntityMutation { .await?; } - repo::update_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - entity_model::UpdateEntity { - name: input.name, - kind: parse_optional_entity_kind(input.kind), - alias: input.alias.into(), - tenant_id, - profile_id, - profile_version_id, - status: input.status.map(Into::into), - attributes: input.attributes, + // Status and tenant are both part of `atom:v1:entity_status:*`'s + // payload (see `src/cache/entries.rs`), so any update invalidates + // it — cheap and correct even when neither actually changed. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || { + repo::update_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + entity_model::UpdateEntity { + name: input.name, + kind: parse_optional_entity_kind(input.kind), + alias: input.alias.into(), + tenant_id, + profile_id, + profile_version_id, + status: input.status.map(Into::into), + attributes: input.attributes, + }, + "entity.update", + details.clone(), + ) }, - "entity.update", - details.clone(), ) .await } @@ -360,12 +370,49 @@ impl EntityMutation { ) .await?; } - repo::delete_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + // Enumerate *before* the delete revokes them — the revoke + // UPDATEs' own `WHERE ... IS NULL`/`= 'active'` filters no + // longer match these rows afterward. + let session_ids = repo::entity_active_session_ids(&state.pool, id).await?; + let credential_ids = repo::entity_active_access_token_ids(&state.pool, id).await?; + let session_keys: Vec = session_ids + .iter() + .map(|sid| crate::cache::keys::session(*sid)) + .collect(); + let credential_keys: Vec = credential_ids + .iter() + .map(|cid| crate::cache::keys::credential(*cid)) + .collect(); + let entity_status_keys = [crate::cache::keys::entity_status(id)]; + // `entity_status` invalidation alone is *not* sufficient: + // `delete_entity` also revokes this entity's sessions and + // access-token credentials in the same transaction, and + // `restoreEntity` deliberately does not reinstate them (an + // identity must re-authenticate after restore). A stale cached + // session/credential survives the tombstoned window untouched + // (masked only by the entity_status miss forcing a fresh + // Postgres check), then becomes a full cache hit again the + // moment `restoreEntity` repopulates entity_status as active — + // despite being revoked in Postgres and meant to stay that way. + crate::cache::invalidate::guarded_multi_mutation( + state.cache.as_deref(), + &[ + ( + crate::cache::CacheCategory::EntityStatus, + &entity_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + (crate::cache::CacheCategory::Credential, &credential_keys), + ], + || { + repo::delete_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + }, ) .await } @@ -406,12 +453,21 @@ impl EntityMutation { let result = async { crate::auth::require_any_capability(&state.pool, &auth, &[("manage", Scope::Platform)]) .await?; - repo::restore_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + // Separate function from `update_entity`/`change_entity_status`, + // so it needs its own `entity_status` invalidation too. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || { + repo::restore_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + }, ) .await } @@ -669,23 +725,30 @@ async fn change_entity_status(ctx: &Context<'_>, id: ID, status: EntityStatus) - ], ) .await?; - repo::update_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - entity_id, - entity_model::UpdateEntity { - name: None, - kind: None, - alias: None, - tenant_id: None, - profile_id: None, - profile_version_id: None, - status: Some(status), - attributes: None, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(entity_id)), + || { + repo::update_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + entity_id, + entity_model::UpdateEntity { + name: None, + kind: None, + alias: None, + tenant_id: None, + profile_id: None, + profile_version_id: None, + status: Some(status), + attributes: None, + }, + event, + details.clone(), + ) }, - event, - details.clone(), ) .await } diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 107dc10..0771ec6 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -394,12 +394,47 @@ impl TenantMutation { let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::soft_delete_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - Some(auth.entity_id), + // Enumerate *before* the delete revokes them — the revoke + // UPDATE's own `WHERE revoked_at IS NULL` no longer matches these + // rows afterward. + let session_ids = + tenant_repo::tenant_active_session_ids(&state.pool, tenant_id).await?; + let session_keys: Vec = session_ids + .iter() + .map(|id| crate::cache::keys::session(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + // `soft_delete_tenant` is a separate function from + // `change_tenant_status` (it also bulk-revokes sessions and + // credentials). `tenant_status` invalidation alone is *not* + // sufficient: while the tenant stays deleted, a stale cached + // session survives untouched (masked only by the tenant_status + // miss forcing a fresh Postgres check), but the moment + // `restoreTenant` repopulates tenant_status as active again, that + // stale session becomes a full cache hit and authenticates + // despite being revoked in Postgres. Credentials don't need the + // same treatment here — `restore_tenant`'s own invalidation + // (see `tenant_restore_reactivated_credential_ids`) already + // covers the credential side by the time a restore could ever + // matter. + crate::cache::invalidate::guarded_multi_mutation( + state.cache.as_deref(), + &[ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + ], + || { + tenant_repo::soft_delete_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + }, ) .await } @@ -438,12 +473,42 @@ impl TenantMutation { let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::restore_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - Some(auth.entity_id), + // `restore_tenant` is a separate function from + // `change_tenant_status` and touches two cache categories in one + // transaction: the tenant's own status, and every credential it + // reactivates. Both need invalidating — unlike the + // tenant-status-only case in `delete_tenant`/ + // `change_tenant_status`, a stale cached credential status is + // checked *first* in `verify_api_key_snapshot` and isn't + // overridden by a fresher tenant_status check afterward, so it + // must be invalidated explicitly or a just-restored API key + // keeps getting denied until its own cache entry's TTL expires. + let reactivated_credential_ids = + tenant_repo::tenant_restore_reactivated_credential_ids(&state.pool, tenant_id) + .await?; + let credential_keys: Vec = reactivated_credential_ids + .iter() + .map(|id| crate::cache::keys::credential(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + crate::cache::invalidate::guarded_multi_mutation( + state.cache.as_deref(), + &[ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Credential, &credential_keys), + ], + || { + tenant_repo::restore_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + }, ) .await } @@ -568,10 +633,16 @@ impl TenantMutation { async fn accept_tenant_invitation(&self, ctx: &Context<'_>, tenant_id: ID) -> Result { let auth = require_auth(ctx)?; let state = ctx.data::()?; - tenant_repo::accept_invitation( - &state.pool, - parse_id(tenant_id, "tenantId")?, - auth.entity_id, + let tenant_id = parse_id(tenant_id, "tenantId")?; + // Invitation acceptance grants tenant membership (and possibly a + // role) to `auth.entity_id` — easy to miss since it doesn't go + // through the "obvious" policy/role-assignment mutation entry + // points. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || tenant_repo::accept_invitation(&state.pool, tenant_id, auth.entity_id), ) .await .map_err(gql_error)?; @@ -585,10 +656,14 @@ impl TenantMutation { ) -> Result { let auth = require_auth(ctx)?; let state = ctx.data::()?; - let tenant_id = - tenant_repo::accept_invitation_token(&state.pool, &input.token, auth.entity_id) - .await - .map_err(gql_error)?; + let tenant_id = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || tenant_repo::accept_invitation_token(&state.pool, &input.token, auth.entity_id), + ) + .await + .map_err(gql_error)?; Ok(ID::from(tenant_id.to_string())) } @@ -650,12 +725,19 @@ impl TenantMutation { Scope::Tenant(tenant_id), ) .await?; - tenant_repo::remove_tenant_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + tenant_repo::remove_tenant_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + entity_id, + ) + }, ) .await } @@ -701,13 +783,20 @@ impl TenantMutation { Scope::Tenant(tenant_id), ) .await?; - tenant_repo::add_tenant_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - entity_id, - role_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + tenant_repo::add_tenant_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + entity_id, + role_id, + ) + }, ) .await } @@ -742,13 +831,25 @@ async fn change_tenant_status(ctx: &Context<'_>, id: ID, status: TenantStatus) - let status_detail = status.clone(); let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - tenant_repo::change_tenant_status_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - status, - event, + // Only `tenant_status`, not `grants`: the PDP's tenant-lifecycle deny + // check runs before grant matching (see `authz::engine:: + // load_decision_context`), so a stale tenant-membership-implicit + // grant inside a cached `grants` entry is harmless as long as this + // key itself invalidates. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::TenantStatus, + std::slice::from_ref(&crate::cache::keys::tenant_status(tenant_id)), + || { + tenant_repo::change_tenant_status_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, + ) + }, ) .await } diff --git a/src/identity/handlers.rs b/src/identity/handlers.rs index b48a64c..b889616 100644 --- a/src/identity/handlers.rs +++ b/src/identity/handlers.rs @@ -193,7 +193,7 @@ pub async fn reset_password( State(state): State, Json(req): Json, ) -> Result { - service::reset_password(&state.pool, req).await?; + service::reset_password(&state.pool, state.cache.as_deref(), req).await?; Ok(StatusCode::NO_CONTENT) } @@ -244,7 +244,29 @@ pub async fn logout( ) -> Result { let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; if let Some(session_id) = auth.session_id { - repo::revoke_session_in_tx(&mut tx, session_id).await?; + // See `src/graphql/auth.rs`'s `logout` for why this is inlined + // rather than going through `guarded_mutation`: the revoke must stay + // inside `tx` for outbox atomicity, but still needs the cache + // barrier around it. + let session_key = crate::cache::keys::session(session_id); + if let Some(cache) = state.cache.as_deref() { + cache + .begin( + crate::cache::CacheCategory::Session, + std::slice::from_ref(&session_key), + ) + .await?; + } + let result = repo::revoke_session_in_tx(&mut tx, session_id).await; + if let Some(cache) = state.cache.as_deref() { + cache + .end( + crate::cache::CacheCategory::Session, + std::slice::from_ref(&session_key), + ) + .await; + } + result?; } audit::commit_with_audit( &state.pool, @@ -398,7 +420,13 @@ pub async fn update_entity( scope_for_tenant(existing.tenant_id), ) .await?; - let entity = repo::update_entity(&state.pool, id, req).await?; + let entity = crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || repo::update_entity(&state.pool, id, req), + ) + .await?; Ok(Json(entity)) } @@ -417,7 +445,13 @@ pub async fn delete_entity( ) .await?; } - repo::delete_entity(&state.pool, id, Some(auth.entity_id)).await?; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::EntityStatus, + std::slice::from_ref(&crate::cache::keys::entity_status(id)), + || repo::delete_entity(&state.pool, id, Some(auth.entity_id)), + ) + .await?; Ok(StatusCode::NO_CONTENT) } @@ -548,20 +582,31 @@ pub async fn revoke_credential( } else { require_credential_management(&state, &auth, entity_id).await? }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - service::revoke_credential_in_tx(&mut tx, entity_id, cred_id).await?; - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id, - target_kind: Some("entity"), - target_id: Some(entity_id), - event: "credential.revoke", - outcome: AuditOutcome::Allow, - details: serde_json::json!({"credential_id": cred_id}), + // `revoke_credential` isn't kind-filtered — it can revoke an access-token + // credential too, not just password, so it must invalidate the same + // `atom:v1:credential:*` key `revoke_access_token` does. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Credential, + std::slice::from_ref(&crate::cache::keys::credential(cred_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + service::revoke_credential_in_tx(&mut tx, entity_id, cred_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("entity"), + target_id: Some(entity_id), + event: "credential.revoke", + outcome: AuditOutcome::Allow, + details: serde_json::json!({"credential_id": cred_id}), + }, + ) + .await }, ) .await?; diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 24a9dfb..b949fe3 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -695,6 +695,42 @@ fn entity_kind_as_str(kind: &EntityKind) -> &'static str { } } +/// The exact set of active session ids `delete_entity` is about to revoke — +/// mirrors that function's `UPDATE sessions` `WHERE` clause precisely, so +/// callers can invalidate `atom:v1:session:*` cache entries for them *before* +/// the delete runs (afterward, `revoked_at IS NULL` no longer matches these +/// rows). See `src/cache/mod.rs`'s consistency model. +pub async fn entity_active_session_ids( + pool: &PgPool, + entity_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar("SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL") + .bind(entity_id) + .fetch_all(pool) + .await + .map_err(db_err) +} + +/// The exact set of active access-token credential ids `delete_entity` is +/// about to revoke — restricted to the one credential kind this codebase +/// caches under `CacheCategory::Credential` (certificates are tracked via the +/// CRL instead, not this cache). Callers invalidate `atom:v1:credential:*` +/// entries for these *before* the delete runs. See `src/cache/mod.rs`'s +/// consistency model. +pub async fn entity_active_access_token_ids( + pool: &PgPool, + entity_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar( + "SELECT id FROM credentials WHERE entity_id = $1 AND status = 'active' AND kind = $2", + ) + .bind(entity_id) + .bind(crate::models::enums::CredentialKind::AccessToken) + .fetch_all(pool) + .await + .map_err(db_err) +} + /// Soft-delete an entity: mark it inactive, set the tombstone, and immediately /// cut off access by revoking its credentials and active sessions. Physical /// removal is deferred to the purge cron. Hard delete (the old behavior) relied @@ -1209,6 +1245,121 @@ pub async fn list_groups(pool: &PgPool, params: ListGroups) -> Result, + events_enabled: bool, + actor_id: Option, + id: Uuid, + req: UpdateGroup, + event_name: &str, + audit_details: Value, +) -> Result { + let attributes = req.attributes.map(normalize_attributes); + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let Some(tenant_id) = tenant_id else { + return Err(AppError::not_found(format!("group {id} not found"))); + }; + crate::tenants::repo::lock_optional_active_tenant(tx, tenant_id).await?; + sqlx::query_as::<_, Group>( + r#"WITH p AS ( + UPDATE principal_groups + SET name = COALESCE($2, name), + description = COALESCE($3, description), + status = COALESCE($4, status), + attributes = COALESCE($5, attributes), + updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING id, name, tenant_id, 'principal'::text AS group_type, description, + (SELECT parent_id FROM principal_group_hierarchy WHERE child_id = principal_groups.id) AS parent_id, + status, attributes, deleted_at, deleted_by, created_at, updated_at + ), + o AS ( + UPDATE object_groups + SET name = COALESCE($2, name), + description = COALESCE($3, description), + status = COALESCE($4, status), + attributes = COALESCE($5, attributes), + updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING id, name, tenant_id, 'object'::text AS group_type, description, + (SELECT parent_id FROM object_group_hierarchy WHERE child_id = object_groups.id) AS parent_id, + status, attributes, deleted_at, deleted_by, created_at, updated_at + ) + SELECT * FROM p + UNION ALL + SELECT * FROM o"#, + ) + .bind(id) + .bind(req.name) + .bind(req.description) + .bind(req.status) + .bind(attributes) + .fetch_one(&mut **tx) + .await + .map_err(|e| match e { + sqlx::Error::RowNotFound => AppError::not_found(format!("group {id} not found")), + other => AppError::Database(other), + })?; + + let meta = crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: event_name, + }; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &audit_details).await?; + Ok(group) +} + +pub async fn update_group(pool: &PgPool, id: Uuid, req: UpdateGroup) -> Result { + update_group_with_audit( + pool, + false, + None, + id, + req, + "group.update", + serde_json::json!({}), + ) + .await +} + +pub async fn update_group_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + req: UpdateGroup, + event_name: &str, + audit_details: Value, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let group = update_group_in_tx( + &mut tx, + events_enabled, + actor_id, + id, + req, + event_name, + audit_details, + ) + .await?; + tx.commit().await.map_err(db_err)?; + Ok(group) +} + pub async fn set_group_parent( pool: &PgPool, child_id: Uuid, @@ -1224,12 +1375,32 @@ pub async fn set_group_parent_with_audit( child_id: Uuid, parent_id: Uuid, ) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + set_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id, parent_id).await?; + tx.commit().await.map_err(db_err)?; + get_group(pool, child_id).await +} + +/// [`set_group_parent`]'s body, minus opening/committing its own +/// transaction and the post-commit `get_group` read — see +/// `authz::repo::create_role_assignment_in_tx`'s doc comment for the caller +/// contract (the resolver locks `child_id`'s closure via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same +/// `tx` first — reparenting `child_id` only changes what it and its +/// descendants inherit from above, never `group_hierarchy` rows below it, so +/// that closure is exactly what this mutation can affect). +pub(crate) async fn set_group_parent_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + child_id: Uuid, + parent_id: Uuid, +) -> Result<(), AppError> { if child_id == parent_id { return Err(AppError::bad_request("group cannot be its own parent")); } use sqlx::Row; - let mut tx = pool.begin().await.map_err(db_err)?; let child = sqlx::query( r#"SELECT tenant_id, group_type FROM groups @@ -1238,7 +1409,7 @@ pub async fn set_group_parent_with_audit( LIMIT 1"#, ) .bind(child_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("group {child_id} not found")), @@ -1252,7 +1423,7 @@ pub async fn set_group_parent_with_audit( LIMIT 1"#, ) .bind(parent_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => { @@ -1288,7 +1459,7 @@ pub async fn set_group_parent_with_audit( } else { "object_groups" }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, child_tenant_id).await?; + crate::tenants::repo::lock_optional_active_tenant(tx, child_tenant_id).await?; let lock_sql = format!( "SELECT id FROM {group_table} WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL @@ -1296,7 +1467,7 @@ pub async fn set_group_parent_with_audit( ); let locked_ids: Vec = sqlx::query_scalar(&lock_sql) .bind(vec![child_id, parent_id]) - .fetch_all(&mut *tx) + .fetch_all(&mut **tx) .await .map_err(db_err)?; if locked_ids.len() != 2 { @@ -1316,7 +1487,7 @@ pub async fn set_group_parent_with_audit( let creates_cycle: bool = sqlx::query_scalar(&creates_cycle_sql) .bind(parent_id) .bind(child_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if creates_cycle { @@ -1335,7 +1506,7 @@ pub async fn set_group_parent_with_audit( .bind(parent_id) .bind(child_id) .bind(child_tenant_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -1346,7 +1517,7 @@ pub async fn set_group_parent_with_audit( ORDER BY id FOR UPDATE"#, ) .bind(vec![child_id, parent_id]) - .fetch_all(&mut *tx) + .fetch_all(&mut **tx) .await .map_err(db_err)?; if principal_ids.len() == 2 { @@ -1361,13 +1532,12 @@ pub async fn set_group_parent_with_audit( .bind(parent_id) .bind(child_id) .bind(child_tenant_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } } - let group = fetch_group(&mut *tx, child_id).await?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id: child_tenant_id, @@ -1376,8 +1546,8 @@ pub async fn set_group_parent_with_audit( event: "group.parent.set", }; let details = serde_json::json!({ "parent_id": parent_id }); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(group) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(()) } pub async fn remove_group_parent(pool: &PgPool, child_id: Uuid) -> Result<(), AppError> { @@ -1391,28 +1561,41 @@ pub async fn remove_group_parent_with_audit( child_id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + remove_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id).await?; + tx.commit().await.map_err(db_err) +} + +/// [`remove_group_parent`]'s body, minus opening/committing its own +/// transaction — see [`set_group_parent_in_tx`]'s doc comment for the caller +/// contract. +pub(crate) async fn remove_group_parent_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + child_id: Uuid, +) -> Result<(), AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { return Err(AppError::not_found(format!("group {child_id} not found"))); }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, tenant_id).await?; + crate::tenants::repo::lock_optional_active_tenant(tx, tenant_id).await?; let object_locked: Option = sqlx::query_scalar( "SELECT id FROM object_groups WHERE id = $1 AND deleted_at IS NULL FOR UPDATE", ) .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let principal_locked: Option = sqlx::query_scalar( "SELECT id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL FOR UPDATE", ) .bind(child_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if object_locked.is_none() && principal_locked.is_none() { @@ -1425,7 +1608,7 @@ pub async fn remove_group_parent_with_audit( DELETE FROM object_group_hierarchy WHERE child_id = $1"#, ) .bind(child_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; let meta = crate::audit::AuditMeta { @@ -1436,7 +1619,7 @@ pub async fn remove_group_parent_with_audit( event: "group.parent.remove", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(()) } @@ -1462,105 +1645,35 @@ pub async fn list_child_groups( .await } -pub async fn update_group_with_audit( +pub async fn delete_group_with_audit( pool: &PgPool, events_enabled: bool, actor_id: Option, id: Uuid, - req: UpdateGroup, - event_name: &str, - audit_details: Value, -) -> Result { - let attributes = req.attributes.clone().map(normalize_attributes); + deleted_by: Option, +) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - let tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") - .bind(id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - let Some(tenant_id) = tenant_id else { - return Err(AppError::not_found(format!("group {id} not found"))); - }; - crate::tenants::repo::lock_optional_active_tenant(&mut tx, tenant_id).await?; - let group = sqlx::query_as::<_, Group>( - r#"WITH p AS ( - UPDATE principal_groups - SET name = COALESCE($2, name), - description = COALESCE($3, description), - status = COALESCE($4, status), - attributes = COALESCE($5, attributes), - updated_at = now() - WHERE id = $1 AND deleted_at IS NULL - RETURNING id, name, tenant_id, 'principal'::text AS group_type, description, - (SELECT parent_id FROM principal_group_hierarchy WHERE child_id = principal_groups.id) AS parent_id, - status, attributes, deleted_at, deleted_by, created_at, updated_at - ), - o AS ( - UPDATE object_groups - SET name = COALESCE($2, name), - description = COALESCE($3, description), - status = COALESCE($4, status), - attributes = COALESCE($5, attributes), - updated_at = now() - WHERE id = $1 AND deleted_at IS NULL - RETURNING id, name, tenant_id, 'object'::text AS group_type, description, - (SELECT parent_id FROM object_group_hierarchy WHERE child_id = object_groups.id) AS parent_id, - status, attributes, deleted_at, deleted_by, created_at, updated_at - ) - SELECT * FROM p - UNION ALL - SELECT * FROM o"#, - ) - .bind(id) - .bind(req.name) - .bind(req.description) - .bind(req.status) - .bind(attributes) - .fetch_one(&mut *tx) - .await - .map_err(|e| match e { - sqlx::Error::RowNotFound => AppError::not_found(format!("group {id} not found")), - other => AppError::Database(other), - })?; - - let meta = crate::audit::AuditMeta { - actor_entity_id: actor_id, - tenant_id: group.tenant_id, - target_kind: "group", - target_id: Some(id), - event: event_name, - }; - let details = audit_details; - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(group) + delete_group_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err) } -pub async fn update_group(pool: &PgPool, id: Uuid, req: UpdateGroup) -> Result { - update_group_with_audit( - pool, - false, - None, - id, - req, - "group.update", - serde_json::json!({}), - ) - .await -} - -pub async fn delete_group_with_audit( - pool: &PgPool, +/// [`delete_group`]'s body, minus opening/committing its own transaction — +/// see `authz::repo::create_role_assignment_in_tx`'s doc comment for the +/// caller contract (the resolver locks this group's closure via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same +/// `tx` first — `group_hierarchy` rows aren't touched by a soft delete, only +/// `deleted_at`, so the closure is unaffected by this mutation's own effect). +pub(crate) async fn delete_group_in_tx( + tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, deleted_by: Option, ) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { @@ -1582,7 +1695,7 @@ pub async fn delete_group_with_audit( ) .bind(id) .bind(deleted_by) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if result.is_none() { @@ -1597,7 +1710,7 @@ pub async fn delete_group_with_audit( event: "group.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(()) } @@ -1616,9 +1729,45 @@ pub async fn restore_group_with_audit( id: Uuid, restored_by: Option, ) -> Result<(), AppError> { - let _ = restored_by; let mut tx = pool.begin().await.map_err(db_err)?; + restore_group_in_tx(&mut tx, events_enabled, actor_id, id, restored_by).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation inside `restore_group_in_tx` via `observe_in_tx`. + let tenant_id = get_group(pool, id).await.ok().and_then(|g| g.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("group"), + target_id: Some(id), + event: "group.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} +/// [`restore_group`]'s body, minus opening/committing its own transaction — +/// see `authz::repo::create_role_assignment_in_tx`'s doc comment for the +/// caller contract (the resolver locks this group's closure via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same +/// `tx` first — that lock works regardless of the group's own `deleted_at` +/// status, so it applies here unchanged). +pub(crate) async fn restore_group_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result<(), AppError> { + let _ = restored_by; let tenant_info: Option<(Option, bool)> = sqlx::query_as( "SELECT g.tenant_id, (t.deleted_at IS NOT NULL) FROM groups g @@ -1626,7 +1775,7 @@ pub async fn restore_group_with_audit( WHERE g.id = $1 AND g.deleted_at IS NOT NULL", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let (tenant_id, _is_tenant_deleted) = match tenant_info { @@ -1657,20 +1806,18 @@ pub async fn restore_group_with_audit( SELECT id FROM o"#, ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(restore_conflict)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("group"), + target_kind: "group", target_id: Some(id), event: "group.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; Ok(()) } diff --git a/src/identity/service.rs b/src/identity/service.rs index efd9f2c..71b772b 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -727,6 +727,7 @@ pub async fn request_password_reset( pub async fn reset_password( pool: &PgPool, + cache: Option<&crate::cache::CacheClient>, req: PasswordResetConfirmRequest, ) -> Result<(), AppError> { if let Some(confirm_password) = req.confirm_password.as_deref() { @@ -773,53 +774,75 @@ pub async fn reset_password( .map_err(db_err)?; let password_hash = hash_secret(req.password.as_bytes())?; - let mut tx = pool.begin().await.map_err(db_err)?; - if super::repo::lock_active_entity(&mut tx, entity_id) - .await? - .is_none() - { - return Err(AppError::bad_request("invalid password reset token")); - } - let updated = sqlx::query( - "UPDATE password_reset_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", - ) - .bind(token_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - if updated.rows_affected() == 0 { - return Err(AppError::bad_request("password reset token expired")); - } - sqlx::query( - r#"UPDATE credentials - SET status = 'revoked' - WHERE entity_id = $1 AND kind = 'password' AND status = 'active'"#, + // Every currently-active session is about to be bulk-revoked below (by + // entity_id, not by session_id) — collect their cache keys first so the + // barrier can cover all of them, not just one. + let session_keys: Vec = sqlx::query_scalar::<_, Uuid>( + "SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL", ) .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - sqlx::query( - r#"INSERT INTO credentials (id, entity_id, kind, identifier, secret_hash) - VALUES ($1, $2, $3, $4, $5)"#, - ) - .bind(Uuid::new_v4()) - .bind(entity_id) - .bind(CredentialKind::Password) - .bind(email) - .bind(password_hash) - .execute(&mut *tx) + .fetch_all(pool) .await - .map_err(db_err)?; - sqlx::query( - "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", + .map_err(db_err)? + .into_iter() + .map(crate::cache::keys::session) + .collect(); + + crate::cache::invalidate::guarded_mutation( + cache, + crate::cache::CacheCategory::Session, + &session_keys, + || async { + let mut tx = pool.begin().await.map_err(db_err)?; + if super::repo::lock_active_entity(&mut tx, entity_id) + .await? + .is_none() + { + return Err(AppError::bad_request("invalid password reset token")); + } + let updated = sqlx::query( + "UPDATE password_reset_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", + ) + .bind(token_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + if updated.rows_affected() == 0 { + return Err(AppError::bad_request("password reset token expired")); + } + sqlx::query( + r#"UPDATE credentials + SET status = 'revoked' + WHERE entity_id = $1 AND kind = 'password' AND status = 'active'"#, + ) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + r#"INSERT INTO credentials (id, entity_id, kind, identifier, secret_hash) + VALUES ($1, $2, $3, $4, $5)"#, + ) + .bind(Uuid::new_v4()) + .bind(entity_id) + .bind(CredentialKind::Password) + .bind(&email) + .bind(password_hash) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", + ) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(()) + }, ) - .bind(entity_id) - .execute(&mut *tx) .await - .map_err(db_err)?; - tx.commit().await.map_err(db_err)?; - Ok(()) } pub async fn oauth_start( diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index 99816f9..67d3779 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -547,6 +547,38 @@ pub async fn update_tenant( update_tenant_with_audit(pool, false, None, id, req, updated_by).await } +/// The exact set of active session ids `soft_delete_tenant` is about to +/// revoke — mirrors that function's `UPDATE sessions` `WHERE` clause +/// precisely, so callers can invalidate `atom:v1:session:*` cache entries for +/// them *before* the delete runs (afterward, `revoked_at IS NULL` no longer +/// matches these rows). See `src/cache/mod.rs`'s consistency model. +pub async fn tenant_active_session_ids( + pool: &PgPool, + tenant_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar( + r#"SELECT id FROM sessions + WHERE revoked_at IS NULL + AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)"#, + ) + .bind(tenant_id) + .fetch_all(pool) + .await + .map_err(db_err) +} + +/// Soft-delete a tenant: mark `status = deleted`, stamp the tombstone, and +/// immediately revoke every active credential and session of entities in the +/// tenant. Physical removal (and the entity cascade) is deferred to the purge +/// cron. +pub async fn soft_delete_tenant( + pool: &PgPool, + id: Uuid, + deleted_by: Option, +) -> Result { + soft_delete_tenant_with_audit(pool, false, None, id, deleted_by).await +} + pub async fn soft_delete_tenant_with_audit( pool: &PgPool, events_enabled: bool, @@ -618,12 +650,51 @@ pub async fn soft_delete_tenant_with_audit( Ok(tenant) } -pub async fn soft_delete_tenant( +/// Reverse a tenant soft delete within the retention window. Reactivates the +/// tenant and clears its tombstone; its children (entities, groups, roles, +/// resources) were never individually tombstoned by `soft_delete_tenant` — they +/// were hidden only via the tenant's `deleted_at` — so they become visible again +/// automatically. +/// +/// To make the restored tenant operational, the non-certificate child +/// credentials (passwords, API keys) that *this* delete revoked — identified by +/// the `tenant_deleted` revocation marker — are reactivated, so members can log +/// in with their existing secrets. Certificates stay revoked (their revocation +/// is published via the CRL and cannot be safely undone — re-issue is required), +/// and sessions stay revoked, so a fresh login is required. Credentials revoked +/// earlier for other reasons (e.g. an individually soft-deleted child) are left +/// untouched. +/// +/// Fails with a conflict if the tenant name/alias was re-taken by a live tenant +/// during the retention window. +/// The exact set of credential ids `restore_tenant` is about to reactivate — +/// mirrors that function's `UPDATE credentials` `WHERE` clause precisely, so +/// callers can invalidate `atom:v1:credential:*` cache entries for them +/// *before* running the restore (see `src/cache/mod.rs`'s consistency +/// model). A stale cached "revoked" credential is a false-deny (fails +/// closed, not a security hole) but is still worth fixing: unlike a tenant +/// or entity status flip, which a later-checked fresh field can catch +/// regardless of earlier stale fields, `verify_api_key_snapshot` checks the +/// credential's own status first — a stale value there is never overridden +/// by anything checked afterward. +pub async fn tenant_restore_reactivated_credential_ids( pool: &PgPool, - id: Uuid, - deleted_by: Option, -) -> Result { - soft_delete_tenant_with_audit(pool, false, None, id, deleted_by).await + tenant_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar( + r#"SELECT c.id + FROM credentials c + JOIN entities e ON c.entity_id = e.id + WHERE e.tenant_id = $1 + AND e.deleted_at IS NULL + AND c.status = 'revoked' + AND c.kind <> 'certificate' + AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, + ) + .bind(tenant_id) + .fetch_all(pool) + .await + .map_err(db_err) } pub async fn restore_tenant_with_audit( diff --git a/tests/m21_soft_delete.rs b/tests/m21_soft_delete.rs index dea6cc1..b475c7e 100644 --- a/tests/m21_soft_delete.rs +++ b/tests/m21_soft_delete.rs @@ -230,6 +230,7 @@ async fn deleted_entity_cannot_consume_existing_password_reset_token() { assert!( service::reset_password( &pool, + None, PasswordResetConfirmRequest { token, password: "replacement-password".to_string(), From 4d7fa44ca5d169128ebe55706cb3ef0d16ddcb83 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 15:14:47 +0300 Subject: [PATCH 03/19] Cache AuthZ grant expansion with race-safe group/role invalidation Caches each subject's effective grant expansion, invalidated whenever a direct policy, role assignment, role's linked permission blocks, or group membership/hierarchy/status changes it. For a group or role subject, enumerating "which subjects are affected" and invalidating their cache entries is done under the same Postgres row lock add_group_member/remove_group_member take, closing a race where a membership change concurrent with the enumeration could leave a new member's stale grant cached until TTL. That lock is acquired in a fixed, consistent order (role before group) everywhere it's needed, to rule out a lock-order deadlock between concurrent admin operations. --- src/authz/engine.rs | 49 ++- src/authz/repo.rs | 480 +++++++++++++++++++++++----- src/graphql/groups.rs | 331 ++++++++++++++++--- src/graphql/policies.rs | 407 ++++++++++++++++++++--- src/identity/repo.rs | 2 +- tests/m12_graphql_identity.rs | 1 + tests/m26_audit_event_publishing.rs | 2 +- 7 files changed, 1090 insertions(+), 182 deletions(-) diff --git a/src/authz/engine.rs b/src/authz/engine.rs index f6a5c7f..96057b3 100644 --- a/src/authz/engine.rs +++ b/src/authz/engine.rs @@ -4,6 +4,7 @@ use uuid::Uuid; use crate::{ authz::conditions::conditions_match, + cache::{CacheCategory, CacheClient}, error::AppError, models::{ access::{ @@ -260,6 +261,7 @@ async fn load_decision_context( pool: &PgPool, req: &AuthzRequest, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { let Some(entity) = repo::load_authz_subject(pool, req.subject_id).await? else { return Ok(denied(AuthzResponse::deny("subject not found"), None, None)); @@ -342,12 +344,21 @@ async fn load_decision_context( // group membership already resolved recursively, each grant carrying its own // scope/effect/conditions. One match loop replaces the direct/role split. // Reuse the caller's freshly loaded grants when supplied so one PDP - // evaluation does not expand the same self-subject twice. + // evaluation does not expand the same self-subject twice. Otherwise (a + // delegated check about a subject other than the caller) this is the + // canonical cache-aware grants loader shared with `AuthContext:: + // effective_grants` — see `src/cache/mod.rs` for the consistency model. let grants = match cached_grants { Some(grants) => grants, - None => { - std::sync::Arc::new(repo::effective_grants_for_subject(pool, req.subject_id).await?) - } + None => std::sync::Arc::new( + crate::cache::cached_or_load( + cache, + CacheCategory::Grants, + &crate::cache::keys::grants(req.subject_id), + || repo::effective_grants_for_subject(pool, req.subject_id), + ) + .await?, + ), }; Ok(DecisionContext::Ready(Box::new(ReadyContext { @@ -382,28 +393,43 @@ fn scope_target<'a>( /// here from the caller's `AuthContext` (`ceiling_for`), never passed by hand, so /// a call site cannot forget to apply it. The caller's freshly loaded grants are /// reused when the checked subject is the caller itself. +/// +/// The caller's `AuthContext` carries the cache handle backing the +/// Redis-backed grant-expansion cache (see `src/cache/mod.rs`) — a `None` +/// cache (caching disabled, or `AuthContext::default()` in tests) reproduces +/// pre-caching behavior exactly. pub async fn evaluate( pool: &PgPool, req: &AuthzRequest, auth: &crate::auth::AuthContext, ) -> Result { + let cache = auth.cache.as_deref(); let cached_grants = if req.subject_id == auth.entity_id { Some(auth.effective_grants(pool).await?) } else { None }; - evaluate_prepared(pool, req, auth.ceiling_for(req.subject_id), cached_grants).await + evaluate_prepared( + pool, + req, + auth.ceiling_for(req.subject_id), + cached_grants, + cache, + ) + .await } /// Low-level PDP entry taking an explicit ceiling. For unit/parity tests and the /// engine's own internals; production code must call [`evaluate`], which derives -/// the ceiling from the authenticated context. +/// the ceiling from the authenticated context. Always uncached — tests exercise +/// the PDP's decision logic directly; cache behavior has its own dedicated test +/// suite against the real `evaluate`/`explain` entry points. pub async fn evaluate_with_ceiling( pool: &PgPool, req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, ) -> Result { - evaluate_prepared(pool, req, ceiling, None).await + evaluate_prepared(pool, req, ceiling, None, None).await } async fn evaluate_prepared( @@ -411,9 +437,10 @@ async fn evaluate_prepared( req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { let start = std::time::Instant::now(); - let result = evaluate_inner(pool, req, ceiling, cached_grants).await; + let result = evaluate_inner(pool, req, ceiling, cached_grants, cache).await; if let Ok(response) = &result { crate::metrics::record_decision(start.elapsed(), response.allowed); } @@ -425,8 +452,9 @@ async fn evaluate_inner( req: &AuthzRequest, ceiling: Option<&repo::CredentialCeiling>, cached_grants: Option>>, + cache: Option<&CacheClient>, ) -> Result { - let ctx = match load_decision_context(pool, req, cached_grants).await? { + let ctx = match load_decision_context(pool, req, cached_grants, cache).await? { DecisionContext::Denied(denied) => return Ok(denied.response), DecisionContext::Ready(ctx) => ctx, }; @@ -552,13 +580,14 @@ pub async fn explain( req: &AuthzRequest, auth: &crate::auth::AuthContext, ) -> Result { + let cache = auth.cache.as_deref(); let cached_grants = if req.subject_id == auth.entity_id { Some(auth.effective_grants(pool).await?) } else { None }; let ceiling = auth.ceiling_for(req.subject_id); - let ctx = match load_decision_context(pool, req, cached_grants).await? { + let ctx = match load_decision_context(pool, req, cached_grants, cache).await? { DecisionContext::Denied(denied) => { return Ok(AuthzExplainResponse { allowed: false, diff --git a/src/authz/repo.rs b/src/authz/repo.rs index 8692372..4d4ceac 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet}; use chrono::Utc; +use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::{PgPool, Postgres, Transaction}; use uuid::Uuid; @@ -826,7 +827,7 @@ fn parent_group_id_from_value(value: &Value) -> Result, AppError> { /// /// This is the single canonical grant representation consumed by the PDP and /// (incrementally) the other authorization readers. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct EffectiveGrant { /// The assignment that confers this grant: the `direct_policies.id` or the /// `role_assignments.id` row. With shared blocks this is what identifies @@ -861,7 +862,7 @@ pub struct EffectiveGrant { /// `scoped` records intent independently of `entries`: a scoped token whose limit /// rows were deleted yields `entries = []` and must fail closed (deny everything), /// never silently widen to the owner's full authority. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CredentialCeiling { pub entries: Vec, } @@ -1225,6 +1226,33 @@ pub async fn replace_role_permission_block_links_with_audit( actor_id: Option, role_id: Uuid, permission_block_ids: &[Uuid], +) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + replace_role_permission_block_links_in_tx( + pool, + &mut tx, + events_enabled, + actor_id, + role_id, + permission_block_ids, + ) + .await?; + tx.commit().await.map_err(db_err) +} + +/// [`replace_role_permission_block_links`]'s body, minus opening/committing +/// its own transaction — see [`create_role_assignment_in_tx`]'s doc comment +/// for the caller contract (the resolver locks the role, and every group +/// currently assigned it, via [`lock_role_and_collect_grants_keys`] on this +/// same `tx` first — `lock_role` below then just re-acquires that same, +/// already-held role lock). +pub(crate) async fn replace_role_permission_block_links_in_tx( + pool: &PgPool, + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + role_id: Uuid, + permission_block_ids: &[Uuid], ) -> Result<(), AppError> { let role_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") @@ -1256,18 +1284,17 @@ pub async fn replace_role_permission_block_links_with_audit( )); } } - let mut tx = pool.begin().await.map_err(db_err)?; - lock_role(&mut tx, role_id).await?; + lock_role(tx, role_id).await?; // Validate under the role lock so a concurrent role assignment cannot commit // a prohibited combination against stale state: any other role-link or // assignment mutator blocks on this lock and re-validates against our result. // Runs on this transaction's own connection — borrowing a second one from // the pool here would deadlock a saturated pool. - crate::guardrails::validate_role_permission_block_links(&mut tx, role_id, &unique_block_ids) + crate::guardrails::validate_role_permission_block_links(&mut *tx, role_id, &unique_block_ids) .await?; sqlx::query("DELETE FROM role_permission_blocks WHERE role_id = $1") .bind(role_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -1279,12 +1306,12 @@ pub async fn replace_role_permission_block_links_with_audit( ) .bind(role_id) .bind(permission_block_id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } - crate::audit::commit_with_observation( + crate::audit::observe_in_tx( tx, events_enabled, &crate::audit::AuditMeta { @@ -2797,10 +2824,26 @@ pub async fn delete_role_with_audit( deleted_by: Option, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + delete_role_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err) +} + +/// [`delete_role`]'s body, minus opening/committing its own transaction — +/// see [`create_role_assignment_in_tx`]'s doc comment for the caller +/// contract (the resolver locks the role, and every group currently +/// assigned it, via [`lock_role_and_collect_grants_keys`] on this same `tx` +/// first). +pub(crate) async fn delete_role_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + deleted_by: Option, +) -> Result<(), AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = tenant_id else { @@ -2812,7 +2855,7 @@ pub async fn delete_role_with_audit( ) .bind(id) .bind(deleted_by) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; if result.rows_affected() == 0 { @@ -2826,7 +2869,7 @@ pub async fn delete_role_with_audit( event: "role.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(()) } @@ -2845,9 +2888,44 @@ pub async fn restore_role_with_audit( id: Uuid, restored_by: Option, ) -> Result<(), AppError> { - let _ = restored_by; let mut tx = pool.begin().await.map_err(db_err)?; + restore_role_in_tx(&mut tx, events_enabled, actor_id, id, restored_by).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation inside `restore_role_in_tx` via `observe_in_tx`. + let tenant_id = get_role(pool, id).await.ok().and_then(|r| r.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("role"), + target_id: Some(id), + event: "role.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} +/// [`restore_role`]'s body, minus opening/committing its own transaction — +/// see [`create_role_assignment_in_tx`]'s doc comment for the caller +/// contract (the resolver locks the role, and every group currently +/// assigned it, via [`lock_role_and_collect_grants_keys`] on this same `tx` +/// first). +pub(crate) async fn restore_role_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result<(), AppError> { + let _ = restored_by; let tenant_info: Option<(Option, bool)> = sqlx::query_as( "SELECT r.tenant_id, (t.deleted_at IS NOT NULL) FROM roles r @@ -2855,7 +2933,7 @@ pub async fn restore_role_with_audit( WHERE r.id = $1 AND r.deleted_at IS NOT NULL", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let (tenant_id, _is_tenant_deleted) = match tenant_info { @@ -2877,20 +2955,18 @@ pub async fn restore_role_with_audit( WHERE id = $1 AND deleted_at IS NOT NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(restore_conflict)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("role"), + target_kind: "role", target_id: Some(id), event: "role.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; Ok(()) } @@ -3993,8 +4069,32 @@ pub async fn create_role_assignment_with_audit( req: CreateRoleAssignment, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; - lock_live_subject(&mut tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; - lock_role(&mut tx, req.role_id).await?; + let assignment = + create_role_assignment_in_tx(pool, &mut tx, events_enabled, actor_id, req).await?; + tx.commit().await.map_err(db_err)?; + Ok(assignment) +} + +/// [`create_role_assignment`]'s body, minus opening/committing its own +/// transaction. For a group subject, the caller (the group-subject mutation +/// resolver path) must have already run +/// [`lock_group_closures_and_collect_grants_keys`] on this same `tx` and +/// called `cache.begin()` on the result before calling this — `lock_role` +/// and `lock_live_subject` below then just re-acquire (safe, same-transaction +/// no-op) locks this function has always taken. The caller commits `tx`, not +/// this function. +pub(crate) async fn create_role_assignment_in_tx( + pool: &PgPool, + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + req: CreateRoleAssignment, +) -> Result { + lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; + // Lock the role and validate under the lock so a concurrent block-link + // mutation cannot add a prohibited block against stale state: it blocks on + // this same lock and re-validates against the assignment we are inserting. + lock_role(tx, req.role_id).await?; validate_role_assignment(pool, &req).await?; let assignment = sqlx::query_as::<_, RoleAssignment>( r#"INSERT INTO role_assignments @@ -4006,7 +4106,7 @@ pub async fn create_role_assignment_with_audit( .bind(req.subject_kind) .bind(req.subject_id) .bind(req.role_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -4018,7 +4118,7 @@ pub async fn create_role_assignment_with_audit( event: "role_assignment.create", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(assignment) } @@ -4029,50 +4129,6 @@ pub async fn create_role_assignment( create_role_assignment_with_audit(pool, false, None, req).await } -pub async fn delete_role_assignment_with_audit( - pool: &PgPool, - events_enabled: bool, - actor_id: Option, - id: Uuid, -) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; - let assignment_tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM role_assignments WHERE id = $1") - .bind(id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - let Some(tenant_id) = assignment_tenant_id else { - return Err(AppError::not_found(format!( - "role assignment {id} not found" - ))); - }; - let result = sqlx::query("DELETE FROM role_assignments WHERE id = $1") - .bind(id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - if result.rows_affected() == 0 { - return Err(AppError::not_found(format!( - "role assignment {id} not found" - ))); - } - let meta = crate::audit::AuditMeta { - actor_entity_id: actor_id, - tenant_id, - target_kind: "role_assignment", - target_id: Some(id), - event: "role_assignment.delete", - }; - let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; - Ok(()) -} - -pub async fn delete_role_assignment(pool: &PgPool, id: Uuid) -> Result<(), AppError> { - delete_role_assignment_with_audit(pool, false, None, id).await -} - /// Returns `true` when a new assignment row was actually inserted, so callers /// can tell a real state change from an idempotent no-op and decide whether the /// operation is worth publishing as a domain event. @@ -4185,20 +4241,96 @@ pub async fn get_role_assignment(pool: &PgPool, id: Uuid) -> Result Result<(), AppError> { + delete_role_assignment_with_audit(pool, false, None, id).await +} + +pub async fn delete_role_assignment_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + delete_role_assignment_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err) +} + +/// [`delete_role_assignment`]'s body, minus opening/committing its own +/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the +/// caller contract (group-subject resolver path locks the subject's group +/// closure on this same `tx` first). +pub(crate) async fn delete_role_assignment_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM role_assignments WHERE id = $1") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let Some(tenant_id) = tenant_id else { + return Err(AppError::not_found(format!( + "role assignment {id} not found" + ))); + }; + // A role assignment is a 'policy' protected object; the policy-object cleanup trigger + // sweeps the permission blocks targeting it when this row is deleted. + let result = sqlx::query("DELETE FROM role_assignments WHERE id = $1") + .bind(id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + if result.rows_affected() == 0 { + return Err(AppError::not_found(format!( + "role assignment {id} not found" + ))); + } + let meta = crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }; + let details = serde_json::json!({}); + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(()) +} + pub async fn create_direct_policy_with_audit( pool: &PgPool, events_enabled: bool, actor_id: Option, req: CreateDirectPolicy, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let policy = + create_direct_policy_in_tx(pool, &mut tx, events_enabled, actor_id, req).await?; + tx.commit().await.map_err(db_err)?; + Ok(policy) +} + +/// [`create_direct_policy`]'s body, minus opening/committing its own +/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the +/// caller contract. +pub(crate) async fn create_direct_policy_in_tx( + pool: &PgPool, + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + req: CreateDirectPolicy, ) -> Result { validate_direct_policy(pool, &req).await?; crate::guardrails::validate_direct_policy(pool, &req).await?; - let mut tx = pool.begin().await.map_err(db_err)?; - lock_live_subject(&mut tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; + lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; let block_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM permission_blocks WHERE id = $1 FOR UPDATE") .bind(req.permission_block_id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; if block_tenant_id != Some(req.tenant_id) { @@ -4216,7 +4348,7 @@ pub async fn create_direct_policy_with_audit( .bind(req.subject_kind) .bind(req.subject_id) .bind(req.permission_block_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -4228,7 +4360,7 @@ pub async fn create_direct_policy_with_audit( event: "direct_policy.create", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(policy) } @@ -4314,10 +4446,23 @@ pub async fn delete_direct_policy_with_audit( id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; + delete_direct_policy_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err) +} + +/// [`delete_direct_policy`]'s body, minus opening/committing its own +/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the +/// caller contract. +pub(crate) async fn delete_direct_policy_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { let policy_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM direct_policies WHERE id = $1") .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(tenant_id) = policy_tenant_id else { @@ -4327,13 +4472,16 @@ pub async fn delete_direct_policy_with_audit( "DELETE FROM direct_policies WHERE id = $1 RETURNING permission_block_id", ) .bind(id) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await .map_err(db_err)?; let Some(block_id) = block_id else { return Err(AppError::not_found(format!("direct policy {id} not found"))); }; - delete_orphaned_blocks(&mut tx, &[block_id]).await?; + // The block is shared: GC it only if removing this policy left it + // unreferenced (mirrors delete_policy). Blocks targeting this policy *as an + // object* are swept by the policy-object cleanup trigger on the delete above. + delete_orphaned_blocks(tx, &[block_id]).await?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, @@ -4343,7 +4491,7 @@ pub async fn delete_direct_policy_with_audit( event: "direct_policy.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(()) } @@ -5795,6 +5943,186 @@ pub async fn effective_grants_for_subject( .collect() } +/// Locks every group in the closure rooted at `root_group_ids` (each root +/// plus every descendant subgroup, via a `group_hierarchy` descent) `FOR +/// UPDATE` in `principal_groups` — in a stable, id-sorted order, so this can +/// never deadlock against another +/// call to this same function locking an overlapping closure — then returns +/// the member entity ids across that (now-locked) closure. +/// +/// # Why this exists +/// +/// A plain "enumerate current members, then invalidate" has a real race: a +/// group-subject mutation (direct policy / role assignment / role +/// block-links / group status-or-hierarchy change) can enumerate members at +/// one instant, while `identity::repo::add_group_member` concurrently +/// commits a *new* member in between that enumeration and the mutation's own +/// commit. That new member's `grants` key is never in the enumerated set, so +/// it never gets invalidated — a stale cached grant (or a stale cached +/// *lack* of one) can then survive until the grants TTL. (Reported by +/// external review, 2026-07-29.) +/// +/// `add_group_member`/`remove_group_member` already lock the group's row in +/// `principal_groups` `FOR UPDATE` before changing membership. Locking that +/// same row here — and holding it for the caller's entire transaction, +/// through commit — closes the race: neither side's transaction can commit +/// while the other holds the lock, so whichever runs first is fully visible +/// (including its own cache invalidation) before the other's enumeration or +/// commit proceeds. +/// +/// # What this does *not* cover +/// +/// The closure itself (`root_group_ids` plus descendants) is computed with +/// one unlocked read before any row is locked, so a concurrent *reparent* +/// that brings a brand-new subgroup into range between that read and the +/// lock isn't covered by this function — an accepted, documented residual +/// gap (bounded by the grants TTL, exactly as every enumeration-based +/// invalidation was before this fix). Closing that would mean locking +/// hierarchy structure itself for every membership mutation everywhere, a +/// far larger scope than the membership-add race this closes. +/// +/// # Contract for callers +/// +/// The returned ids are only exhaustive as long as the lock stays held: the +/// caller must keep `tx` open (no commit) from this call through the end of +/// its own mutation's commit, and should call `cache.begin()` on the +/// resulting keys *before* that commit — mirroring `guarded_mutation`'s +/// begin/mutate/end shape, just with `mutate` running against this +/// already-open, already-locked `tx` instead of opening its own. +pub async fn lock_group_closures_and_collect_member_ids( + tx: &mut Transaction<'_, Postgres>, + root_group_ids: &[Uuid], +) -> Result, AppError> { + if root_group_ids.is_empty() { + return Ok(Vec::new()); + } + let mut closure: Vec = sqlx::query_scalar( + r#"WITH RECURSIVE target_groups(id) AS ( + SELECT id FROM UNNEST($1::uuid[]) AS root(id) + UNION + SELECT gh.child_id + FROM group_hierarchy gh + JOIN target_groups tg ON tg.id = gh.parent_id + ) + SELECT id FROM target_groups"#, + ) + .bind(root_group_ids) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + closure.sort_unstable(); + closure.dedup(); + + // Locks whichever of these ids are principal groups (object groups have + // no members and simply won't match here — harmless, not an error). + sqlx::query("SELECT id FROM principal_groups WHERE id = ANY($1) ORDER BY id FOR UPDATE") + .bind(&closure) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + sqlx::query_scalar("SELECT DISTINCT entity_id FROM group_members WHERE group_id = ANY($1)") + .bind(&closure) + .fetch_all(&mut **tx) + .await + .map_err(db_err) +} + +/// [`lock_group_closures_and_collect_member_ids`], mapped straight to +/// `atom:v1:grants:*` cache keys — the form every locked group-subject +/// mutation call site actually wants. +pub async fn lock_group_closures_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + root_group_ids: &[Uuid], +) -> Result, AppError> { + Ok( + lock_group_closures_and_collect_member_ids(tx, root_group_ids) + .await? + .into_iter() + .map(crate::cache::keys::grants) + .collect(), + ) +} + +/// Like [`lock_group_closures_and_collect_grants_keys`], for a role: locks +/// the role row itself plus every group in the closure of every group +/// directly assigned this role, and returns the combined set of affected +/// `atom:v1:grants:*` keys (entity-direct assignees plus every locked +/// group's members). Entity-direct assignees need no lock of their own — +/// the affected key is exactly their `subject_id`, deterministic and +/// race-free. +/// +/// Locks the role row directly (`SELECT ... FOR UPDATE`, no `deleted_at` +/// filter) rather than going through the private, alive-only `lock_role` — +/// `restore_role` is a caller here and operates on an *already* +/// soft-deleted role, so requiring "alive" would always fail for it. Each +/// caller's own mutation separately validates/enforces whatever alive-ness +/// it needs; this only needs the lock (for closure consistency) and the +/// current assignee set (which was never conditioned on the role's own +/// status either, before this fix). +pub async fn lock_role_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + role_id: Uuid, +) -> Result, AppError> { + sqlx::query_scalar::<_, Uuid>("SELECT id FROM roles WHERE id = $1 FOR UPDATE") + .bind(role_id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + let entity_subject_ids: Vec = sqlx::query_scalar( + "SELECT subject_id FROM role_assignments WHERE role_id = $1 AND subject_kind = 'entity'", + ) + .bind(role_id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + let group_subject_ids: Vec = sqlx::query_scalar( + "SELECT subject_id FROM role_assignments WHERE role_id = $1 AND subject_kind = 'group'", + ) + .bind(role_id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + let mut member_ids = lock_group_closures_and_collect_member_ids(tx, &group_subject_ids).await?; + member_ids.extend(entity_subject_ids); + member_ids.sort_unstable(); + member_ids.dedup(); + Ok(member_ids + .into_iter() + .map(crate::cache::keys::grants) + .collect()) +} + +/// Locks a role and a group (in that order) for the one mutation that needs +/// both in the same transaction: creating a role assignment for a group +/// subject. **Lock order matters here**: [`lock_role_and_collect_grants_keys`] +/// (used by `replaceRolePermissionBlocks`/`deleteRole`/`restoreRole`) always +/// locks the role first, then the closures of every group assigned it. If +/// `createRoleAssignment` locked the *subject group* first and only reached +/// the role lock afterward (inside `create_role_assignment_in_tx`'s own +/// `lock_role` call — which is exactly what the original code did), a +/// concurrent pair of requests could deadlock: one holding the descendant +/// group's row while waiting on the role, the other holding the role while +/// waiting on that same descendant group (reached via an ancestor group +/// already assigned the role). Postgres detects the cycle and aborts one +/// side. (Reported by external review, 2026-07-29.) +/// +/// Locking the role first here — before the group closure — makes +/// `createRoleAssignment`'s lock order match every role-mutation path +/// exactly, closing the inversion. `create_role_assignment_in_tx`'s own +/// `lock_role`/`lock_live_subject` calls afterward re-acquire the same locks +/// (a same-transaction no-op) — kept as-is since that function is also used, +/// unlocked, by the entity-subject path, which has no group lock to order +/// against. +pub async fn lock_role_then_group_closure_and_collect_grants_keys( + tx: &mut Transaction<'_, Postgres>, + role_id: Uuid, + subject_group_id: Uuid, +) -> Result, AppError> { + lock_role(tx, role_id).await?; + lock_group_closures_and_collect_grants_keys(tx, &[subject_group_id]).await +} + pub async fn find_capability_ids_by_name( pool: &PgPool, name: &str, diff --git a/src/graphql/groups.rs b/src/graphql/groups.rs index 2c88b62..db42ac1 100644 --- a/src/graphql/groups.rs +++ b/src/graphql/groups.rs @@ -378,24 +378,78 @@ impl GroupMutation { event: "group.update", }; let details = serde_json::json!({}); + let status: Option = input.status.map(Into::into); + let status_changing = status.is_some(); let result = async { let existing = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, existing.tenant_id).await?; - repo::update_group_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - UpdateGroup { - name: input.name, - description: input.description, - status: input.status.map(Into::into), - attributes: input.attributes, - }, - "group.update", - details.clone(), - ) - .await + let update = UpdateGroup { + name: input.name, + description: input.description, + status, + attributes: input.attributes, + }; + // `subject_effective_grants`'s `subject_groups` CTE requires + // `status = 'active'` at every hop, so any status change (not + // just a move off "active") changes the recursive grant set for + // every member of this group's subtree. + if status_changing { + // Locked (not just enumerated) — see + // `authz::repo::lock_group_closures_and_collect_member_ids` + // for why a concurrent `add_group_member` needs this to be + // safe, not just a plain cache barrier. + let Some(cache) = state.cache.as_deref() else { + return repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + "group.update", + details.clone(), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::update_group_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + "group.update", + details.clone(), + ) + .await; + let outcome = match outcome { + Ok(value) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| value), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome + } else { + repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + "group.update", + details.clone(), + ) + .await + } } .await; if let Err(ref err) = result { @@ -434,14 +488,45 @@ impl GroupMutation { let result = async { let group = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, group.tenant_id).await?; - repo::set_group_parent_with_audit( - &state.pool, + // Reparenting `id` only changes what `id` and its descendants + // inherit from above — it never changes `group_hierarchy` rows + // below `id`, so `id`'s subtree is exactly what this mutation + // can affect. Locked (not just enumerated) — see + // `authz::repo::lock_group_closures_and_collect_member_ids` for + // why a concurrent `add_group_member` needs this to be safe. + let Some(cache) = state.cache.as_deref() else { + return repo::set_group_parent_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + parent_id, + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::set_group_parent_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, parent_id, ) - .await + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; + repo::get_group(&state.pool, id).await } .await; if let Err(ref err) = result { @@ -483,13 +568,38 @@ impl GroupMutation { let group = repo::get_group(&state.pool, id).await?; let tenant_id = group.tenant_id; require_group_manage_app(&state.pool, &auth, id, tenant_id).await?; - repo::remove_group_parent_with_audit( - &state.pool, + // Locked, not just enumerated — see `set_group_parent` above. + let Some(cache) = state.cache.as_deref() else { + repo::remove_group_parent_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::remove_group_parent_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, ) - .await?; + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; Ok(tenant_id) } .await; @@ -538,14 +648,42 @@ impl GroupMutation { let existing = repo::get_group(&state.pool, id).await?; let tenant_id = existing.tenant_id; require_group_manage_app(&state.pool, &auth, id, tenant_id).await?; - repo::delete_group_with_audit( - &state.pool, + // `group_hierarchy` rows aren't touched by a soft delete (only + // `deleted_at` is set), so enumeration is unaffected by timing. + // Locked, not just enumerated — see `set_group_parent` above. + let Some(cache) = state.cache.as_deref() else { + repo::delete_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::delete_group_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, Some(auth.entity_id), ) - .await?; + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; Ok(tenant_id) } .await; @@ -580,14 +718,65 @@ impl GroupMutation { let result = async { crate::auth::require_any_capability(&state.pool, &auth, &[("manage", Scope::Platform)]) .await?; - repo::restore_group_with_audit( - &state.pool, + // Locked, not just enumerated — see `set_group_parent` above (the + // lock works regardless of the group's own `deleted_at` status, + // so it applies here unchanged). + let Some(cache) = state.cache.as_deref() else { + return repo::restore_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::restore_group_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, Some(auth.entity_id), ) - .await + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; + // Mirrors `restore_group_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_group_with_audit` itself. + let tenant_id = repo::get_group(&state.pool, id) + .await + .ok() + .and_then(|g| g.tenant_id); + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("group"), + target_id: Some(id), + event: "group.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) } .await; if let Err(ref err) = result { @@ -664,12 +853,19 @@ impl GroupMutation { ], ) .await?; - repo::add_group_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - group_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + repo::add_group_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + group_id, + entity_id, + ) + }, ) .await?; Ok(tenant_id) @@ -718,12 +914,19 @@ impl GroupMutation { ], ) .await?; - repo::remove_group_member_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - group_id, - entity_id, + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || { + repo::remove_group_member_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + group_id, + entity_id, + ) + }, ) .await?; Ok(tenant_id) @@ -774,21 +977,53 @@ impl GroupMutation { let result = async { let group = repo::get_group(&state.pool, id).await?; require_group_manage_app(&state.pool, &auth, id, group.tenant_id).await?; - repo::update_group_with_audit( - &state.pool, + let update = UpdateGroup { + name: None, + description: None, + status: Some(status), + attributes: None, + }; + // Locked, not just enumerated — see `update_group` above. + let Some(cache) = state.cache.as_deref() else { + return repo::update_group_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + update, + event, + details.clone(), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = repo::update_group_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, - UpdateGroup { - name: None, - description: None, - status: Some(status), - attributes: None, - }, + update, event, details.clone(), ) - .await + .await; + let outcome = match outcome { + Ok(value) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| value), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome } .await; if let Err(ref err) = result { diff --git a/src/graphql/policies.rs b/src/graphql/policies.rs index 310aaa5..3384a53 100644 --- a/src/graphql/policies.rs +++ b/src/graphql/policies.rs @@ -442,14 +442,45 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::replace_role_permission_block_links_with_audit( + // The role's assignees (entity and group) are locked and + // enumerated together as one step — see + // `authz::repo::lock_role_and_collect_grants_keys` — so a + // concurrent `add_group_member` on one of its assigned groups + // can't commit a membership change this misses. + let Some(cache) = state.cache.as_deref() else { + authz_repo::replace_role_permission_block_links_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + role_id, + &permission_block_ids, + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = + authz_repo::lock_role_and_collect_grants_keys(&mut tx, role_id).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::replace_role_permission_block_links_in_tx( &state.pool, + &mut tx, state.config.events.enabled(), Some(auth.entity_id), role_id, &permission_block_ids, ) - .await?; + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; Ok(tenant_id) } .await; @@ -494,14 +525,40 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_role_with_audit( - &state.pool, + // See `replace_role_permission_blocks` above for why the role + // and its assignees are locked (not just enumerated) together. + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_role_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = authz_repo::lock_role_and_collect_grants_keys(&mut tx, id).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::delete_role_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, Some(auth.entity_id), ) - .await?; + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; Ok(tenant_id) } .await; @@ -536,14 +593,63 @@ impl PolicyMutation { let details = serde_json::json!({}); let result = async { require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - authz_repo::restore_role_with_audit( - &state.pool, + // See `replace_role_permission_blocks` above for why the role + // and its assignees are locked (not just enumerated) together. + let Some(cache) = state.cache.as_deref() else { + return authz_repo::restore_role_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = authz_repo::lock_role_and_collect_grants_keys(&mut tx, id).await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::restore_role_in_tx( + &mut tx, state.config.events.enabled(), Some(auth.entity_id), id, Some(auth.entity_id), ) - .await + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; + // Mirrors `restore_role_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_role_with_audit` itself. + let tenant_id = authz_repo::get_role(&state.pool, id) + .await + .ok() + .and_then(|r| r.tenant_id); + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: Some("role"), + target_id: Some(id), + event: "role.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) } .await; if let Err(ref err) = result { @@ -1015,18 +1121,78 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::create_role_assignment_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - CreateRoleAssignment { - tenant_id, - subject_kind, - subject_id, - role_id, - }, - ) - .await + let req = CreateRoleAssignment { + tenant_id, + subject_kind: subject_kind.clone(), + subject_id, + role_id, + }; + match subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::create_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + }, + ) + .await + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + return authz_repo::create_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + // Locks the role *before* `subject_id`'s group closure — + // matching `replaceRolePermissionBlocks`/`deleteRole`/ + // `restoreRole`'s lock order exactly, since locking the + // group first here (with the role locked later, inside + // `create_role_assignment_in_tx`) can deadlock against + // those paths. See + // `authz::repo::lock_role_then_group_closure_and_collect_grants_keys`. + let grants_keys = + authz_repo::lock_role_then_group_closure_and_collect_grants_keys( + &mut tx, role_id, subject_id, + ) + .await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::create_role_assignment_in_tx( + &state.pool, + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + let outcome = match outcome { + Ok(value) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| value), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome + } + } } .await; if let Err(ref err) = result { @@ -1064,13 +1230,61 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_role_assignment_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await?; + match assignment.subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(assignment.subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::delete_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + }, + ) + .await?; + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_role_assignment_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( + &mut tx, + &[assignment.subject_id], + ) + .await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::delete_role_assignment_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; + } + } Ok(tenant_id) } .await; @@ -1114,18 +1328,71 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::create_direct_policy_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - CreateDirectPolicy { - tenant_id, - subject_kind, - subject_id, - permission_block_id, - }, - ) - .await + let req = CreateDirectPolicy { + tenant_id, + subject_kind: subject_kind.clone(), + subject_id, + permission_block_id, + }; + match subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::create_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + }, + ) + .await + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + return authz_repo::create_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( + &mut tx, + &[subject_id], + ) + .await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::create_direct_policy_in_tx( + &state.pool, + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + req, + ) + .await; + let outcome = match outcome { + Ok(value) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| value), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome + } + } } .await; if let Err(ref err) = result { @@ -1163,13 +1430,61 @@ impl PolicyMutation { scope_for_tenant(tenant_id), ) .await?; - authz_repo::delete_direct_policy_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await?; + match policy.subject_kind { + crate::models::enums::SubjectKind::Entity => { + let grants_keys = vec![crate::cache::keys::grants(policy.subject_id)]; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + &grants_keys, + || { + authz_repo::delete_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + }, + ) + .await?; + } + crate::models::enums::SubjectKind::Group => { + let Some(cache) = state.cache.as_deref() else { + authz_repo::delete_direct_policy_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await?; + return Ok(tenant_id); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( + &mut tx, + &[policy.subject_id], + ) + .await?; + cache + .begin(crate::cache::CacheCategory::Grants, &grants_keys) + .await?; + let outcome = authz_repo::delete_direct_policy_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + cache + .end(crate::cache::CacheCategory::Grants, &grants_keys) + .await; + outcome?; + } + } Ok(tenant_id) } .await; diff --git a/src/identity/repo.rs b/src/identity/repo.rs index b949fe3..e01f5ed 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -1271,7 +1271,7 @@ pub(crate) async fn update_group_in_tx( return Err(AppError::not_found(format!("group {id} not found"))); }; crate::tenants::repo::lock_optional_active_tenant(tx, tenant_id).await?; - sqlx::query_as::<_, Group>( + let group = sqlx::query_as::<_, Group>( r#"WITH p AS ( UPDATE principal_groups SET name = COALESCE($2, name), diff --git a/tests/m12_graphql_identity.rs b/tests/m12_graphql_identity.rs index ea5d835..ccbb6fc 100644 --- a/tests/m12_graphql_identity.rs +++ b/tests/m12_graphql_identity.rs @@ -82,6 +82,7 @@ fn authed_scoped_with_credential( credential_id: Some(credential_id), scoped: true, ceiling: Some(std::sync::Arc::new(ceiling)), + cache: None, }) } diff --git a/tests/m26_audit_event_publishing.rs b/tests/m26_audit_event_publishing.rs index 1af2827..799521b 100644 --- a/tests/m26_audit_event_publishing.rs +++ b/tests/m26_audit_event_publishing.rs @@ -50,7 +50,7 @@ async fn build_state(pool: PgPool, config: Config) -> AppState { let active_keys = keys::load_active_keys(&pool, &config.signing_keys) .await .expect("load signing keys"); - AppState::new(pool, config, active_keys, None) + AppState::new(pool, config, active_keys, None, None) } fn authed(query: impl Into) -> Request { From 4a0666a87552b72f011e42c1ea3876fd2eb37104 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 15:14:56 +0300 Subject: [PATCH 04/19] Add cache invalidation test suite Covers direct policies, role assignments, group membership/hierarchy, sessions, credentials, tenant/entity soft-delete and restore, API-key tenant context after an entity moves tenants, and the group/role lock ordering - each warms the cache, mutates through the real GraphQL path, and asserts the effect is visible on the very next read with no wait, never relying on a short TTL happening to expire. --- tests/m25_cache_invalidation.rs | 1425 +++++++++++++++++++++++++++++++ 1 file changed, 1425 insertions(+) create mode 100644 tests/m25_cache_invalidation.rs diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs new file mode 100644 index 0000000..30c6fc4 --- /dev/null +++ b/tests/m25_cache_invalidation.rs @@ -0,0 +1,1425 @@ +//! Cache invalidation correctness tests. +//! +//! Requires both a reachable Postgres at `DATABASE_URL` and a reachable Redis +//! at `ATOM_TEST_REDIS_URL`. Run with: +//! +//! ```bash +//! DATABASE_URL=postgres://... ATOM_TEST_REDIS_URL=redis://... cargo test --test m25_cache_invalidation -- --ignored +//! ``` +//! +//! Every test here follows the same shape: warm the cache with a read, mutate +//! through the real (wired) production path — a GraphQL mutation executed +//! through the schema, exactly as a client would call it — then immediately +//! (no sleep) re-check that the mutation's effect is visible. TTL is a +//! defense-in-depth safety net in this design, never the mechanism under +//! test; a passing test here means invalidation is precise, not merely that +//! a short enough TTL happened to expire. + +mod common; + +use std::sync::Arc; + +use async_graphql::Request; +use atom::{ + auth::{self, AuthContext}, + authz::{engine, repo as authz_repo}, + cache::CacheClient, + config::Config, + graphql::build_schema, + identity::{access_tokens, repo as identity_repo}, + models::{ + enums::{Effect, SubjectKind}, + group::CreateGroup, + policy::{AuthzRequest, CreateDirectPolicy, CreatePermissionBlock, CreateRoleAssignment}, + role::CreateRole, + token::CreateAccessToken, + }, + state::AppState, +}; +use common::{cache_client, pool}; +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +/// Builds an `AppState` wired to a fresh Redis-backed cache, plus an +/// `Arc` handle to the *same* instance (mirroring how +/// production shares one cache between `state.cache` and `AuthContext:: +/// cache` — see `auth_from_jwt`/`auth_from_api_key`'s `cache: +/// state.cache.clone()`). Uses a real rotated EC signing key (not an empty +/// placeholder) since several tests here actually encode/verify JWTs. +async fn state_with_cache(pool: PgPool) -> (AppState, Arc) { + let cfg = Config::for_tests(); + let active_keys = atom::keys::rotate(&pool, &cfg.signing_keys) + .await + .expect("rotate test signing key"); + let state = AppState::new(pool, cfg, active_keys, None, Some(cache_client().await)); + let cache = state.cache.clone().expect("cache configured"); + (state, cache) +} + +fn auth_context(entity_id: Uuid, cache: Arc) -> AuthContext { + AuthContext { + entity_id, + tenant_id: None, + session_id: None, + credential_id: None, + scoped: false, + ceiling: None, + cache: Some(cache), + } +} + +fn authed(entity_id: Uuid, cache: Arc, query: impl Into) -> Request { + Request::new(query).data(auth_context(entity_id, cache)) +} + +async fn active_entity(pool: &PgPool, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO entities (id, kind, name, status) VALUES ($1, $2, $3, 'active')") + .bind(id) + .bind(kind) + .bind(format!("cache-test-{kind}-{id}")) + .execute(pool) + .await + .expect("insert entity"); + id +} + +async fn resource(pool: &PgPool, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO resources (id, kind, name) VALUES ($1, $2, $3)") + .bind(id) + .bind(kind) + .bind(format!("cache-test-res-{id}")) + .execute(pool) + .await + .expect("insert resource"); + id +} + +async fn read_action_id(pool: &PgPool) -> Uuid { + sqlx::query_scalar("SELECT id FROM actions WHERE name = 'read' LIMIT 1") + .fetch_one(pool) + .await + .expect("read action") +} + +/// A platform-scope allow block granting `read`, ready to attach via a +/// direct policy or a role. +async fn make_read_block(pool: &PgPool) -> Uuid { + let action_id = read_action_id(pool).await; + authz_repo::create_permission_block( + pool, + CreatePermissionBlock { + tenant_id: None, + scope_mode: "platform".into(), + object_kind: None, + object_type: None, + object_id: None, + group_id: None, + effect: Effect::Allow, + conditions: json!({}), + action_ids: vec![action_id], + }, + ) + .await + .expect("create permission block") + .id +} + +async fn new_group(pool: &PgPool, label: &str) -> Uuid { + identity_repo::create_group( + pool, + CreateGroup { + id: None, + name: format!("cache-test-{label}-{}", Uuid::new_v4()), + tenant_id: None, + group_type: Some("principal".into()), + description: None, + attributes: json!({}), + }, + ) + .await + .expect("create group") + .id +} + +/// A standalone Redis connection for tests that need to inspect cache state +/// directly (e.g. asserting a key was *not* touched), independent of +/// `CacheClient`'s own (private) connection pool. +async fn raw_redis_conn() -> redis::aio::MultiplexedConnection { + let url = std::env::var("ATOM_TEST_REDIS_URL") + .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); + redis::Client::open(url) + .expect("valid redis url") + .get_multiplexed_async_connection() + .await + .expect("connect to test redis") +} + +async fn evaluate_read( + pool: &PgPool, + subject_id: Uuid, + resource_id: Uuid, + cache: Arc, +) -> bool { + let req = AuthzRequest { + subject_id, + action: "read".into(), + resource_id: Some(resource_id), + object_kind: None, + object_id: None, + context: json!({}), + }; + let auth = auth_context(subject_id, cache); + engine::evaluate(pool, &req, &auth) + .await + .expect("evaluate") + .allowed +} + +// ─── Direct policy ────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn direct_policy_revoke_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let subject = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let policy = authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: subject, + permission_block_id: block_id, + }, + ) + .await + .expect("create direct policy"); + + assert!( + evaluate_read(&p, subject, res, cache.clone()).await, + "direct policy should grant read" + ); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteDirectPolicy(id: "{}") }}"#, policy.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, subject, res, cache.clone()).await, + "revoked direct policy must deny immediately, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn direct_policy_revoke_for_group_subject_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let group = new_group(&p, "direct-policy-group").await; + identity_repo::add_group_member(&p, group, member) + .await + .expect("add member"); + + let policy = authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: group, + permission_block_id: block_id, + }, + ) + .await + .expect("create group direct policy"); + + assert!(evaluate_read(&p, member, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteDirectPolicy(id: "{}") }}"#, policy.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "group-subject revoke must immediately deny the group's member" + ); +} + +// ─── Role assignment / role delete ───────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn role_assignment_revoke_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let subject = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + + let assignment = authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: subject, + role_id: role.id, + }, + ) + .await + .expect("create role assignment"); + + assert!(evaluate_read(&p, subject, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ deleteRoleAssignment(id: "{}") }}"#, + assignment.id + ), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, subject, res, cache.clone()).await, + "revoked role assignment must deny immediately" + ); +} + +/// The fan-out case: a role assigned to a group three levels of nesting away +/// from the actual member must invalidate that member's cached grants too. +#[tokio::test] +#[ignore] +async fn role_delete_invalidates_three_level_nested_group_members() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + // grandparent -> parent -> child, member belongs to `child`, role is + // assigned to `grandparent`. + let grandparent = new_group(&p, "gp").await; + let parent = new_group(&p, "parent").await; + let child = new_group(&p, "child").await; + + identity_repo::set_group_parent(&p, parent, grandparent) + .await + .expect("parent under grandparent"); + identity_repo::set_group_parent(&p, child, parent) + .await + .expect("child under parent"); + identity_repo::add_group_member(&p, child, member) + .await + .expect("add member to child"); + + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-nested-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: grandparent, + role_id: role.id, + }, + ) + .await + .expect("assign role to grandparent"); + + assert!( + evaluate_read(&p, member, res, cache.clone()).await, + "member three levels deep should inherit the grandparent's role grant" + ); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteRole(id: "{}") }}"#, role.id), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "deleting the role must immediately deny the deeply-nested member, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn group_membership_removal_is_immediately_reflected() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + let res = resource(&p, "channel").await; + let block_id = make_read_block(&p).await; + + let group = new_group(&p, "membership-remove").await; + identity_repo::add_group_member(&p, group, member) + .await + .expect("add member"); + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("cache-test-membership-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + authz_repo::replace_role_permission_block_links(&p, role.id, &[block_id]) + .await + .expect("link block"); + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: group, + role_id: role.id, + }, + ) + .await + .expect("assign role to group"); + + assert!(evaluate_read(&p, member, res, cache.clone()).await); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ removeGroupMember(groupId: "{group}", entityId: "{member}") }}"# + ), + )) + .await; + assert!(resp.errors.is_empty(), "remove failed: {:?}", resp.errors); + + assert!( + !evaluate_read(&p, member, res, cache.clone()).await, + "removing group membership must immediately deny the role grant it carried" + ); +} + +// ─── Session / JWT ────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn session_revoke_immediately_rejects_the_next_authentication() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + // `logout` reads `auth.session_id` from the request context — unlike + // `authed()`'s default (`session_id: None`, fine for every other test + // here since they don't exercise session-bearing mutations), this must + // carry the real session_id or there is nothing for `logout` to revoke. + let logout_auth = AuthContext { + session_id: Some(session_id), + ..auth_context(entity_id, cache.clone()) + }; + let schema = build_schema(state.clone()); + let resp = schema + .execute(Request::new("mutation { logout }").data(logout_auth)) + .await; + assert!(resp.errors.is_empty(), "logout failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "revoked session must be rejected on the very next authentication, not after a TTL" + ); +} + +#[tokio::test] +#[ignore] +async fn entity_deactivation_immediately_rejects_an_existing_valid_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache for this session. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + // Deactivate through the real, wired GraphQL path. + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "deactivating the entity must immediately reject its existing session, not after a TTL" + ); +} + +// ─── Credential ───────────────────────────────────────────────────────────── + +#[tokio::test] +#[ignore] +async fn credential_revoke_immediately_rejects_the_next_authentication() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + // Warm the credential cache with a successful authentication. + auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ revokeAccessToken(credentialId: "{}") }}"#, + minted.credential_id + ), + )) + .await; + assert!(resp.errors.is_empty(), "revoke failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &minted.token).await; + assert!( + result.is_err(), + "revoked credential must be rejected on the very next authentication, not after a TTL" + ); +} + +// ─── Tenant delete / restore ──────────────────────────────────────────────── + +async fn tenant(pool: &PgPool) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query("INSERT INTO tenants (id, name, status) VALUES ($1, $2, 'active')") + .bind(id) + .bind(format!("cache-test-tenant-{id}")) + .execute(pool) + .await + .expect("insert tenant"); + id +} + +async fn active_entity_in_tenant(pool: &PgPool, tenant_id: Uuid, kind: &str) -> Uuid { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO entities (id, kind, name, tenant_id, status) VALUES ($1, $2, $3, $4, 'active')", + ) + .bind(id) + .bind(kind) + .bind(format!("cache-test-{kind}-{id}")) + .bind(tenant_id) + .execute(pool) + .await + .expect("insert entity"); + id +} + +/// Regression test for a review finding: `soft_delete_tenant` (behind +/// `deleteTenant`) is a separate function from `change_tenant_status` and +/// was found to be completely unwired — a stale `tenant_status` cache entry +/// let an existing JWT session for a member of the deleted tenant keep +/// authenticating. Both the session cache *and* the tenant_status cache are +/// warmed stale-valid here before the delete, so this only passes if +/// `tenant_status` is genuinely invalidated (a stale-but-unrevoked session +/// cache entry alone would otherwise still let auth through). +#[tokio::test] +#[ignore] +async fn tenant_delete_immediately_rejects_an_existing_session_of_a_member() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status/tenant_status caches, all showing valid. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "delete failed: {:?}", resp.errors); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "deleting the tenant must immediately reject an existing member session, not after a TTL \ + (this only fails if tenant_status wasn't actually invalidated, since the session cache \ + entry alone is stale-valid and would otherwise let auth through)" + ); +} + +/// Regression test for a review finding: `restore_tenant` reactivates +/// credentials but the resolver originally only invalidated `tenant_status`, +/// not the credential entries it reactivates. +/// +/// Important nuance found while developing this test: an *end-to-end* +/// authentication test for this can't actually distinguish "the credential +/// invalidation ran" from "it didn't", because `restore_tenant`'s own (also +/// necessary) `tenant_status` invalidation forces `auth_from_api_key` through +/// its fresh-Postgres-reload branch regardless — that branch bypasses +/// whatever is in the credential cache entirely, masking the credential fix +/// either way. Both an "authenticate while deleted" attempt (to naturally +/// populate a stale entry) and a full auth-behavior assertion after restore +/// were tried and found to pass even with the credential invalidation +/// stripped out, for exactly this reason — confirmed by reverting the fix +/// and rerunning. So this test checks the one thing that actually isolates +/// it: that `restoreTenant` clears the credential's cache entry directly, +/// verified against Redis, independent of which auth code path would +/// subsequently be taken. +/// +/// The fix is kept as a matter of defensive correctness even though the +/// review's literal "cache captures revoked, stays stuck after restore" +/// scenario isn't reachable via the normal auth flow today (the tenant's own +/// `deleted_at` join filter means nothing can populate a credential's cache +/// entry while its tenant is genuinely deleted, so there is no live window +/// in which the entry could be wrong when restore runs) — relying on that +/// incidental protection instead of an explicit invalidation would be +/// fragile against future changes to either the join or the fast-path +/// gating in `auth_from_api_key`. +#[tokio::test] +#[ignore] +async fn tenant_restore_clears_reactivated_credential_cache_entries() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-tenant-restore-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let cred_key = atom::cache::keys::credential(minted.credential_id); + seed_hit( + &cred_key, + &atom::cache::entries::CredentialCacheEntry { + entity_id, + status: atom::models::enums::CredentialStatus::Revoked, + secret_hash: None, + secret_lookup_hash: Some(vec![0u8; 32]), + expires_at: None, + scoped: false, + }, + ) + .await; + let before = hmget_raw(&cred_key).await; + assert!( + before.2.is_some(), + "seeded entry should have a payload before restore: {before:?}" + ); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + let after = hmget_raw(&cred_key).await; + assert!( + after.2.is_none(), + "restoreTenant must clear the reactivated credential's cache payload \ + (begin() deletes it, end() never restores it) — got {after:?}, expected the \ + payload field to be absent" + ); + assert_ne!( + after.0, before.0, + "the credential key's version must have been bumped by restoreTenant's barrier" + ); +} + +/// Regression test for a review finding: `soft_delete_tenant` revokes +/// members' sessions in Postgres, but the delete resolver only invalidated +/// `tenant_status`, never the sessions themselves. During the deleted window +/// this is harmless — the `tenant_status` miss forces a fresh Postgres check, +/// which denies correctly — but once `restoreTenant` repopulates +/// `tenant_status` as active again, a session cached as valid *before* the +/// delete would resurface as a full cache hit and authenticate despite being +/// revoked, since `restoreTenant` deliberately never reinstates sessions. +/// +/// The first authentication attempt after restore always denies correctly +/// regardless of this fix, because restore's own `tenant_status` invalidation +/// forces that one request through a fresh Postgres reload (which, as a side +/// effect, also repopulates `tenant_status` as a hit). It's the *second* +/// attempt — once `tenant_status` is a hit again — that actually distinguishes +/// "the session was invalidated at delete time" from "it wasn't"; confirmed +/// by temporarily reverting the fix and rerunning. +#[tokio::test] +#[ignore] +async fn tenant_delete_then_restore_immediately_rejects_a_pre_existing_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status/tenant_status caches, all showing valid. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteTenant(id: "{tenant_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by tenant delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by tenant delete must stay rejected even once tenant_status is a \ + cache hit again post-restore — this only fails if the session cache entry wasn't \ + invalidated at delete time and survived as a stale 'valid' hit" + ); +} + +/// Regression test for a review finding: `delete_entity` revokes the +/// entity's sessions in Postgres, but the delete resolver only invalidated +/// `entity_status`. Same masking shape as the tenant case above: the first +/// post-restore attempt always denies correctly (entity_status's own +/// invalidation forces a fresh reload), and it's the second attempt that +/// actually proves the session cache entry itself was invalidated at delete +/// time, not left as a stale 'valid' hit for `restoreEntity` to unmask. +#[tokio::test] +#[ignore] +async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_session() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by entity delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &token).await.is_err(), + "a session revoked by entity delete must stay rejected even once entity_status is a \ + cache hit again post-restore — this only fails if the session cache entry wasn't \ + invalidated at delete time and survived as a stale 'valid' hit" + ); +} + +/// Same shape as the session test above, for the credential side: `delete_entity` +/// also revokes the entity's access-token credentials in Postgres, and +/// `restoreEntity` intentionally never reinstates them. Requires two +/// post-restore authentication attempts for the same reason as the session +/// test — the first always denies via a forced fresh reload; the second is +/// the one that actually proves the credential cache entry itself was +/// invalidated at delete time. +#[tokio::test] +#[ignore] +async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_access_token() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-entity-restore-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let del = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ deleteEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!(del.errors.is_empty(), "delete failed: {:?}", del.errors); + + let restore = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ restoreEntity(id: "{entity_id}") }}"#), + )) + .await; + assert!( + restore.errors.is_empty(), + "restore failed: {:?}", + restore.errors + ); + + assert!( + auth::authenticate_token(&state, &minted.token) + .await + .is_err(), + "an access token revoked by entity delete must stay rejected immediately after restore" + ); + assert!( + auth::authenticate_token(&state, &minted.token) + .await + .is_err(), + "an access token revoked by entity delete must stay rejected even once entity_status is \ + a cache hit again post-restore — this only fails if the credential cache entry wasn't \ + invalidated at delete time and survived as a stale 'active' hit" + ); +} + +// ─── API-key tenant context after an entity moves tenants ────────────────── + +/// Regression test for a review finding: the API-key fast path derived +/// `AuthContext.tenant_id` from the credential cache entry's own duplicated +/// `tenant_id` field, which only got invalidated when the credential itself +/// changed — not when the owning entity moved to a different tenant (only +/// `entity_status` is invalidated on that mutation). The fix removed the +/// duplicated field entirely and derives tenant context from the entity's +/// own (correctly invalidated) cache entry. +/// +/// As with the delete/restore tests above, the first authentication attempt +/// after the move always reflects the new tenant correctly regardless of the +/// fix, because the move's `entity_status` invalidation forces that one +/// request through a fresh Postgres reload. It's the *second* attempt — once +/// `entity_status` is a hit again — that actually distinguishes "tenant +/// context comes from the fresh entity entry" from "it comes from the stale +/// credential copy"; confirmed by temporarily reverting the fix and +/// rerunning. +#[tokio::test] +#[ignore] +async fn api_key_auth_reflects_the_current_tenant_after_an_entity_moves_tenants() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let old_tenant = tenant(&p).await; + let new_tenant = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, old_tenant, "service").await; + + let minted = access_tokens::create_access_token( + &p, + &state.config.signing_keys, + entity_id, + CreateAccessToken { + name: "cache-test-tenant-move-token".into(), + description: None, + expires_at: None, + permissions: vec![], + }, + false, + ) + .await + .expect("create access token"); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("initial authentication should succeed"); + assert_eq!(ctx.tenant_id, Some(old_tenant)); + + let schema = build_schema(state.clone()); + let mv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ updateEntity(id: "{entity_id}", input: {{ tenantId: "{new_tenant}" }}) {{ id }} }}"# + ), + )) + .await; + assert!(mv.errors.is_empty(), "tenant move failed: {:?}", mv.errors); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("authentication right after the move should still succeed"); + assert_eq!( + ctx.tenant_id, + Some(new_tenant), + "not the interesting assertion by itself" + ); + + let ctx = auth::authenticate_token(&state, &minted.token) + .await + .expect("authentication should still succeed once entity_status is a hit again"); + assert_eq!( + ctx.tenant_id, + Some(new_tenant), + "AuthContext.tenant_id must reflect the entity's current tenant, not a stale copy that \ + would have been cached in the credential entry from before the move" + ); +} + +// ─── Group-subject mutation vs. concurrent membership change ─────────────── + +/// Regression test for a review finding: resolving a group-subject +/// mutation's affected `grants` keys from "current members" alone has a +/// race against a concurrent `add_group_member` for the same group — a +/// member added in between the enumeration and the mutation's own commit +/// would never be in the enumerated set, so their `grants` key would never +/// be invalidated, letting a stale cached grant (or a stale cached lack of +/// one) survive until the grants TTL. +/// +/// Fixed by locking the group's row — the same lock +/// `identity::repo::add_group_member`/`remove_group_member` already take — +/// via `authz::repo::lock_group_closures_and_collect_member_ids`, for the +/// caller's entire transaction (enumeration through commit). This test +/// proves the lock actually serializes against a concurrent membership +/// change, mirroring `m8_guardrails.rs`'s +/// `concurrent_block_link_and_role_assignment_serialize` for the analogous +/// role-lock case. +#[tokio::test] +#[ignore] +async fn concurrent_group_membership_change_serializes_against_the_group_subject_lock() { + let p = pool().await; + let group = new_group(&p, "concurrent-lock").await; + let existing_member = active_entity(&p, "service").await; + identity_repo::add_group_member(&p, group, existing_member) + .await + .expect("seed existing member"); + + // Hold the lock a group-subject mutation resolver would (e.g. + // `deleteDirectPolicy`'s) — the exact enumeration step, kept open (not + // yet committed). + let mut tx = p.begin().await.expect("begin tx"); + let member_ids = authz_repo::lock_group_closures_and_collect_member_ids(&mut tx, &[group]) + .await + .expect("lock and enumerate"); + assert_eq!( + member_ids, + vec![existing_member], + "enumeration should see only the pre-existing member while the lock is held" + ); + + // `add_group_member` locks the same group row, so it must block until + // the transaction above commits — not silently add a member our + // (already-captured) enumeration has already missed. + let p2 = p.clone(); + let new_member = active_entity(&p, "service").await; + let handle = + tokio::spawn(async move { identity_repo::add_group_member(&p2, group, new_member).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "add_group_member must block on the same group-row lock the enumeration holds, not \ + commit a membership change the enumeration has already missed" + ); + + tx.commit().await.expect("commit lock-holding tx"); + handle + .await + .expect("join add_group_member task") + .expect("add_group_member should succeed once unblocked"); +} + +/// Regression test for a review finding: `createRoleAssignment`'s +/// group-subject path locked the subject group's closure first and only +/// reached the role lock afterward (inside `create_role_assignment_in_tx`'s +/// own `lock_role` call) — the reverse of every role mutation +/// (`replaceRolePermissionBlocks`/`deleteRole`/`restoreRole`, via +/// `lock_role_and_collect_grants_keys`), which always locks the role first, +/// then any assigned groups' closures. If role R is already assigned to an +/// ancestor group and a concurrent pair of requests ran — assign R to a +/// descendant group; mutate R's blocks — they could hold the descendant +/// group's row and the role row in opposite order: a genuine wait-for cycle, +/// which Postgres detects and resolves by aborting one side with a +/// "deadlock detected" error. +/// +/// Fixed by having `createRoleAssignment`'s group-subject path lock the role +/// *before* the group closure too, via +/// `authz::repo::lock_role_then_group_closure_and_collect_grants_keys`. +/// +/// A first version of this test held only the role lock and asserted +/// `createRoleAssignment` blocked on it — but that passes under *either* +/// lock order (the old order still reaches the role lock eventually, via +/// `create_role_assignment_in_tx`'s own `lock_role` call, just later), so it +/// couldn't actually distinguish fixed from unfixed (confirmed: it passed +/// even with the fix reverted). This version instead reconstructs the +/// actual two-resource cycle: hold the role lock, let the spawned +/// `createRoleAssignment` block on it, then — *while it's still blocked* — +/// try to lock the descendant group ourselves. With the fix, the spawned +/// task hasn't touched the group yet (it's queued for the role first), so +/// our lock attempt succeeds immediately. With the old order, the spawned +/// task would already be holding the group (locked before it ever reached +/// for the role), so our attempt would itself block — the two sides +/// waiting on each other — which is exactly the cycle this fix closes. +#[tokio::test] +#[ignore] +async fn create_role_assignment_for_group_subject_locks_the_role_before_the_group() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let ancestor = new_group(&p, "lock-order-ancestor").await; + let descendant = new_group(&p, "lock-order-descendant").await; + identity_repo::set_group_parent(&p, descendant, ancestor) + .await + .expect("set parent"); + let role = authz_repo::create_role( + &p, + CreateRole { + name: format!("m25-lock-order-role-{}", Uuid::new_v4()), + tenant_id: None, + description: None, + }, + ) + .await + .expect("create role"); + // Assigning R to the ancestor puts `descendant` in R's assigned-group + // closure — exactly what `lock_role_and_collect_grants_keys` would lock + // for a mutation on R. + authz_repo::create_role_assignment( + &p, + CreateRoleAssignment { + tenant_id: None, + subject_kind: SubjectKind::Group, + subject_id: ancestor, + role_id: role.id, + }, + ) + .await + .expect("assign role to ancestor"); + + // Hold only the role lock — exactly what a role mutation + // (`lock_role_and_collect_grants_keys`) holds at the point it's already + // past its own first step, before it reaches for the assigned groups' + // closures. + let mut tx = p.begin().await.expect("begin tx"); + sqlx::query("SELECT id FROM roles WHERE id = $1 FOR UPDATE") + .bind(role.id) + .fetch_one(&mut *tx) + .await + .expect("lock role"); + + // Concurrently: assign R to the descendant group, through the real, + // wired GraphQL path. + let schema = build_schema(state.clone()); + let cache2 = cache.clone(); + let role_id = role.id; + let handle = tokio::spawn(async move { + schema + .execute(authed( + common::admin_id(), + cache2, + format!( + r#"mutation {{ + createRoleAssignment(input: {{ + subjectKind: group, + subjectId: "{descendant}", + roleId: "{role_id}" + }}) {{ id }} + }}"# + ), + )) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "createRoleAssignment must block on the role lock" + ); + + // The decisive check: while the spawned task is blocked waiting for the + // role lock, we must still be able to lock the descendant group + // immediately — proving the spawned task has not already acquired it + // (which the old, reversed order would have done before ever touching + // the role). + tokio::time::timeout( + std::time::Duration::from_millis(500), + sqlx::query("SELECT id FROM principal_groups WHERE id = $1 FOR UPDATE") + .bind(descendant) + .fetch_one(&mut *tx), + ) + .await + .expect( + "locking the descendant group must not block while createRoleAssignment is blocked on \ + the role lock — a block here means it's holding the group while waiting on the role, \ + i.e. the exact deadlock cycle this fix closes", + ) + .expect("lock descendant group"); + + tx.commit().await.expect("commit tx"); + let resp = handle.await.expect("join createRoleAssignment task"); + assert!( + resp.errors.is_empty(), + "createRoleAssignment failed: {:?}", + resp.errors + ); +} + +// An end-to-end complement to the test above — racing a real +// `add_group_member` against the real `deleteDirectPolicy` mutation, then +// asserting the new member's grants are correctly denied — was tried and +// dropped. It passed even with the group-row lock reverted (confirmed by +// reverting and rerunning, 5/5), because it only calls `evaluate_read` once, +// *after* both operations have fully committed: that read is always a fresh +// cold-cache load reflecting final Postgres state, so it can never observe +// the stale intermediate value the race would have produced. Reproducing the +// actual staleness would require a read to land in the narrow window between +// the add's commit and the delete's commit — not achievable through ordinary +// concurrent scheduling without injecting a hook into the read path. The +// mechanism test above (proving the lock itself blocks a concurrent +// `add_group_member`) is the one that actually distinguishes fixed from +// unfixed here, the same lesson as the credential-restore investigation +// documented on `tenant_restore_clears_reactivated_credential_cache_entries`. + +async fn hmget_raw(key: &str) -> (Option, Option, Option>) { + let mut conn = raw_redis_conn().await; + redis::cmd("HMGET") + .arg(key) + .arg("v") + .arg("dirty") + .arg("p") + .query_async(&mut conn) + .await + .expect("hmget") +} + +/// Directly writes a clean (non-dirty, version 1), already-populated cache +/// entry — bypassing `try_populate`'s version check entirely, since this is +/// for hand-seeding test fixtures, not exercising the barrier itself. +async fn seed_hit(key: &str, value: &T) { + let payload = serde_json::to_vec(value).expect("serialize seed value"); + let mut conn = raw_redis_conn().await; + let _: () = redis::cmd("HSET") + .arg(key) + .arg("v") + .arg(1) + .arg("dirty") + .arg("0") + .arg("p") + .arg(payload) + .query_async(&mut conn) + .await + .expect("seed cache entry"); +} + +// ─── Negative control ─────────────────────────────────────────────────────── + +/// Proves the "tenant status short-circuits before grants" design decision is +/// actually followed, not just accidentally correct: a tenant status change +/// must invalidate `tenant_status`, but must NOT touch the `grants` key. +#[tokio::test] +#[ignore] +async fn tenant_status_change_does_not_touch_the_grants_cache_key() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let member = active_entity(&p, "service").await; + + let tenant_id = Uuid::new_v4(); + sqlx::query("INSERT INTO tenants (id, name, status) VALUES ($1, $2, 'active')") + .bind(tenant_id) + .bind(format!("cache-test-tenant-{tenant_id}")) + .execute(&p) + .await + .expect("insert tenant"); + + // Warm the member's grants cache (an empty grant set is still a cached + // entry — what matters is whether the key gets touched, not its value). + let req = AuthzRequest { + subject_id: member, + action: "read".into(), + resource_id: None, + object_kind: Some("tenant".into()), + object_id: Some(tenant_id), + context: json!({}), + }; + let auth = auth_context(member, cache.clone()); + let _ = engine::evaluate(&p, &req, &auth).await.expect("evaluate"); + + let grants_key = atom::cache::keys::grants(member); + let mut conn = raw_redis_conn().await; + let before_exists: bool = redis::cmd("EXISTS") + .arg(&grants_key) + .query_async(&mut conn) + .await + .expect("exists check"); + + let schema = build_schema(state); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ disableTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "disable failed: {:?}", resp.errors); + + let after_exists: bool = redis::cmd("EXISTS") + .arg(&grants_key) + .query_async(&mut conn) + .await + .expect("exists check"); + + assert_eq!( + before_exists, after_exists, + "tenant status change must not invalidate the grants cache key — the PDP's \ + tenant-lifecycle deny check runs before grant matching, so grants invalidation \ + is unnecessary here by design (see authz::engine::load_decision_context)" + ); +} From 1ec70f144f67245558d6525b445d458bee250d03 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 16:19:45 +0300 Subject: [PATCH 05/19] Fix enumerate-before-lock race in bulk session/credential invalidation deleteEntity, deleteTenant, restoreTenant, and password reset enumerated affected session/credential cache keys via a plain pool query before any transaction or lock was taken. A session or credential created concurrently in that window was never covered by the cache barrier and could keep serving as a stale hit indefinitely, even past a later restore. Fixed by moving enumeration inside the same transaction as the row-locking mutation that starts each of these flows, mirroring the existing group-membership invalidation fix. Adds a shared begin/end helper for establishing a multi-category cache barrier around an already-open transaction, and concurrency tests proving the lock now closes the window. --- src/cache/invalidate.rs | 36 +++++ src/graphql/entities.rs | 108 +++++++++---- src/graphql/tenants.rs | 178 ++++++++++++++------- src/identity/repo.rs | 142 +++++++++++++---- src/identity/service.rs | 137 +++++++++------- src/tenants/repo.rs | 271 +++++++++++++++++++++----------- tests/m25_cache_invalidation.rs | 128 +++++++++++++++ 7 files changed, 735 insertions(+), 265 deletions(-) diff --git a/src/cache/invalidate.rs b/src/cache/invalidate.rs index 347d623..22cb8ea 100644 --- a/src/cache/invalidate.rs +++ b/src/cache/invalidate.rs @@ -82,3 +82,39 @@ where } result } + +/// Establishes a barrier on every `(category, keys)` group, in order. Used by +/// callers that already hold an open `Transaction` across their own +/// lock/enumerate/mutate/commit sequence (so the barrier can't be established +/// via a `FnOnce` closure the way [`guarded_mutation`]/[`guarded_multi_mutation`] +/// do — a closure can't cleanly borrow `&mut Transaction` across an await +/// point on stable Rust). If a later group's barrier can't be established, the +/// ones already established are cleared immediately rather than left to +/// self-heal on their barrier TTL. Pair with [`end_all`], called +/// unconditionally after the mutation regardless of outcome. +pub async fn begin_all( + cache: &CacheClient, + groups: &[(CacheCategory, &[String])], +) -> Result<(), AppError> { + let mut established = Vec::with_capacity(groups.len()); + for &(category, keys) in groups { + match cache.begin(category, keys).await { + Ok(()) => established.push((category, keys)), + Err(err) => { + for (category, keys) in established { + cache.end(category, keys).await; + } + return Err(err); + } + } + } + Ok(()) +} + +/// Clears the barrier on every `(category, keys)` group established by a +/// prior [`begin_all`] call. Always best-effort, mirroring [`CacheClient::end`]. +pub async fn end_all(cache: &CacheClient, groups: &[(CacheCategory, &[String])]) { + for &(category, keys) in groups { + cache.end(category, keys).await; + } +} diff --git a/src/graphql/entities.rs b/src/graphql/entities.rs index a5af067..ba67aa0 100644 --- a/src/graphql/entities.rs +++ b/src/graphql/entities.rs @@ -370,20 +370,6 @@ impl EntityMutation { ) .await?; } - // Enumerate *before* the delete revokes them — the revoke - // UPDATEs' own `WHERE ... IS NULL`/`= 'active'` filters no - // longer match these rows afterward. - let session_ids = repo::entity_active_session_ids(&state.pool, id).await?; - let credential_ids = repo::entity_active_access_token_ids(&state.pool, id).await?; - let session_keys: Vec = session_ids - .iter() - .map(|sid| crate::cache::keys::session(*sid)) - .collect(); - let credential_keys: Vec = credential_ids - .iter() - .map(|cid| crate::cache::keys::credential(*cid)) - .collect(); - let entity_status_keys = [crate::cache::keys::entity_status(id)]; // `entity_status` invalidation alone is *not* sufficient: // `delete_entity` also revokes this entity's sessions and // access-token credentials in the same transaction, and @@ -394,27 +380,83 @@ impl EntityMutation { // Postgres check), then becomes a full cache hit again the // moment `restoreEntity` repopulates entity_status as active — // despite being revoked in Postgres and meant to stay that way. - crate::cache::invalidate::guarded_multi_mutation( - state.cache.as_deref(), - &[ - ( - crate::cache::CacheCategory::EntityStatus, - &entity_status_keys, - ), - (crate::cache::CacheCategory::Session, &session_keys), - (crate::cache::CacheCategory::Credential, &credential_keys), - ], - || { - repo::delete_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), - ) + // + // The session/credential ids are enumerated *inside* the same + // transaction as the status flip (see + // `deactivate_entity_and_collect_revocation_ids_in_tx`), not via + // a pre-transaction pool query — that lock is what stops a + // concurrently-created session/credential from being missed and + // left permanently uninvalidated. See `src/cache/mod.rs`'s + // consistency model. + let Some(cache) = state.cache.as_deref() else { + return repo::delete_entity_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let (session_ids, credential_ids) = + repo::deactivate_entity_and_collect_revocation_ids_in_tx( + &mut tx, + id, + Some(auth.entity_id), + ) + .await?; + let session_keys: Vec = session_ids + .iter() + .map(|sid| crate::cache::keys::session(*sid)) + .collect(); + let credential_keys: Vec = credential_ids + .iter() + .map(|cid| crate::cache::keys::credential(*cid)) + .collect(); + let entity_status_keys = [crate::cache::keys::entity_status(id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 3] = [ + ( + crate::cache::CacheCategory::EntityStatus, + &entity_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + (crate::cache::CacheCategory::Credential, &credential_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = repo::finish_entity_deletion_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + id, + ) + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + outcome?; + // Mirrors `delete_entity_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `delete_entity_with_audit` itself. + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: existing.tenant_id, + target_kind: Some("entity"), + target_id: Some(id), + event: "entity.delete", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), }, ) - .await + .await; + Ok(()) } .await; diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 0771ec6..6ebc01a 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -392,18 +392,8 @@ impl TenantMutation { }; let details = serde_json::json!({}); - let result = async { + let result: std::result::Result<(), AppError> = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - // Enumerate *before* the delete revokes them — the revoke - // UPDATE's own `WHERE revoked_at IS NULL` no longer matches these - // rows afterward. - let session_ids = - tenant_repo::tenant_active_session_ids(&state.pool, tenant_id).await?; - let session_keys: Vec = session_ids - .iter() - .map(|id| crate::cache::keys::session(*id)) - .collect(); - let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; // `soft_delete_tenant` is a separate function from // `change_tenant_status` (it also bulk-revokes sessions and // credentials). `tenant_status` invalidation alone is *not* @@ -414,29 +404,62 @@ impl TenantMutation { // stale session becomes a full cache hit and authenticates // despite being revoked in Postgres. Credentials don't need the // same treatment here — `restore_tenant`'s own invalidation - // (see `tenant_restore_reactivated_credential_ids`) already - // covers the credential side by the time a restore could ever - // matter. - crate::cache::invalidate::guarded_multi_mutation( - state.cache.as_deref(), - &[ - ( - crate::cache::CacheCategory::TenantStatus, - &tenant_status_keys, - ), - (crate::cache::CacheCategory::Session, &session_keys), - ], - || { - tenant_repo::soft_delete_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - Some(auth.entity_id), - ) - }, + // (see `reactivate_tenant_and_collect_credential_ids_in_tx`) + // already covers the credential side by the time a restore could + // ever matter. + // + // Session ids are enumerated *inside* the same transaction as the + // status flip (see + // `deactivate_tenant_and_collect_session_ids_in_tx`), not via a + // pre-transaction pool query — that lock is what stops a + // concurrently-created session from being missed and left + // permanently uninvalidated. See `src/cache/mod.rs`'s + // consistency model. + let Some(cache) = state.cache.as_deref() else { + tenant_repo::soft_delete_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + .await?; + return Ok(()); + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let (_tenant, session_ids) = + tenant_repo::deactivate_tenant_and_collect_session_ids_in_tx( + &mut tx, + tenant_id, + Some(auth.entity_id), + ) + .await?; + let session_keys: Vec = session_ids + .iter() + .map(|id| crate::cache::keys::session(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::finish_tenant_soft_delete_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, ) - .await + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + outcome } .await; @@ -483,34 +506,79 @@ impl TenantMutation { // overridden by a fresher tenant_status check afterward, so it // must be invalidated explicitly or a just-restored API key // keeps getting denied until its own cache entry's TTL expires. - let reactivated_credential_ids = - tenant_repo::tenant_restore_reactivated_credential_ids(&state.pool, tenant_id) - .await?; - let credential_keys: Vec = reactivated_credential_ids + // + // Credential ids are enumerated *inside* the same transaction as + // the status flip (see + // `reactivate_tenant_and_collect_credential_ids_in_tx`), not via + // a pre-transaction pool query, mirroring `delete_tenant`'s fix + // for the same class of race — see `src/cache/mod.rs`'s + // consistency model. + let Some(cache) = state.cache.as_deref() else { + return tenant_repo::restore_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + Some(auth.entity_id), + ) + .await; + }; + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let (tenant, credential_ids) = + tenant_repo::reactivate_tenant_and_collect_credential_ids_in_tx( + &mut tx, + tenant_id, + Some(auth.entity_id), + ) + .await?; + let credential_keys: Vec = credential_ids .iter() .map(|id| crate::cache::keys::credential(*id)) .collect(); let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; - crate::cache::invalidate::guarded_multi_mutation( - state.cache.as_deref(), - &[ - ( - crate::cache::CacheCategory::TenantStatus, - &tenant_status_keys, - ), - (crate::cache::CacheCategory::Credential, &credential_keys), - ], - || { - tenant_repo::restore_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - Some(auth.entity_id), - ) + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Credential, &credential_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::finish_tenant_restore_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + ) + .await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + outcome?; + // Mirrors `restore_tenant_with_audit`'s own post-commit audit + // write (fire-and-forget, after the mutation durably commits) — + // see `audit::commit_with_audit`'s doc comment. Only needed on + // this locked path; the cache-disabled fallback above already + // gets it from `restore_tenant_with_audit` itself. + audit::write( + &state.pool, + false, + audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: Some(tenant.id), + target_kind: Some("tenant"), + target_id: Some(tenant.id), + event: "tenant.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({ + "tenant_name": tenant.name, + }), }, ) - .await + .await; + Ok(tenant) } .await; diff --git a/src/identity/repo.rs b/src/identity/repo.rs index e01f5ed..8a9c7ae 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -731,30 +731,28 @@ pub async fn entity_active_access_token_ids( .map_err(db_err) } -/// Soft-delete an entity: mark it inactive, set the tombstone, and immediately -/// cut off access by revoking its credentials and active sessions. Physical -/// removal is deferred to the purge cron. Hard delete (the old behavior) relied -/// on FK cascade for the credential/session cleanup, so the revocations are now -/// explicit. -pub async fn delete_entity_with_audit( - pool: &PgPool, - events_enabled: bool, - actor_id: Option, +/// Flips the entity to inactive/tombstoned and, in the *same* transaction, +/// enumerates the exact session and access-token credential ids this delete +/// is about to revoke. Callers invalidate `atom:v1:session:*` / +/// `atom:v1:credential:*` cache entries for these before calling +/// [`finish_entity_deletion_in_tx`]. +/// +/// The status-flip `UPDATE` below takes an exclusive row lock on the entity — +/// the same lock `create_session`/`create_access_token` take (via +/// `lock_active_entity`) before inserting. So a session or access token +/// created concurrently for this entity either committed before this `UPDATE` +/// acquired the lock (and is therefore visible to the enumeration below, +/// which runs after it, in the same transaction) or is blocked until this +/// transaction commits (and then fails, since the entity is no longer +/// active). Enumerating via a plain pre-transaction pool query — the previous +/// shape of this code — could miss a session/credential created in that +/// window, leaving its cache entry uninvalidated indefinitely. See +/// `src/cache/mod.rs`'s consistency model. +pub async fn deactivate_entity_and_collect_revocation_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, id: Uuid, deleted_by: Option, -) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; - - let tenant_id: Option> = - sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1 AND deleted_at IS NULL") - .bind(id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - let Some(tenant_id) = tenant_id else { - return Err(AppError::not_found(format!("entity {id} not found"))); - }; - +) -> Result<(Vec, Vec), AppError> { let result = sqlx::query( "UPDATE entities SET status = 'inactive', deleted_at = now(), deleted_by = $2, updated_at = now() @@ -762,13 +760,46 @@ pub async fn delete_entity_with_audit( ) .bind(id) .bind(deleted_by) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; if result.rows_affected() == 0 { return Err(AppError::not_found(format!("entity {id} not found"))); } + let session_ids: Vec = + sqlx::query_scalar("SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL") + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + // Restricted to the one credential kind this codebase caches under + // `CacheCategory::Credential` (certificates are tracked via the CRL + // instead, not this cache). + let credential_ids: Vec = sqlx::query_scalar( + "SELECT id FROM credentials WHERE entity_id = $1 AND status = 'active' AND kind = $2", + ) + .bind(id) + .bind(crate::models::enums::CredentialKind::AccessToken) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok((session_ids, credential_ids)) +} + +/// Finishes the entity soft-delete started by +/// [`deactivate_entity_and_collect_revocation_ids_in_tx`] in the same +/// transaction: revokes active credentials (all kinds) and sessions, and +/// tombstones the email. Does not commit — the caller commits after this +/// succeeds, once the cache barrier established on the enumerated ids covers +/// the whole transaction. +pub async fn finish_entity_deletion_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { let revoked_certificates: i64 = sqlx::query_scalar( r#"WITH revoked AS ( UPDATE credentials @@ -787,17 +818,17 @@ pub async fn delete_entity_with_audit( SELECT COUNT(*) FILTER (WHERE kind = 'certificate') FROM revoked"#, ) .bind(id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if revoked_certificates > 0 { - crate::certs::repo::mark_crl_dirty_tx(&mut tx).await?; + crate::certs::repo::mark_crl_dirty_tx(tx).await?; } sqlx::query( "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -806,20 +837,23 @@ pub async fn delete_entity_with_audit( WHERE entity_id = $1 AND deleted_at IS NULL", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; - let event = crate::audit::AuditEvent { + let tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1") + .bind(id) + .fetch_one(&mut **tx) + .await + .map_err(db_err)?; + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, - target_kind: Some("entity"), + target_kind: "entity", target_id: Some(id), event: "entity.delete", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; Ok(()) } @@ -831,6 +865,50 @@ pub async fn delete_entity( delete_entity_with_audit(pool, false, None, id, deleted_by).await } +/// Soft-delete an entity: mark it inactive, set the tombstone, and immediately +/// cut off access by revoking its credentials and active sessions. Physical +/// removal is deferred to the purge cron. Hard delete (the old behavior) relied +/// on FK cascade for the credential/session cleanup, so the revocations are now +/// explicit. +/// +/// Used directly only when no cache is configured; the cache-aware path +/// (`graphql::entities::delete_entity`) calls +/// [`deactivate_entity_and_collect_revocation_ids_in_tx`] and +/// [`finish_entity_deletion_in_tx`] itself so it can establish the cache +/// barrier between them. +pub async fn delete_entity_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + deleted_by: Option, +) -> Result<(), AppError> { + let mut tx = pool.begin().await.map_err(db_err)?; + deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by).await?; + finish_entity_deletion_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid delete) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation above via `observe_in_tx`. + let tenant_id = get_entity(pool, id).await.ok().and_then(|e| e.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id, + target_kind: Some("entity"), + target_id: Some(id), + event: "entity.delete", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} + pub async fn restore_entity_with_audit( pool: &PgPool, events_enabled: bool, diff --git a/src/identity/service.rs b/src/identity/service.rs index 71b772b..eab917f 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -774,75 +774,100 @@ pub async fn reset_password( .map_err(db_err)?; let password_hash = hash_secret(req.password.as_bytes())?; + let mut tx = pool.begin().await.map_err(db_err)?; + if super::repo::lock_active_entity(&mut tx, entity_id) + .await? + .is_none() + { + return Err(AppError::bad_request("invalid password reset token")); + } + // Every currently-active session is about to be bulk-revoked below (by - // entity_id, not by session_id) — collect their cache keys first so the - // barrier can cover all of them, not just one. + // entity_id, not by session_id) — enumerated *inside* the transaction, + // immediately after the entity lock above: `create_session` takes the + // same lock before inserting, so a session created concurrently for this + // entity either committed before this transaction acquired the lock (and + // is therefore visible here) or is blocked until this transaction + // finishes. Enumerating via a plain pre-transaction pool query — the + // previous shape of this code — could miss a session created in that + // window, leaving its cache entry uninvalidated indefinitely. See + // `src/cache/mod.rs`'s consistency model. let session_keys: Vec = sqlx::query_scalar::<_, Uuid>( "SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL", ) .bind(entity_id) - .fetch_all(pool) + .fetch_all(&mut *tx) .await .map_err(db_err)? .into_iter() .map(crate::cache::keys::session) .collect(); - crate::cache::invalidate::guarded_mutation( - cache, - crate::cache::CacheCategory::Session, - &session_keys, - || async { - let mut tx = pool.begin().await.map_err(db_err)?; - if super::repo::lock_active_entity(&mut tx, entity_id) - .await? - .is_none() - { - return Err(AppError::bad_request("invalid password reset token")); - } - let updated = sqlx::query( - "UPDATE password_reset_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", - ) - .bind(token_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - if updated.rows_affected() == 0 { - return Err(AppError::bad_request("password reset token expired")); - } - sqlx::query( - r#"UPDATE credentials - SET status = 'revoked' - WHERE entity_id = $1 AND kind = 'password' AND status = 'active'"#, - ) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - sqlx::query( - r#"INSERT INTO credentials (id, entity_id, kind, identifier, secret_hash) - VALUES ($1, $2, $3, $4, $5)"#, - ) - .bind(Uuid::new_v4()) - .bind(entity_id) - .bind(CredentialKind::Password) - .bind(&email) - .bind(password_hash) - .execute(&mut *tx) - .await - .map_err(db_err)?; - sqlx::query( - "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", - ) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; - tx.commit().await.map_err(db_err)?; - Ok(()) - }, + if let Some(cache) = cache { + cache + .begin(crate::cache::CacheCategory::Session, &session_keys) + .await?; + } + let outcome = + finish_password_reset_in_tx(&mut tx, token_id, entity_id, &email, password_hash).await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(db_err), + Err(err) => Err(err), + }; + if let Some(cache) = cache { + cache + .end(crate::cache::CacheCategory::Session, &session_keys) + .await; + } + outcome +} + +async fn finish_password_reset_in_tx( + tx: &mut Transaction<'_, Postgres>, + token_id: Uuid, + entity_id: Uuid, + email: &str, + password_hash: String, +) -> Result<(), AppError> { + let updated = sqlx::query( + "UPDATE password_reset_tokens SET consumed_at = now() WHERE id = $1 AND consumed_at IS NULL", ) + .bind(token_id) + .execute(&mut **tx) .await + .map_err(db_err)?; + if updated.rows_affected() == 0 { + return Err(AppError::bad_request("password reset token expired")); + } + sqlx::query( + r#"UPDATE credentials + SET status = 'revoked' + WHERE entity_id = $1 AND kind = 'password' AND status = 'active'"#, + ) + .bind(entity_id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + sqlx::query( + r#"INSERT INTO credentials (id, entity_id, kind, identifier, secret_hash) + VALUES ($1, $2, $3, $4, $5)"#, + ) + .bind(Uuid::new_v4()) + .bind(entity_id) + .bind(CredentialKind::Password) + .bind(email) + .bind(password_hash) + .execute(&mut **tx) + .await + .map_err(db_err)?; + sqlx::query( + "UPDATE sessions SET revoked_at = now() WHERE entity_id = $1 AND revoked_at IS NULL", + ) + .bind(entity_id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + Ok(()) } pub async fn oauth_start( diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index 67d3779..f94a484 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -547,47 +547,28 @@ pub async fn update_tenant( update_tenant_with_audit(pool, false, None, id, req, updated_by).await } -/// The exact set of active session ids `soft_delete_tenant` is about to -/// revoke — mirrors that function's `UPDATE sessions` `WHERE` clause -/// precisely, so callers can invalidate `atom:v1:session:*` cache entries for -/// them *before* the delete runs (afterward, `revoked_at IS NULL` no longer -/// matches these rows). See `src/cache/mod.rs`'s consistency model. -pub async fn tenant_active_session_ids( - pool: &PgPool, - tenant_id: Uuid, -) -> Result, AppError> { - sqlx::query_scalar( - r#"SELECT id FROM sessions - WHERE revoked_at IS NULL - AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)"#, - ) - .bind(tenant_id) - .fetch_all(pool) - .await - .map_err(db_err) -} - -/// Soft-delete a tenant: mark `status = deleted`, stamp the tombstone, and -/// immediately revoke every active credential and session of entities in the -/// tenant. Physical removal (and the entity cascade) is deferred to the purge -/// cron. -pub async fn soft_delete_tenant( - pool: &PgPool, - id: Uuid, - deleted_by: Option, -) -> Result { - soft_delete_tenant_with_audit(pool, false, None, id, deleted_by).await -} - -pub async fn soft_delete_tenant_with_audit( - pool: &PgPool, - events_enabled: bool, - actor_id: Option, +/// Flips the tenant to `deleted` and, in the *same* transaction, enumerates +/// the exact active session ids of its member entities that +/// [`finish_tenant_soft_delete_in_tx`] is about to revoke. Callers invalidate +/// `atom:v1:session:*` cache entries for these before calling it. +/// +/// The status-flip `UPDATE` below takes an exclusive row lock on the tenant — +/// the same lock `lock_active_tenant`/`lock_optional_active_tenant` take +/// (transitively, via `lock_active_entity`) before any session or credential +/// can be created for an entity in this tenant. So a session created +/// concurrently for a member entity either committed before this `UPDATE` +/// acquired the lock (and is therefore visible to the enumeration below, +/// which runs after it, in the same transaction) or is blocked until this +/// transaction commits (and then fails, since the tenant is no longer +/// active). Enumerating via a plain pre-transaction pool query — the previous +/// shape of this code — could miss a session created in that window, leaving +/// its cache entry uninvalidated indefinitely. See `src/cache/mod.rs`'s +/// consistency model. +pub async fn deactivate_tenant_and_collect_session_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, id: Uuid, deleted_by: Option, -) -> Result { - let mut tx = pool.begin().await.map_err(db_err)?; - +) -> Result<(Tenant, Vec), AppError> { let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = 'deleted', deleted_at = now(), deleted_by = $2, @@ -597,13 +578,42 @@ pub async fn soft_delete_tenant_with_audit( )) .bind(id) .bind(deleted_by) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), other => AppError::Database(other), })?; + let session_ids: Vec = sqlx::query_scalar( + r#"SELECT id FROM sessions + WHERE revoked_at IS NULL + AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)"#, + ) + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok((tenant, session_ids)) +} + +/// Finishes the tenant soft-delete started by +/// [`deactivate_tenant_and_collect_session_ids_in_tx`] in the same +/// transaction: revokes every active credential and session belonging to the +/// tenant's entities. Does not commit — the caller commits after this +/// succeeds, once the cache barrier established on the enumerated session ids +/// covers the whole transaction. +pub async fn finish_tenant_soft_delete_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { + // Stamp the tenant-delete marker on every revoked credential (not just + // certificates) so restore_tenant can reverse exactly the revocations this + // delete caused, without disturbing credentials revoked earlier for other + // reasons. let revoked_certificates: i64 = sqlx::query_scalar( r#"WITH revoked AS ( UPDATE credentials c @@ -621,11 +631,11 @@ pub async fn soft_delete_tenant_with_audit( SELECT COUNT(*) FROM revoked WHERE kind = 'certificate'"#, ) .bind(id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if revoked_certificates > 0 { - crate::certs::repo::mark_crl_dirty_tx(&mut tx).await?; + crate::certs::repo::mark_crl_dirty_tx(tx).await?; } sqlx::query( @@ -634,7 +644,7 @@ pub async fn soft_delete_tenant_with_audit( AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; @@ -646,7 +656,39 @@ pub async fn soft_delete_tenant_with_audit( event: "tenant.delete", }; let details = serde_json::json!({}); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; + Ok(()) +} + +/// Soft-delete a tenant: mark `status = deleted`, stamp the tombstone, and +/// immediately revoke every active credential and session of entities in the +/// tenant. Physical removal (and the entity cascade) is deferred to the purge +/// cron. +pub async fn soft_delete_tenant( + pool: &PgPool, + id: Uuid, + deleted_by: Option, +) -> Result { + soft_delete_tenant_with_audit(pool, false, None, id, deleted_by).await +} + +/// Used directly only when no cache is configured; the cache-aware path +/// (`graphql::tenants::delete_tenant`) calls +/// [`deactivate_tenant_and_collect_session_ids_in_tx`] and +/// [`finish_tenant_soft_delete_in_tx`] itself so it can establish the cache +/// barrier between them. +pub async fn soft_delete_tenant_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + deleted_by: Option, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let (tenant, _session_ids) = + deactivate_tenant_and_collect_session_ids_in_tx(&mut tx, id, deleted_by).await?; + finish_tenant_soft_delete_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; Ok(tenant) } @@ -667,45 +709,31 @@ pub async fn soft_delete_tenant_with_audit( /// /// Fails with a conflict if the tenant name/alias was re-taken by a live tenant /// during the retention window. -/// The exact set of credential ids `restore_tenant` is about to reactivate — -/// mirrors that function's `UPDATE credentials` `WHERE` clause precisely, so -/// callers can invalidate `atom:v1:credential:*` cache entries for them -/// *before* running the restore (see `src/cache/mod.rs`'s consistency -/// model). A stale cached "revoked" credential is a false-deny (fails -/// closed, not a security hole) but is still worth fixing: unlike a tenant -/// or entity status flip, which a later-checked fresh field can catch -/// regardless of earlier stale fields, `verify_api_key_snapshot` checks the -/// credential's own status first — a stale value there is never overridden -/// by anything checked afterward. -pub async fn tenant_restore_reactivated_credential_ids( - pool: &PgPool, - tenant_id: Uuid, -) -> Result, AppError> { - sqlx::query_scalar( - r#"SELECT c.id - FROM credentials c - JOIN entities e ON c.entity_id = e.id - WHERE e.tenant_id = $1 - AND e.deleted_at IS NULL - AND c.status = 'revoked' - AND c.kind <> 'certificate' - AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, - ) - .bind(tenant_id) - .fetch_all(pool) - .await - .map_err(db_err) -} - -pub async fn restore_tenant_with_audit( - pool: &PgPool, - events_enabled: bool, - actor_id: Option, +/// +/// Flips the tenant back to `active` and, in the *same* transaction, +/// enumerates the exact credential ids [`finish_tenant_restore_in_tx`] is +/// about to reactivate — mirrors that function's `UPDATE credentials` `WHERE` +/// clause precisely, so callers can invalidate `atom:v1:credential:*` cache +/// entries for them (see `src/cache/mod.rs`'s consistency model). A stale +/// cached "revoked" credential is a false-deny (fails closed, not a security +/// hole) but is still worth fixing: unlike a tenant or entity status flip, +/// which a later-checked fresh field can catch regardless of earlier stale +/// fields, `verify_api_key_snapshot` checks the credential's own status first +/// — a stale value there is never overridden by anything checked afterward. +/// +/// The status-flip `UPDATE` below takes an exclusive row lock on the tenant, +/// so the enumeration that follows it (in the same transaction) sees a +/// consistent snapshot with respect to any concurrent restore/purge of the +/// same tenant — mirroring the same lock-then-enumerate shape used by +/// [`deactivate_tenant_and_collect_session_ids_in_tx`], even though (unlike +/// that case) no *new* matching credential can appear here: only +/// `soft_delete_tenant` ever stamps the `tenant_deleted` revocation reason, +/// and it cannot run again against an already-deleted tenant. +pub async fn reactivate_tenant_and_collect_credential_ids_in_tx( + tx: &mut Transaction<'_, Postgres>, id: Uuid, restored_by: Option, -) -> Result { - let mut tx = pool.begin().await.map_err(db_err)?; - +) -> Result<(Tenant, Vec), AppError> { let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = 'active', deleted_at = NULL, deleted_by = NULL, @@ -715,7 +743,7 @@ pub async fn restore_tenant_with_audit( )) .bind(id) .bind(restored_by) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => { @@ -724,6 +752,35 @@ pub async fn restore_tenant_with_audit( other => restore_conflict(other), })?; + let credential_ids: Vec = sqlx::query_scalar( + r#"SELECT c.id + FROM credentials c + JOIN entities e ON c.entity_id = e.id + WHERE e.tenant_id = $1 + AND e.deleted_at IS NULL + AND c.status = 'revoked' + AND c.kind <> 'certificate' + AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, + ) + .bind(id) + .fetch_all(&mut **tx) + .await + .map_err(db_err)?; + + Ok((tenant, credential_ids)) +} + +/// Finishes the tenant restore started by +/// [`reactivate_tenant_and_collect_credential_ids_in_tx`] in the same +/// transaction: reactivates exactly the credentials it enumerated. Does not +/// commit — the caller commits after this succeeds, once the cache barrier +/// established on those credential ids covers the whole transaction. +pub async fn finish_tenant_restore_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, +) -> Result<(), AppError> { sqlx::query( r#"UPDATE credentials c SET status = 'active', @@ -737,21 +794,19 @@ pub async fn restore_tenant_with_audit( AND c.metadata->>'revocation_reason' = 'tenant_deleted'"#, ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; - let event = crate::audit::AuditEvent { + let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id: Some(id), - target_kind: Some("tenant"), + target_kind: "tenant", target_id: Some(id), event: "tenant.restore", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), }; - crate::audit::commit_with_audit(pool, tx, events_enabled, &event).await?; - Ok(tenant) + crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; + Ok(()) } pub async fn restore_tenant( @@ -762,6 +817,44 @@ pub async fn restore_tenant( restore_tenant_with_audit(pool, false, None, id, restored_by).await } +/// Used directly only when no cache is configured; the cache-aware path +/// (`graphql::tenants::restore_tenant`) calls +/// [`reactivate_tenant_and_collect_credential_ids_in_tx`] and +/// [`finish_tenant_restore_in_tx`] itself so it can establish the cache +/// barrier between them. +pub async fn restore_tenant_with_audit( + pool: &PgPool, + events_enabled: bool, + actor_id: Option, + id: Uuid, + restored_by: Option, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let (tenant, _credential_ids) = + reactivate_tenant_and_collect_credential_ids_in_tx(&mut tx, id, restored_by).await?; + finish_tenant_restore_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + // The audit_logs row is deliberately written after commit (fire-and-forget, + // never blocks an already-valid restore) — see `audit::commit_with_audit`'s + // doc comment. The outbox row, by contrast, went in atomically with the + // mutation above via `observe_in_tx`. + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: Some("tenant"), + target_id: Some(id), + event: "tenant.restore", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(tenant) +} + pub(crate) async fn tenant_purge_object_ids( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, tenant_ids: &[Uuid], diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 30c6fc4..6e3c64c 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -35,6 +35,7 @@ use atom::{ token::CreateAccessToken, }, state::AppState, + tenants::repo as tenant_repo, }; use common::{cache_client, pool}; use serde_json::json; @@ -1037,6 +1038,133 @@ async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_access_to ); } +// ─── Enumerate-before-lock race (session/credential bulk invalidation) ───── + +/// Regression test for a review finding: `deleteEntity`'s session/access-token +/// cache-key enumeration used to run via a plain pool query *before* any +/// transaction or lock was taken. A session or access-token credential +/// created for the entity concurrently — after that enumeration but before +/// the delete's own revoke ran — was never included in the cache barrier and +/// could keep serving as a valid cache hit indefinitely (past the delete, and +/// even across a later `restoreEntity`), despite being revoked in Postgres. +/// This is the identical race shape already closed for group-membership +/// invalidation (see `concurrent_group_membership_change_serializes_against_ +/// the_group_subject_lock` below), just previously unfixed for +/// sessions/credentials. +/// +/// Fixed by moving the enumeration inside the same transaction as the +/// status-flip `UPDATE` +/// (`identity::repo::deactivate_entity_and_collect_revocation_ids_in_tx`), +/// which takes an exclusive lock on the entity row — the same lock +/// `create_session`/`create_access_token` take (via `lock_active_entity`) +/// before inserting. As with the group-membership fix, an end-to-end test +/// racing a real `create_session` against a real `deleteEntity` mutation +/// can't actually distinguish fixed from unfixed (a post-hoc read is always a +/// fresh cold-cache load reflecting final Postgres state); this test instead +/// calls the new function directly and proves there is no window in which a +/// concurrent `create_session` can slip past it uncounted: either it's +/// blocked until the transaction commits, or (once committed) it correctly +/// fails against the now-deactivated entity — there is no third outcome +/// where it silently succeeds outside the transaction's lock. +#[tokio::test] +#[ignore] +async fn concurrent_session_creation_cannot_evade_the_entity_delete_enumeration() { + let p = pool().await; + let entity = active_entity(&p, "service").await; + + let mut tx = p.begin().await.expect("begin tx"); + let (session_ids, credential_ids) = + identity_repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, entity, None) + .await + .expect("lock, deactivate, and enumerate"); + assert!( + session_ids.is_empty() && credential_ids.is_empty(), + "no sessions/credentials exist yet for this fresh entity" + ); + + // `create_session` takes the same entity-row lock before inserting, so it + // must block until the transaction above commits — not race ahead of the + // still-open delete transaction and create a session the enumeration has + // already missed. + let p2 = p.clone(); + let handle = + tokio::spawn(async move { identity_repo::create_session(&p2, entity, 3600).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "create_session must block on the same entity-row lock the delete's enumeration holds, \ + not commit a session the enumeration has already missed" + ); + + tx.commit().await.expect("commit lock-holding tx"); + + // Once committed, the entity is deactivated (the same effect + // `deleteEntity`'s cache-aware path has by this point). The + // previously-blocked `create_session` must now correctly fail rather than + // slip through and create a session outside of any cache barrier the + // delete established. + let outcome = handle.await.expect("join create_session task"); + assert!( + outcome.is_err(), + "create_session must fail once the entity has been deactivated by the transaction it \ + was blocked on — succeeding here would mean a session was created in a window the \ + delete's enumeration could never see, exactly the race this fix closes" + ); +} + +/// Tenant-level counterpart to the test above: `deleteTenant`'s session +/// cache-key enumeration used to run via a plain pool query before any +/// transaction/lock was taken, so a session created for a member entity in +/// the window between enumeration and the tenant's own bulk revoke was never +/// included in the cache barrier. Fixed by moving the enumeration inside the +/// same transaction as the tenant's status-flip `UPDATE` +/// (`tenants::repo::deactivate_tenant_and_collect_session_ids_in_tx`), which +/// takes an exclusive lock on the tenant row — the same lock +/// `lock_active_entity` takes (via `lock_optional_active_tenant`) before any +/// session/credential can be created for *any* entity in the tenant. +#[tokio::test] +#[ignore] +async fn concurrent_session_creation_cannot_evade_the_tenant_delete_enumeration() { + let p = pool().await; + let tenant_id = tenant(&p).await; + let entity = active_entity_in_tenant(&p, tenant_id, "service").await; + + let mut tx = p.begin().await.expect("begin tx"); + let (_tenant, session_ids) = + tenant_repo::deactivate_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id, None) + .await + .expect("lock, deactivate, and enumerate"); + assert!( + session_ids.is_empty(), + "no sessions exist yet for this fresh tenant's member" + ); + + let p2 = p.clone(); + let handle = + tokio::spawn(async move { identity_repo::create_session(&p2, entity, 3600).await }); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle.is_finished(), + "create_session for an entity in this tenant must block on the tenant-row lock the \ + delete's enumeration holds, not commit a session the enumeration has already missed" + ); + + tx.commit().await.expect("commit lock-holding tx"); + + // Once committed, the tenant is deleted (so `lock_active_entity`'s own + // `lock_optional_active_tenant` check fails) — the previously-blocked + // `create_session` must now correctly fail rather than slip through. + let outcome = handle.await.expect("join create_session task"); + assert!( + outcome.is_err(), + "create_session must fail once the tenant has been deactivated by the transaction it \ + was blocked on — succeeding here would mean a session was created in a window the \ + delete's enumeration could never see, exactly the race this fix closes" + ); +} + // ─── API-key tenant context after an entity moves tenants ────────────────── /// Regression test for a review finding: the API-key fast path derived From 1f60c2aa30d7d83ee4cfffdf67b5cdc04fdd2986 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Wed, 29 Jul 2026 16:39:50 +0300 Subject: [PATCH 06/19] Add Redis service to CI so cache-gated tests run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache-gated tests (src/cache/mod.rs's unit tests and tests/m25_cache_invalidation.rs) are #[ignore]d so they're skippable without Redis, but CI runs the full suite with --include-ignored and had no Redis service or ATOM_TEST_REDIS_URL configured — they panicked on missing config, which aborted the test script before the per-binary loop even reached the integration test files. --- .github/workflows/rust.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cc89e8c..8ad040f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -34,6 +34,23 @@ jobs: --health-timeout 5s --health-retries 5 + # Cache-gated tests (src/cache/mod.rs's unit tests, tests/m25_cache_invalidation.rs) + # are #[ignore]d specifically so they're skippable without Redis, but + # `--include-ignored` below runs them anyway — they need a reachable + # Redis, unlike the Postgres-gated tests which need their own fresh + # database per binary, these use randomly-suffixed keys and don't share + # mutable state across tests/binaries, so one shared instance for the + # whole job is sufficient. + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v4 @@ -66,6 +83,7 @@ jobs: env: PGPASSWORD: atom MAINT_URL: postgres://atom:atom@localhost:5432/atom + ATOM_TEST_REDIS_URL: redis://localhost:6379/0 run: | set -euo pipefail command -v psql >/dev/null || { From cbfd8deabc2920609b4b13c896c2113913081684 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Thu, 30 Jul 2026 11:13:36 +0300 Subject: [PATCH 07/19] Trim redundant doc-comment boilerplate in the cache-invalidation code Condenses the repeated "body, minus opening/committing its own transaction" template across the _in_tx twin functions in authz/repo.rs and identity/repo.rs into a short pointer plus the caller-specific detail. Also removes resolver-level comments that fully restated their own repo function's doc comment (delete_entity, delete_tenant, restore_tenant, reset_password). --- src/authz/repo.rs | 60 ++++++++++++++++++----------------------- src/graphql/entities.rs | 26 ++++++------------ src/graphql/tenants.rs | 57 +++++++++++++-------------------------- src/identity/repo.rs | 52 ++++++++++++++++------------------- src/identity/service.rs | 17 +++++------- 5 files changed, 83 insertions(+), 129 deletions(-) diff --git a/src/authz/repo.rs b/src/authz/repo.rs index 4d4ceac..2a8ba1e 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -1240,12 +1240,10 @@ pub async fn replace_role_permission_block_links_with_audit( tx.commit().await.map_err(db_err) } -/// [`replace_role_permission_block_links`]'s body, minus opening/committing -/// its own transaction — see [`create_role_assignment_in_tx`]'s doc comment -/// for the caller contract (the resolver locks the role, and every group -/// currently assigned it, via [`lock_role_and_collect_grants_keys`] on this -/// same `tx` first — `lock_role` below then just re-acquires that same, -/// already-held role lock). +/// Body of [`replace_role_permission_block_links`]; caller contract per +/// [`create_role_assignment_in_tx`]. The resolver must already hold the role +/// lock via [`lock_role_and_collect_grants_keys`] on this `tx` — `lock_role` +/// below just re-acquires it (same-transaction no-op). pub(crate) async fn replace_role_permission_block_links_in_tx( pool: &PgPool, tx: &mut Transaction<'_, Postgres>, @@ -2828,11 +2826,9 @@ pub async fn delete_role_with_audit( tx.commit().await.map_err(db_err) } -/// [`delete_role`]'s body, minus opening/committing its own transaction — -/// see [`create_role_assignment_in_tx`]'s doc comment for the caller -/// contract (the resolver locks the role, and every group currently -/// assigned it, via [`lock_role_and_collect_grants_keys`] on this same `tx` -/// first). +/// Body of [`delete_role`]; caller contract per +/// [`create_role_assignment_in_tx`] — the resolver must already hold the +/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. pub(crate) async fn delete_role_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -2913,11 +2909,9 @@ pub async fn restore_role_with_audit( Ok(()) } -/// [`restore_role`]'s body, minus opening/committing its own transaction — -/// see [`create_role_assignment_in_tx`]'s doc comment for the caller -/// contract (the resolver locks the role, and every group currently -/// assigned it, via [`lock_role_and_collect_grants_keys`] on this same `tx` -/// first). +/// Body of [`restore_role`]; caller contract per +/// [`create_role_assignment_in_tx`] — the resolver must already hold the +/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. pub(crate) async fn restore_role_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -4075,14 +4069,15 @@ pub async fn create_role_assignment_with_audit( Ok(assignment) } -/// [`create_role_assignment`]'s body, minus opening/committing its own -/// transaction. For a group subject, the caller (the group-subject mutation -/// resolver path) must have already run -/// [`lock_group_closures_and_collect_grants_keys`] on this same `tx` and -/// called `cache.begin()` on the result before calling this — `lock_role` -/// and `lock_live_subject` below then just re-acquire (safe, same-transaction -/// no-op) locks this function has always taken. The caller commits `tx`, not -/// this function. +/// Body of [`create_role_assignment`], callable directly against a caller-held +/// `tx` (not committed here). For a group subject, the caller must already +/// have run [`lock_group_closures_and_collect_grants_keys`] on this same `tx` +/// and called `cache.begin()` on the result — `lock_role`/`lock_live_subject` +/// below then just re-acquire those same locks (safe, same-transaction no-op). +/// Every other `_in_tx` twin in this module and `identity::repo` follows this +/// same convention: caller locks + begins the cache barrier first, this kind +/// of function re-acquires (never re-validates) those locks, and the caller +/// commits. pub(crate) async fn create_role_assignment_in_tx( pool: &PgPool, tx: &mut Transaction<'_, Postgres>, @@ -4256,10 +4251,9 @@ pub async fn delete_role_assignment_with_audit( tx.commit().await.map_err(db_err) } -/// [`delete_role_assignment`]'s body, minus opening/committing its own -/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the -/// caller contract (group-subject resolver path locks the subject's group -/// closure on this same `tx` first). +/// Body of [`delete_role_assignment`]; caller contract per +/// [`create_role_assignment_in_tx`] — the group-subject resolver path must +/// already hold the subject's group closure lock on this `tx`. pub(crate) async fn delete_role_assignment_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -4314,9 +4308,8 @@ pub async fn create_direct_policy_with_audit( Ok(policy) } -/// [`create_direct_policy`]'s body, minus opening/committing its own -/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the -/// caller contract. +/// Body of [`create_direct_policy`]; caller contract per +/// [`create_role_assignment_in_tx`]. pub(crate) async fn create_direct_policy_in_tx( pool: &PgPool, tx: &mut Transaction<'_, Postgres>, @@ -4450,9 +4443,8 @@ pub async fn delete_direct_policy_with_audit( tx.commit().await.map_err(db_err) } -/// [`delete_direct_policy`]'s body, minus opening/committing its own -/// transaction — see [`create_role_assignment_in_tx`]'s doc comment for the -/// caller contract. +/// Body of [`delete_direct_policy`]; caller contract per +/// [`create_role_assignment_in_tx`]. pub(crate) async fn delete_direct_policy_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, diff --git a/src/graphql/entities.rs b/src/graphql/entities.rs index ba67aa0..201cbb1 100644 --- a/src/graphql/entities.rs +++ b/src/graphql/entities.rs @@ -370,24 +370,14 @@ impl EntityMutation { ) .await?; } - // `entity_status` invalidation alone is *not* sufficient: - // `delete_entity` also revokes this entity's sessions and - // access-token credentials in the same transaction, and - // `restoreEntity` deliberately does not reinstate them (an - // identity must re-authenticate after restore). A stale cached - // session/credential survives the tombstoned window untouched - // (masked only by the entity_status miss forcing a fresh - // Postgres check), then becomes a full cache hit again the - // moment `restoreEntity` repopulates entity_status as active — - // despite being revoked in Postgres and meant to stay that way. - // - // The session/credential ids are enumerated *inside* the same - // transaction as the status flip (see - // `deactivate_entity_and_collect_revocation_ids_in_tx`), not via - // a pre-transaction pool query — that lock is what stops a - // concurrently-created session/credential from being missed and - // left permanently uninvalidated. See `src/cache/mod.rs`'s - // consistency model. + // `entity_status` invalidation alone is *not* sufficient: this + // also revokes sessions and access-token credentials, which + // `restoreEntity` deliberately never reinstates. Without their + // own invalidation, a stale cached session/credential would + // become a full hit again the moment `restoreEntity` repopulates + // entity_status as active — despite staying revoked in Postgres. + // See `deactivate_entity_and_collect_revocation_ids_in_tx` for + // why the ids are enumerated inside the same locked transaction. let Some(cache) = state.cache.as_deref() else { return repo::delete_entity_with_audit( &state.pool, diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 6ebc01a..4908623 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -394,27 +394,15 @@ impl TenantMutation { let result: std::result::Result<(), AppError> = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - // `soft_delete_tenant` is a separate function from - // `change_tenant_status` (it also bulk-revokes sessions and - // credentials). `tenant_status` invalidation alone is *not* - // sufficient: while the tenant stays deleted, a stale cached - // session survives untouched (masked only by the tenant_status - // miss forcing a fresh Postgres check), but the moment - // `restoreTenant` repopulates tenant_status as active again, that - // stale session becomes a full cache hit and authenticates - // despite being revoked in Postgres. Credentials don't need the - // same treatment here — `restore_tenant`'s own invalidation - // (see `reactivate_tenant_and_collect_credential_ids_in_tx`) - // already covers the credential side by the time a restore could - // ever matter. - // - // Session ids are enumerated *inside* the same transaction as the - // status flip (see - // `deactivate_tenant_and_collect_session_ids_in_tx`), not via a - // pre-transaction pool query — that lock is what stops a - // concurrently-created session from being missed and left - // permanently uninvalidated. See `src/cache/mod.rs`'s - // consistency model. + // `tenant_status` invalidation alone is *not* sufficient: this + // also bulk-revokes sessions, which would otherwise become a + // stale cache hit again the moment `restoreTenant` repopulates + // tenant_status as active. Credentials don't need the same + // treatment — `restore_tenant`'s own invalidation (see + // `reactivate_tenant_and_collect_credential_ids_in_tx`) already + // covers that side by the time a restore could matter. See + // `deactivate_tenant_and_collect_session_ids_in_tx` for why + // session ids are enumerated inside the same locked transaction. let Some(cache) = state.cache.as_deref() else { tenant_repo::soft_delete_tenant_with_audit( &state.pool, @@ -496,23 +484,16 @@ impl TenantMutation { let result = async { crate::auth::require_capability(&state.pool, &auth, "manage", Scope::Platform).await?; - // `restore_tenant` is a separate function from - // `change_tenant_status` and touches two cache categories in one - // transaction: the tenant's own status, and every credential it - // reactivates. Both need invalidating — unlike the - // tenant-status-only case in `delete_tenant`/ - // `change_tenant_status`, a stale cached credential status is - // checked *first* in `verify_api_key_snapshot` and isn't - // overridden by a fresher tenant_status check afterward, so it - // must be invalidated explicitly or a just-restored API key - // keeps getting denied until its own cache entry's TTL expires. - // - // Credential ids are enumerated *inside* the same transaction as - // the status flip (see - // `reactivate_tenant_and_collect_credential_ids_in_tx`), not via - // a pre-transaction pool query, mirroring `delete_tenant`'s fix - // for the same class of race — see `src/cache/mod.rs`'s - // consistency model. + // Unlike the tenant-status-only case in `delete_tenant`, this + // also reactivates credentials, which need their own + // invalidation: `verify_api_key_snapshot` checks a credential's + // status first and isn't overridden by a fresher tenant_status + // check afterward, so a just-restored API key would keep + // getting denied until its own cache entry's TTL expires + // otherwise. See + // `reactivate_tenant_and_collect_credential_ids_in_tx` for why + // credential ids are enumerated inside the same locked + // transaction. let Some(cache) = state.cache.as_deref() else { return tenant_repo::restore_tenant_with_audit( &state.pool, diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 8a9c7ae..4dc0604 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -1323,12 +1323,10 @@ pub async fn list_groups(pool: &PgPool, params: ListGroups) -> Result, events_enabled: bool, @@ -1459,14 +1457,13 @@ pub async fn set_group_parent_with_audit( get_group(pool, child_id).await } -/// [`set_group_parent`]'s body, minus opening/committing its own -/// transaction and the post-commit `get_group` read — see -/// `authz::repo::create_role_assignment_in_tx`'s doc comment for the caller -/// contract (the resolver locks `child_id`'s closure via -/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same -/// `tx` first — reparenting `child_id` only changes what it and its -/// descendants inherit from above, never `group_hierarchy` rows below it, so -/// that closure is exactly what this mutation can affect). +/// Body of [`set_group_parent`] (minus its post-commit `get_group` read); +/// caller contract per `authz::repo::create_role_assignment_in_tx`. The +/// resolver must already hold `child_id`'s closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx` — +/// reparenting only changes what `child_id` and its descendants inherit from +/// above, never `group_hierarchy` rows below it, so that closure is exactly +/// what this mutation can affect. pub(crate) async fn set_group_parent_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -1643,9 +1640,8 @@ pub async fn remove_group_parent_with_audit( tx.commit().await.map_err(db_err) } -/// [`remove_group_parent`]'s body, minus opening/committing its own -/// transaction — see [`set_group_parent_in_tx`]'s doc comment for the caller -/// contract. +/// Body of [`remove_group_parent`]; caller contract per +/// [`set_group_parent_in_tx`]. pub(crate) async fn remove_group_parent_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -1735,12 +1731,12 @@ pub async fn delete_group_with_audit( tx.commit().await.map_err(db_err) } -/// [`delete_group`]'s body, minus opening/committing its own transaction — -/// see `authz::repo::create_role_assignment_in_tx`'s doc comment for the -/// caller contract (the resolver locks this group's closure via -/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same -/// `tx` first — `group_hierarchy` rows aren't touched by a soft delete, only -/// `deleted_at`, so the closure is unaffected by this mutation's own effect). +/// Body of [`delete_group`]; caller contract per +/// `authz::repo::create_role_assignment_in_tx`. The resolver must already +/// hold this group's closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx` — +/// a soft delete only sets `deleted_at`, leaving `group_hierarchy` untouched, +/// so the closure is unaffected by this mutation's own effect. pub(crate) async fn delete_group_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, @@ -1832,12 +1828,10 @@ pub async fn restore_group_with_audit( Ok(()) } -/// [`restore_group`]'s body, minus opening/committing its own transaction — -/// see `authz::repo::create_role_assignment_in_tx`'s doc comment for the -/// caller contract (the resolver locks this group's closure via -/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this same -/// `tx` first — that lock works regardless of the group's own `deleted_at` -/// status, so it applies here unchanged). +/// Body of [`restore_group`]; caller contract per +/// `authz::repo::create_role_assignment_in_tx`. The resolver must already +/// hold this group's closure lock via +/// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx`. pub(crate) async fn restore_group_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, diff --git a/src/identity/service.rs b/src/identity/service.rs index eab917f..503994e 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -782,16 +782,13 @@ pub async fn reset_password( return Err(AppError::bad_request("invalid password reset token")); } - // Every currently-active session is about to be bulk-revoked below (by - // entity_id, not by session_id) — enumerated *inside* the transaction, - // immediately after the entity lock above: `create_session` takes the - // same lock before inserting, so a session created concurrently for this - // entity either committed before this transaction acquired the lock (and - // is therefore visible here) or is blocked until this transaction - // finishes. Enumerating via a plain pre-transaction pool query — the - // previous shape of this code — could miss a session created in that - // window, leaving its cache entry uninvalidated indefinitely. See - // `src/cache/mod.rs`'s consistency model. + // Every active session is about to be bulk-revoked below — enumerated + // *inside* the transaction, right after the entity lock: `create_session` + // takes that same lock, so a concurrently created session either + // committed before we acquired it (visible here) or is blocked until we + // finish. A pre-transaction pool query could miss one, leaving its cache + // entry uninvalidated indefinitely. See `src/cache/mod.rs`'s consistency + // model. let session_keys: Vec = sqlx::query_scalar::<_, Uuid>( "SELECT id FROM sessions WHERE entity_id = $1 AND revoked_at IS NULL", ) From 8beb0a5762f6592c8ca217fb7dcf8f8272a6e8d2 Mon Sep 17 00:00:00 2001 From: dusan Date: Fri, 31 Jul 2026 12:12:06 +0200 Subject: [PATCH 08/19] Fix cache poisoning and under-invalidation in the auth/authz cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-tenant tenant_status poisoning: the JWT miss path built the tenant_status key from the token's `tid` claim but populated it with the payload of the entity's *current* tenant, since the miss loader joins tenants through entities.tenant_id. A token outliving a tenant move wrote one tenant's status under another tenant's key, and because the populate ran before check_session_entity_tenant, even the request rejected for the tid mismatch poisoned it — marking a frozen tenant active for all its members until the TTL elapsed. Now populates only when the observed version belongs to the key the payload describes. The API-key path had the same shape in its credential-hit/entity-miss branch, keying entity_status off the cached credential's entity_id while taking the payload from the fresh row. REST handlers were left behind when the GraphQL resolvers were wired to the cache: delete_entity invalidated only entity_status, so a later restoreEntity resurrected sessions and access tokens that Postgres had revoked and restore deliberately never reinstates; add/remove_group_member did no grants invalidation at all, leaving a removed member exercising the group's roles until the grants TTL. Entity deletion now has one entry point, identity::service::delete_entity, shared by both callers. Drop the deleted_at fields from the entity/tenant cache entries. Both miss loaders filter deleted_at IS NULL, so the cached copy was always None by construction and no check ever read it — a dead copy of a security predicate that implied a tombstone check existed where none did. Guardrail validators now take &mut PgConnection and run on the caller's transaction. The *_in_tx refactor left them reading through the pool while the resolver held an open transaction with FOR UPDATE locks: the reads could not see the locked state they were meant to validate against, and acquiring a second connection per in-flight mutation deadlocks the pool under concurrency. Also: the barrier end script re-applies PEXPIRE, since HINCRBY recreates an already-expired key and left it immortal; barrier_ttl saturates and ATOM_CACHE_TTL_* is bounded at startup rather than panicking Duration multiplication on the mutation path; remove_group_member takes the group row lock its sibling takes, which the closure-locking helper's doc already claimed as an invariant. CI flushes Redis with the database recreate between test binaries. m25 caches under keys derived from the fixed seeded admin id, so the admin's grant expansion outlived the database it described and authorized against a tenant graph that no longer existed. --- .github/workflows/rust.yml | 19 +++-- src/auth.rs | 54 ++++++++----- src/authz/repo.rs | 54 +++++++------ src/cache/entries.rs | 14 +++- src/cache/mod.rs | 16 +++- src/config.rs | 20 ++++- src/graphql/entities.rs | 79 ++----------------- src/graphql/policies.rs | 2 - src/guardrails.rs | 132 ++++++++++++++++---------------- src/identity/handlers.rs | 25 ++++-- src/identity/repo.rs | 27 +++++-- src/identity/service.rs | 88 +++++++++++++++++++++ src/tenants/repo.rs | 7 +- tests/common/mod.rs | 11 ++- tests/m25_cache_invalidation.rs | 112 +++++++++++++++++++++++++++ 15 files changed, 444 insertions(+), 216 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8ad040f..d4f998d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,10 +37,12 @@ jobs: # Cache-gated tests (src/cache/mod.rs's unit tests, tests/m25_cache_invalidation.rs) # are #[ignore]d specifically so they're skippable without Redis, but # `--include-ignored` below runs them anyway — they need a reachable - # Redis, unlike the Postgres-gated tests which need their own fresh - # database per binary, these use randomly-suffixed keys and don't share - # mutable state across tests/binaries, so one shared instance for the - # whole job is sufficient. + # Redis. One instance serves the whole job, but it is flushed between + # binaries alongside the database recreate: m25 caches under keys derived + # from the *fixed* seeded admin id, so without a flush the admin's grant + # expansion outlives the database it was derived from (and outlives a job + # re-run), and a later binary authorizes against a tenant/role graph that + # no longer exists. redis: image: redis:7-alpine ports: @@ -86,13 +88,18 @@ jobs: ATOM_TEST_REDIS_URL: redis://localhost:6379/0 run: | set -euo pipefail - command -v psql >/dev/null || { - sudo apt-get update && sudo apt-get install -y postgresql-client + command -v psql >/dev/null && command -v redis-cli >/dev/null || { + sudo apt-get update && sudo apt-get install -y postgresql-client redis-tools } psql "$MAINT_URL" -c "SELECT 1" >/dev/null + redis-cli -h localhost -p 6379 PING >/dev/null run_one() { psql "$MAINT_URL" -c "DROP DATABASE IF EXISTS atom_test;" >/dev/null psql "$MAINT_URL" -c "CREATE DATABASE atom_test;" >/dev/null + # Redis must be reset with the database, not just alongside it: + # cached entries keyed off fixed ids (the seeded admin) would + # otherwise describe the database this just dropped. + redis-cli -h localhost -p 6379 FLUSHALL >/dev/null DATABASE_URL="postgres://atom:atom@localhost:5432/atom_test" \ cargo test "$@" -- --include-ignored --test-threads=1 } diff --git a/src/auth.rs b/src/auth.rs index 6e39a23..727fae0 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -379,22 +379,33 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result Result Result Result Result Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; replace_role_permission_block_links_in_tx( - pool, &mut tx, events_enabled, actor_id, @@ -1244,8 +1245,12 @@ pub async fn replace_role_permission_block_links_with_audit( /// [`create_role_assignment_in_tx`]. The resolver must already hold the role /// lock via [`lock_role_and_collect_grants_keys`] on this `tx` — `lock_role` /// below just re-acquires it (same-transaction no-op). +/// +/// Every read runs on `tx`, never on a pooled connection: the validation +/// below is only meaningful under the role lock this transaction holds, and a +/// second connection acquired mid-transaction is a pool-exhaustion deadlock +/// under concurrency. pub(crate) async fn replace_role_permission_block_links_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, @@ -1255,7 +1260,7 @@ pub(crate) async fn replace_role_permission_block_links_in_tx( let role_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(role_id) - .fetch_optional(pool) + .fetch_optional(&mut **tx) .await .map_err(db_err)? .ok_or_else(|| AppError::not_found(format!("role {role_id} not found")))?; @@ -1273,7 +1278,7 @@ pub(crate) async fn replace_role_permission_block_links_in_tx( ) .bind(&unique_block_ids) .bind(role_tenant_id) - .fetch_one(pool) + .fetch_one(&mut **tx) .await .map_err(db_err)?; if count != unique_block_ids.len() as i64 { @@ -3048,7 +3053,9 @@ pub async fn add_role_capability( let scope_ref = role.tenant_id.map(|tenant_id| tenant_id.to_string()); validate_capabilities_against_role_scope(pool, &scope_kind, scope_ref.as_deref(), &[cap_id]) .await?; - crate::guardrails::validate_role_capability(pool, role_id, cap_id).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; + crate::guardrails::validate_role_capability(&mut conn, role_id, cap_id).await?; + drop(conn); let mut tx = pool.begin().await.map_err(db_err)?; lock_role(&mut tx, role_id).await?; insert_role_capability_as_permission_block( @@ -3926,7 +3933,9 @@ pub async fn create_policy( pool: &PgPool, req: CreatePolicyBinding, ) -> Result { - crate::guardrails::validate_policy(pool, &req).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; + crate::guardrails::validate_policy(&mut conn, &req).await?; + drop(conn); let id = Uuid::new_v4(); let membership_tenant_id = req.tenant_id; let membership_entity_id = req.subject_id; @@ -4128,13 +4137,12 @@ pub async fn create_role_assignment( /// can tell a real state change from an idempotent no-op and decide whether the /// operation is worth publishing as a domain event. pub(crate) async fn create_role_assignment_if_missing_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, req: &CreateRoleAssignment, ) -> Result { lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; lock_role(tx, req.role_id).await?; - validate_role_assignment_in_tx(pool, tx, req).await?; + validate_role_assignment_in_tx(tx, req).await?; let inserted = sqlx::query( r#"INSERT INTO role_assignments (tenant_id, subject_kind, subject_id, role_id) @@ -4302,23 +4310,22 @@ pub async fn create_direct_policy_with_audit( req: CreateDirectPolicy, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; - let policy = - create_direct_policy_in_tx(pool, &mut tx, events_enabled, actor_id, req).await?; + let policy = create_direct_policy_in_tx(&mut tx, events_enabled, actor_id, req).await?; tx.commit().await.map_err(db_err)?; Ok(policy) } /// Body of [`create_direct_policy`]; caller contract per -/// [`create_role_assignment_in_tx`]. +/// [`create_role_assignment_in_tx`]. Validates on `tx` rather than a pooled +/// connection — see [`replace_role_permission_block_links_in_tx`]. pub(crate) async fn create_direct_policy_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, req: CreateDirectPolicy, ) -> Result { - validate_direct_policy(pool, &req).await?; - crate::guardrails::validate_direct_policy(pool, &req).await?; + validate_direct_policy_in_tx(tx, &req).await?; + crate::guardrails::validate_direct_policy(tx, &req).await?; lock_live_subject(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; let block_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM permission_blocks WHERE id = $1 FOR UPDATE") @@ -4508,8 +4515,9 @@ async fn validate_role_assignment( )); } validate_subject_boundary(pool, req.tenant_id, &req.subject_kind, req.subject_id).await?; + let mut conn = pool.acquire().await.map_err(db_err)?; crate::guardrails::validate_role_assignment( - pool, + &mut conn, req.tenant_id, req.subject_kind.clone(), req.subject_id, @@ -4519,7 +4527,6 @@ async fn validate_role_assignment( } async fn validate_role_assignment_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, req: &CreateRoleAssignment, ) -> Result<(), AppError> { @@ -4537,7 +4544,7 @@ async fn validate_role_assignment_in_tx( } validate_subject_boundary_in_tx(tx, req.tenant_id, &req.subject_kind, req.subject_id).await?; crate::guardrails::validate_role_assignment( - pool, + tx, req.tenant_id, req.subject_kind.clone(), req.subject_id, @@ -4546,11 +4553,14 @@ async fn validate_role_assignment_in_tx( .await } -async fn validate_direct_policy(pool: &PgPool, req: &CreateDirectPolicy) -> Result<(), AppError> { +async fn validate_direct_policy_in_tx( + tx: &mut Transaction<'_, Postgres>, + req: &CreateDirectPolicy, +) -> Result<(), AppError> { let block_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM permission_blocks WHERE id = $1") .bind(req.permission_block_id) - .fetch_optional(pool) + .fetch_optional(&mut **tx) .await .map_err(db_err)? .ok_or_else(|| { @@ -4561,7 +4571,7 @@ async fn validate_direct_policy(pool: &PgPool, req: &CreateDirectPolicy) -> Resu "direct policy tenantId must match permission block tenantId", )); } - validate_subject_boundary(pool, req.tenant_id, &req.subject_kind, req.subject_id).await + validate_subject_boundary_in_tx(tx, req.tenant_id, &req.subject_kind, req.subject_id).await } async fn validate_subject_boundary_in_tx( diff --git a/src/cache/entries.rs b/src/cache/entries.rs index 0860ec1..f86aa07 100644 --- a/src/cache/entries.rs +++ b/src/cache/entries.rs @@ -22,17 +22,27 @@ pub struct SessionCacheEntry { /// Shared between JWT and API-key authentication — one entity deactivation /// invalidates both paths' view of the entity at once. +/// +/// Deliberately has no `deleted_at` field. Both miss loaders +/// (`auth::load_session_entity_tenant`, `auth::load_credential_row`) filter +/// `e.deleted_at IS NULL`, so an entry can only ever be populated from a +/// live row — a cached `deleted_at` would be `None` by construction and any +/// check against it a no-op. Denying a *subsequently* soft-deleted entity is +/// the soft-delete path's invalidation duty (`entity_status`, plus the +/// entity's sessions and credentials), not this entry's. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EntityStatusCacheEntry { pub status: EntityStatus, - pub deleted_at: Option>, pub tenant_id: Option, } +/// Has no `deleted_at` field, for the same reason as +/// [`EntityStatusCacheEntry`]: both miss loaders filter `t.deleted_at IS +/// NULL`, and denying a later-deleted tenant is the tenant delete path's +/// invalidation duty. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TenantStatusCacheEntry { pub status: TenantStatus, - pub deleted_at: Option>, } /// Never carries the plaintext API-key secret — only what diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 2c67eda..1f5c6ce 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -86,10 +86,17 @@ end return 1 "#; +// Re-applies `PEXPIRE` for the same reason `begin` sets it: `HINCRBY` +// recreates a key that has already expired, and a recreated barrier entry +// with no TTL would never be reclaimed. An `end` that lands after its own +// barrier expired (a long mutation, or a bulk invalidation chunking through +// many keys) would otherwise leak an immortal hash per key. const END_SCRIPT_SRC: &str = r#" +local ttl_ms = ARGV[1] for i, key in ipairs(KEYS) do redis.call('HINCRBY', key, 'v', 1) redis.call('HSET', key, 'dirty', '0') + redis.call('PEXPIRE', key, ttl_ms) end return 1 "#; @@ -423,6 +430,7 @@ impl CacheClient { if keys.is_empty() { return; } + let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); for chunk in keys.chunks(BULK_CHUNK_SIZE) { let mut conn = match self.get_conn().await { Ok(conn) => conn, @@ -436,6 +444,7 @@ impl CacheClient { for key in chunk { invocation.key(key); } + invocation.arg(barrier_ttl.as_millis() as i64); let outcome = tokio::time::timeout(self.op_timeout, invocation.invoke_async::(&mut conn)) .await; @@ -457,8 +466,13 @@ impl CacheClient { /// The barrier key's own expiry: long enough to comfortably outlast any /// realistic Postgres mutation + `end` call, so a lost `end` self-heals by /// the whole entry expiring outright rather than staying dirty forever. +/// +/// Saturating rather than `*`: `Duration`'s multiplication panics on +/// overflow, and this runs inside `begin`/`end` — on the mutation path, long +/// after a nonsensically large `ATOM_CACHE_TTL_*` would have been accepted at +/// startup. `cache_from_env` bounds those values, so this is belt-and-braces. fn barrier_ttl(entry_ttl: Duration) -> Duration { - entry_ttl * 5 + entry_ttl.saturating_mul(5) } /// `get_or_load`, but tolerant of caching being disabled entirely — the diff --git a/src/config.rs b/src/config.rs index dd0bdf8..89f5a18 100644 --- a/src/config.rs +++ b/src/config.rs @@ -101,6 +101,10 @@ impl Default for DbPoolConfig { } } +/// Upper bound on any single `ATOM_CACHE_TTL_*` value, enforced by +/// [`cache_from_env`]. +const MAX_CACHE_TTL_SECS: u64 = 24 * 60 * 60; + /// Per-category TTLs, applied to cached entries as a defense-in-depth safety /// net (not the primary invalidation mechanism — see `src/cache/mod.rs`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -825,18 +829,26 @@ fn cache_from_env() -> Result { anyhow::bail!("ATOM_CACHE_OP_TIMEOUT_MS must be greater than zero"); } let ttl = &cfg.ttl; - if [ + let ttls = [ ttl.session_secs, ttl.entity_status_secs, ttl.tenant_status_secs, ttl.credential_secs, ttl.credential_ceiling_secs, ttl.grants_secs, - ] - .contains(&0) - { + ]; + if ttls.contains(&0) { anyhow::bail!("ATOM_CACHE_TTL_* values must all be greater than zero"); } + // Bounded at startup rather than left to fail on the mutation path: + // `cache::barrier_ttl` scales these by 5, and the resulting `Duration` + // has to stay representable. A day is already far beyond any sane + // staleness window for auth/authz state. + if ttls.iter().any(|secs| *secs > MAX_CACHE_TTL_SECS) { + anyhow::bail!( + "ATOM_CACHE_TTL_* values must not exceed {MAX_CACHE_TTL_SECS} seconds (24h)" + ); + } } Ok(cfg) } diff --git a/src/graphql/entities.rs b/src/graphql/entities.rs index 201cbb1..eb1855d 100644 --- a/src/graphql/entities.rs +++ b/src/graphql/entities.rs @@ -370,83 +370,14 @@ impl EntityMutation { ) .await?; } - // `entity_status` invalidation alone is *not* sufficient: this - // also revokes sessions and access-token credentials, which - // `restoreEntity` deliberately never reinstates. Without their - // own invalidation, a stale cached session/credential would - // become a full hit again the moment `restoreEntity` repopulates - // entity_status as active — despite staying revoked in Postgres. - // See `deactivate_entity_and_collect_revocation_ids_in_tx` for - // why the ids are enumerated inside the same locked transaction. - let Some(cache) = state.cache.as_deref() else { - return repo::delete_entity_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), - ) - .await; - }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let (session_ids, credential_ids) = - repo::deactivate_entity_and_collect_revocation_ids_in_tx( - &mut tx, - id, - Some(auth.entity_id), - ) - .await?; - let session_keys: Vec = session_ids - .iter() - .map(|sid| crate::cache::keys::session(*sid)) - .collect(); - let credential_keys: Vec = credential_ids - .iter() - .map(|cid| crate::cache::keys::credential(*cid)) - .collect(); - let entity_status_keys = [crate::cache::keys::entity_status(id)]; - let groups: [(crate::cache::CacheCategory, &[String]); 3] = [ - ( - crate::cache::CacheCategory::EntityStatus, - &entity_status_keys, - ), - (crate::cache::CacheCategory::Session, &session_keys), - (crate::cache::CacheCategory::Credential, &credential_keys), - ]; - crate::cache::invalidate::begin_all(cache, &groups).await?; - let outcome = repo::finish_entity_deletion_in_tx( - &mut tx, + crate::identity::service::delete_entity( + &state.pool, + state.cache.as_deref(), state.config.events.enabled(), - Some(auth.entity_id), id, + Some(auth.entity_id), ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - crate::cache::invalidate::end_all(cache, &groups).await; - outcome?; - // Mirrors `delete_entity_with_audit`'s own post-commit audit - // write (fire-and-forget, after the mutation durably commits) — - // see `audit::commit_with_audit`'s doc comment. Only needed on - // this locked path; the cache-disabled fallback above already - // gets it from `delete_entity_with_audit` itself. - audit::write( - &state.pool, - false, - audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: existing.tenant_id, - target_kind: Some("entity"), - target_id: Some(id), - event: "entity.delete", - outcome: crate::models::enums::AuditOutcome::Allow, - details: serde_json::json!({}), - }, - ) - .await; - Ok(()) + .await } .await; diff --git a/src/graphql/policies.rs b/src/graphql/policies.rs index 3384a53..ab3b75c 100644 --- a/src/graphql/policies.rs +++ b/src/graphql/policies.rs @@ -465,7 +465,6 @@ impl PolicyMutation { .begin(crate::cache::CacheCategory::Grants, &grants_keys) .await?; let outcome = authz_repo::replace_role_permission_block_links_in_tx( - &state.pool, &mut tx, state.config.events.enabled(), Some(auth.entity_id), @@ -1372,7 +1371,6 @@ impl PolicyMutation { .begin(crate::cache::CacheCategory::Grants, &grants_keys) .await?; let outcome = authz_repo::create_direct_policy_in_tx( - &state.pool, &mut tx, state.config.events.enabled(), Some(auth.entity_id), diff --git a/src/guardrails.rs b/src/guardrails.rs index 92f3043..68f3e13 100644 --- a/src/guardrails.rs +++ b/src/guardrails.rs @@ -1,4 +1,14 @@ -use sqlx::PgPool; +//! Assignment guardrails: the rules that decide whether a grant may be +//! created at all, independently of who is asking. +//! +//! Every entry point takes `&mut PgConnection` rather than `&PgPool`. Callers +//! run these validations under the row locks their mutation already holds, so +//! the reads must go through *that* transaction: a second pooled connection +//! neither sees the transaction's uncommitted state nor respects its locks, +//! and acquiring one while holding a transaction risks exhausting the pool +//! (every request holding one connection and waiting for a second). + +use sqlx::PgConnection; use uuid::Uuid; use crate::{ @@ -91,17 +101,20 @@ impl Rule { } } -pub async fn validate_policy(pool: &PgPool, req: &CreatePolicyBinding) -> Result<(), AppError> { - let assignments = assignments_for_policy(pool, req).await?; - validate_assignments(pool, &assignments).await +pub async fn validate_policy( + conn: &mut PgConnection, + req: &CreatePolicyBinding, +) -> Result<(), AppError> { + let assignments = assignments_for_policy(&mut *conn, req).await?; + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_capability( - pool: &PgPool, + conn: &mut PgConnection, role_id: Uuid, capability_id: Uuid, ) -> Result<(), AppError> { - let capability_names = capability_names(pool, &[capability_id]).await?; + let capability_names = capability_names(&mut *conn, &[capability_id]).await?; let rows = sqlx::query( r#"WITH RECURSIVE assigned_groups(edge_id, group_id) AS ( SELECT pb.id, pb.subject_id @@ -124,7 +137,7 @@ pub async fn validate_role_capability( JOIN entities e ON e.id = gm.entity_id"#, ) .bind(role_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; @@ -145,22 +158,22 @@ pub async fn validate_role_capability( })); } - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_assignment( - pool: &PgPool, + conn: &mut PgConnection, tenant_id: Option, subject_kind: SubjectKind, subject_id: Uuid, role_id: Uuid, ) -> Result<(), AppError> { - let entity_kinds = subject_entity_kinds(pool, subject_kind, subject_id).await?; + let entity_kinds = subject_entity_kinds(&mut *conn, subject_kind, subject_id).await?; if entity_kinds.is_empty() { return Ok(()); } - let role_capabilities = role_permission_assignments(pool, &[role_id]).await?; + let role_capabilities = role_permission_assignments(&mut *conn, &[role_id]).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -174,11 +187,11 @@ pub async fn validate_role_assignment( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_composite_role_assignment_plan( - pool: &PgPool, + conn: &mut PgConnection, entity_ids: &[Uuid], child_role_ids: &[Uuid], tenant_id: Option, @@ -197,14 +210,14 @@ pub async fn validate_composite_role_assignment_plan( let entity_kinds = sqlx::query_scalar::<_, String>("SELECT kind FROM entities WHERE id = ANY($1::uuid[])") .bind(&unique_entity_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; if entity_kinds.len() != unique_entity_ids.len() { return Err(AppError::bad_request("invalid member reference")); } - let role_capabilities = role_permission_assignments(pool, &unique_child_role_ids).await?; + let role_capabilities = role_permission_assignments(&mut *conn, &unique_child_role_ids).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -218,11 +231,11 @@ pub async fn validate_composite_role_assignment_plan( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } pub async fn validate_role_assignment_plan( - pool: &PgPool, + conn: &mut PgConnection, entity_ids: &[Uuid], capability_ids: &[Uuid], tenant_id: Option, @@ -243,14 +256,14 @@ pub async fn validate_role_assignment_plan( let entity_kinds = sqlx::query_scalar::<_, String>("SELECT kind FROM entities WHERE id = ANY($1::uuid[])") .bind(&unique_entity_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)?; if entity_kinds.len() != unique_entity_ids.len() { return Err(AppError::bad_request("invalid member reference")); } - let capability_names = capability_names(pool, &unique_capability_ids).await?; + let capability_names = capability_names(&mut *conn, &unique_capability_ids).await?; if capability_names.len() != unique_capability_ids.len() { return Err(AppError::bad_request("invalid capability reference")); } @@ -273,7 +286,7 @@ pub async fn validate_role_assignment_plan( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } /// Takes the caller's connection rather than the pool: every caller runs this @@ -281,7 +294,7 @@ pub async fn validate_role_assignment_plan( /// while holding one deadlocks a saturated pool (and hangs outright at /// `max_connections = 1`). pub async fn validate_group_member( - conn: &mut sqlx::PgConnection, + conn: &mut PgConnection, group_id: Uuid, entity_id: Uuid, ) -> Result<(), AppError> { @@ -340,11 +353,13 @@ pub async fn validate_group_member( } pub async fn validate_direct_policy( - pool: &PgPool, + conn: &mut PgConnection, req: &CreateDirectPolicy, ) -> Result<(), AppError> { - let entity_kinds = subject_entity_kinds(pool, req.subject_kind.clone(), req.subject_id).await?; - let permission_blocks = permission_block_assignments(pool, &[req.permission_block_id]).await?; + let entity_kinds = + subject_entity_kinds(&mut *conn, req.subject_kind.clone(), req.subject_id).await?; + let permission_blocks = + permission_block_assignments(&mut *conn, &[req.permission_block_id]).await?; let assignments = entity_kinds .into_iter() .flat_map(|entity_kind| { @@ -358,12 +373,12 @@ pub async fn validate_direct_policy( }) .collect::>(); - validate_assignments(pool, &assignments).await + validate_assignments(&mut *conn, &assignments).await } /// Takes the caller's connection, not the pool — see [`validate_group_member`]. pub async fn validate_role_permission_block_links( - conn: &mut sqlx::PgConnection, + conn: &mut PgConnection, role_id: Uuid, permission_block_ids: &[Uuid], ) -> Result<(), AppError> { @@ -407,13 +422,14 @@ pub async fn validate_role_permission_block_links( } async fn assignments_for_policy( - pool: &PgPool, + conn: &mut PgConnection, req: &CreatePolicyBinding, ) -> Result, AppError> { - let entity_kinds = subject_entity_kinds(pool, req.subject_kind.clone(), req.subject_id).await?; + let entity_kinds = + subject_entity_kinds(&mut *conn, req.subject_kind.clone(), req.subject_id).await?; let capability_names = match req.grant_kind { - GrantKind::Capability => capability_names(pool, &[req.grant_id]).await?, - GrantKind::Role => role_capability_names(pool, req.grant_id).await?, + GrantKind::Capability => capability_names(&mut *conn, &[req.grant_id]).await?, + GrantKind::Role => role_capability_names(&mut *conn, req.grant_id).await?, }; let (object_kind, object_type) = scope_to_object(req.scope_kind.clone(), req.scope_ref.as_deref()); @@ -436,30 +452,24 @@ async fn assignments_for_policy( .collect()) } -async fn validate_assignments<'e, E>( - executor: E, +async fn validate_assignments( + conn: &mut PgConnection, assignments: &[Assignment], -) -> Result<(), AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +) -> Result<(), AppError> { if assignments.is_empty() { return Ok(()); } - let rules = load_rules(executor).await?; + let rules = load_rules(&mut *conn).await?; decide(assignments, &rules).map_err(AppError::bad_request) } -async fn load_rules<'e, E>(executor: E) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn load_rules(conn: &mut PgConnection) -> Result, AppError> { use sqlx::Row; sqlx::query( r#"SELECT tenant_id, entity_kind, action_name AS capability_name, object_kind, object_type, decision, is_absolute FROM action_assignment_rules"#, ) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() @@ -478,14 +488,14 @@ where } async fn subject_entity_kinds( - pool: &PgPool, + conn: &mut PgConnection, subject_kind: SubjectKind, subject_id: Uuid, ) -> Result, AppError> { match subject_kind { SubjectKind::Entity => sqlx::query_scalar("SELECT kind FROM entities WHERE id = $1") .bind(subject_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err), SubjectKind::Group => sqlx::query_scalar( @@ -502,27 +512,24 @@ async fn subject_entity_kinds( WHERE gm.group_id IN (SELECT group_id FROM subject_groups)"#, ) .bind(subject_id) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err), } } -async fn capability_names<'e, E>(executor: E, ids: &[Uuid]) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn capability_names(conn: &mut PgConnection, ids: &[Uuid]) -> Result, AppError> { sqlx::query_scalar("SELECT name FROM actions WHERE id = ANY($1::uuid[])") .bind(ids) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err) } -async fn role_capability_names<'e, E>(executor: E, role_id: Uuid) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +async fn role_capability_names( + conn: &mut PgConnection, + role_id: Uuid, +) -> Result, AppError> { sqlx::query_scalar( r#"SELECT DISTINCT c.name FROM (SELECT $1::uuid AS role_id) roles @@ -530,7 +537,7 @@ where JOIN actions c ON c.id = rc.capability_id"#, ) .bind(role_id) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err) } @@ -543,7 +550,7 @@ struct RoleCapabilityAssignment { } async fn role_permission_assignments( - pool: &PgPool, + conn: &mut PgConnection, role_ids: &[Uuid], ) -> Result, AppError> { if role_ids.is_empty() { @@ -602,7 +609,7 @@ async fn role_permission_assignments( WHERE rpb.role_id = ANY($1::uuid[])"#, ) .bind(role_ids) - .fetch_all(pool) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() @@ -616,13 +623,10 @@ async fn role_permission_assignments( .collect() } -async fn permission_block_assignments<'e, E>( - executor: E, +async fn permission_block_assignments( + conn: &mut PgConnection, permission_block_ids: &[Uuid], -) -> Result, AppError> -where - E: sqlx::Executor<'e, Database = sqlx::Postgres>, -{ +) -> Result, AppError> { if permission_block_ids.is_empty() { return Ok(Vec::new()); } @@ -678,7 +682,7 @@ where WHERE pb.id = ANY($1::uuid[])"#, ) .bind(permission_block_ids) - .fetch_all(executor) + .fetch_all(&mut *conn) .await .map_err(db_err)? .into_iter() diff --git a/src/identity/handlers.rs b/src/identity/handlers.rs index b889616..d74493d 100644 --- a/src/identity/handlers.rs +++ b/src/identity/handlers.rs @@ -445,11 +445,12 @@ pub async fn delete_entity( ) .await?; } - crate::cache::invalidate::guarded_mutation( + service::delete_entity( + &state.pool, state.cache.as_deref(), - crate::cache::CacheCategory::EntityStatus, - std::slice::from_ref(&crate::cache::keys::entity_status(id)), - || repo::delete_entity(&state.pool, id, Some(auth.entity_id)), + state.config.events.enabled(), + id, + Some(auth.entity_id), ) .await?; Ok(StatusCode::NO_CONTENT) @@ -815,7 +816,13 @@ pub async fn add_group_member( scope_for_tenant(group.tenant_id), ) .await?; - repo::add_group_member(&state.pool, group_id, req.entity_id).await?; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(req.entity_id)), + || repo::add_group_member(&state.pool, group_id, req.entity_id), + ) + .await?; Ok(StatusCode::NO_CONTENT) } @@ -843,7 +850,13 @@ pub async fn remove_group_member( scope_for_tenant(group.tenant_id), ) .await?; - repo::remove_group_member(&state.pool, group_id, entity_id).await?; + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(entity_id)), + || repo::remove_group_member(&state.pool, group_id, entity_id), + ) + .await?; Ok(StatusCode::NO_CONTENT) } diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 4dc0604..3306c67 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -2057,6 +2057,13 @@ pub async fn add_group_member_with_audit( Ok(()) } +/// Takes the group's `principal_groups` row lock before deleting, matching +/// [`add_group_member`]. `authz::repo::lock_group_closures_and_collect_member_ids` +/// documents this lock as the reason a group-subject mutation may enumerate +/// members under its own closure lock and trust the result: without it a +/// removal is not serialized against that enumeration. Unlike `add_group_member` +/// this deliberately does not require the group to be active or live — a +/// membership must stay removable from a suspended or soft-deleted group. pub async fn remove_group_member( pool: &PgPool, group_id: Uuid, @@ -2076,13 +2083,19 @@ pub async fn remove_group_member_with_audit( // Removal stays idempotent: a missing group or a missing membership row is // not an error. But only an actual deletion is a domain event — publishing // `group_member.remove` for a no-op would lie to downstream consumers. - let tenant_id: Option> = sqlx::query_scalar( - "SELECT tenant_id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(group_id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; + // + // Deliberately no `deleted_at IS NULL` filter, unlike `add_group_member` + // — a membership must stay removable from a suspended or soft-deleted + // group. Takes the group's row lock, matching `add_group_member`'s — + // `authz::repo::lock_group_closures_and_collect_member_ids` documents + // this lock as the reason a group-subject mutation may enumerate members + // under its own closure lock and trust the result. + let tenant_id: Option> = + sqlx::query_scalar("SELECT tenant_id FROM principal_groups WHERE id = $1 FOR UPDATE") + .bind(group_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; let Some(tenant_id) = tenant_id else { return Ok(()); }; diff --git a/src/identity/service.rs b/src/identity/service.rs index 503994e..3fcd37f 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -2281,6 +2281,94 @@ fn make_shared_key(cred_id: Uuid) -> String { ) } +/// Soft-deletes an entity with a cache barrier spanning the whole +/// transaction. The one entry point for entity deletion on every caller +/// (REST and GraphQL alike), so no caller can under-invalidate. +/// +/// `entity_status` invalidation alone is *not* sufficient: this also revokes +/// the entity's sessions and access-token credentials, which +/// [`super::repo::restore_entity`] deliberately never reinstates. Without their own +/// invalidation, a stale cached session or credential becomes a full hit +/// again the moment a later restore repopulates `entity_status` as active — +/// despite staying revoked in Postgres. See +/// [`super::repo::deactivate_entity_and_collect_revocation_ids_in_tx`] for why the +/// ids are enumerated inside the same locked transaction. +pub async fn delete_entity( + pool: &PgPool, + cache: Option<&crate::cache::CacheClient>, + events_enabled: bool, + id: Uuid, + deleted_by: Option, +) -> Result<(), AppError> { + let Some(cache) = cache else { + return super::repo::delete_entity_with_audit( + pool, + events_enabled, + deleted_by, + id, + deleted_by, + ) + .await; + }; + + let mut tx = pool.begin().await.map_err(db_err)?; + let (session_ids, credential_ids) = + super::repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by) + .await?; + let session_keys: Vec = session_ids + .iter() + .copied() + .map(crate::cache::keys::session) + .collect(); + let credential_keys: Vec = credential_ids + .iter() + .copied() + .map(crate::cache::keys::credential) + .collect(); + let entity_status_keys = [crate::cache::keys::entity_status(id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 3] = [ + ( + crate::cache::CacheCategory::EntityStatus, + &entity_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + (crate::cache::CacheCategory::Credential, &credential_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = + super::repo::finish_entity_deletion_in_tx(&mut tx, events_enabled, deleted_by, id).await; + let outcome = match outcome { + Ok(()) => tx.commit().await.map_err(db_err), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + outcome?; + // Mirrors `delete_entity_with_audit`'s own post-commit audit write + // (fire-and-forget, after the mutation durably commits) — see + // `audit::commit_with_audit`'s doc comment. Only needed on this locked + // path; the cache-disabled fallback above already gets it from + // `delete_entity_with_audit` itself. + let tenant_id = super::repo::get_entity(pool, id) + .await + .ok() + .and_then(|e| e.tenant_id); + crate::audit::write( + pool, + false, + crate::audit::AuditEvent { + actor_entity_id: deleted_by, + tenant_id, + target_kind: Some("entity"), + target_id: Some(id), + event: "entity.delete", + outcome: crate::models::enums::AuditOutcome::Allow, + details: serde_json::json!({}), + }, + ) + .await; + Ok(()) +} + pub async fn revoke_credential( pool: &PgPool, entity_id: Uuid, diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index f94a484..dffed4f 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -1387,7 +1387,6 @@ pub async fn add_tenant_member_with_audit( let mut role_assigned = false; if let Some(role_id) = role_id { role_assigned = crate::authz::repo::create_role_assignment_if_missing_in_tx( - pool, &mut tx, &CreateRoleAssignment { tenant_id: Some(tenant_id), @@ -1492,7 +1491,7 @@ pub async fn accept_invitation( ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; let role_id = accept_invitation_row(&mut tx, tenant_id, invitee_user_id).await?; - grant_invitation_role(&mut tx, pool, tenant_id, invitee_user_id, role_id).await?; + grant_invitation_role(&mut tx, tenant_id, invitee_user_id, role_id).await?; tx.commit().await.map_err(db_err)?; Ok(()) } @@ -1563,7 +1562,7 @@ pub async fn accept_invitation_token( .await .map_err(db_err)?; - grant_invitation_role(&mut tx, pool, tenant_id, actor_id, role_id).await?; + grant_invitation_role(&mut tx, tenant_id, actor_id, role_id).await?; tx.commit().await.map_err(db_err)?; Ok(tenant_id) } @@ -1599,7 +1598,6 @@ async fn accept_invitation_row( async fn grant_invitation_role( tx: &mut Transaction<'_, Postgres>, - pool: &PgPool, tenant_id: Uuid, invitee_user_id: Uuid, role_id: Option, @@ -1624,7 +1622,6 @@ async fn grant_invitation_role( }; crate::authz::repo::create_role_assignment_if_missing_in_tx( - pool, tx, &CreateRoleAssignment { tenant_id: Some(tenant_id), diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 1e889c6..2b3f245 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -34,9 +34,14 @@ pub async fn pool() -> PgPool { pool } -/// Connect to the test Redis with short TTLs so invalidation-correctness -/// tests aren't waiting on production-sized windows, and a fresh v1 -/// namespace-mate configuration otherwise identical to `CacheConfig::default`. +/// Connect to the test Redis with `CacheConfig::default`'s production TTLs, +/// overriding only `enabled`/`redis_url`. Deliberately *not* shortened: every +/// invalidation-correctness test asserts immediately, with no sleep, so a +/// short TTL could only mask a missing invalidation as a pass. +/// +/// Assumes Redis is flushed between test binaries (see the `run_one` helper +/// in `.github/workflows/rust.yml`) — entries keyed off fixed ids such as the +/// seeded admin's would otherwise outlive the database they describe. pub async fn cache_client() -> CacheClient { let url = std::env::var("ATOM_TEST_REDIS_URL") .expect("ATOM_TEST_REDIS_URL must be set for cache-gated tests"); diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 6e3c64c..32ca268 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -1244,6 +1244,118 @@ async fn api_key_auth_reflects_the_current_tenant_after_an_entity_moves_tenants( ); } +/// Mints a real session row plus a signed JWT carrying `tenant_id` as its +/// `tid` claim, so tests can exercise `auth_from_jwt` end to end. +async fn session_jwt( + state: &AppState, + pool: &PgPool, + entity_id: Uuid, + tenant_id: Option, +) -> String { + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(pool) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + auth::encode_jwt( + entity_id, + session_id, + tenant_id, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt") +} + +/// JWT counterpart to +/// `api_key_auth_reflects_the_current_tenant_after_an_entity_moves_tenants`, +/// and a regression test for a review finding the API-key path did not have: +/// `auth_from_jwt` built the `tenant_status` cache key from the token's `tid` +/// claim but populated it with the payload of the *entity's current* tenant, +/// because the miss loader joins tenants through `entities.tenant_id`. Once a +/// token outlives a tenant move the two differ, so authenticating with the +/// stale token wrote the new tenant's status under the old tenant's key — and +/// since the populate ran *before* `check_session_entity_tenant`, even the +/// request that got rejected for the tid mismatch poisoned it on the way out. +/// A frozen tenant then read back as active for every one of its members +/// until the TTL elapsed. +#[tokio::test] +#[ignore] +async fn a_stale_jwt_from_before_a_tenant_move_cannot_poison_the_old_tenants_status() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let old_tenant = tenant(&p).await; + let new_tenant = tenant(&p).await; + // `mover` leaves `old_tenant` for `new_tenant`; `stayer` remains behind + // and is the one a poisoned `tenant_status:{old_tenant}` would wrongly + // let through. + let mover = active_entity_in_tenant(&p, old_tenant, "service").await; + let stayer = active_entity_in_tenant(&p, old_tenant, "service").await; + + let mover_token = session_jwt(&state, &p, mover, Some(old_tenant)).await; + let stayer_token = session_jwt(&state, &p, stayer, Some(old_tenant)).await; + + // Warm `stayer`'s session and entity_status entries while everything is + // still valid. Freezing the tenant below invalidates `tenant_status` and + // nothing else, so those two stay hits — which is what makes a poisoned + // `tenant_status` entry decisive rather than academic: with all three + // keys hitting, `auth_from_jwt` never consults Postgres at all. + auth::authenticate_token(&state, &stayer_token) + .await + .expect("initial authentication should succeed"); + + let schema = build_schema(state.clone()); + let mv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!( + r#"mutation {{ updateEntity(id: "{mover}", input: {{ tenantId: "{new_tenant}" }}) {{ id }} }}"# + ), + )) + .await; + assert!(mv.errors.is_empty(), "tenant move failed: {:?}", mv.errors); + + let sv = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ freezeTenant(id: "{old_tenant}") {{ id }} }}"#), + )) + .await; + assert!( + sv.errors.is_empty(), + "tenant freeze failed: {:?}", + sv.errors + ); + + // `mover`'s token still claims `old_tenant` while the entity now lives in + // `new_tenant`, so this must be rejected on the tid mismatch. + let result = auth::authenticate_token(&state, &mover_token).await; + assert!( + result.is_err(), + "a JWT whose tid no longer matches the entity's tenant must be rejected" + ); + + // The interesting assertion: that rejected authentication must not have + // left `new_tenant`'s active status cached under `old_tenant`'s key. + let result = auth::authenticate_token(&state, &stayer_token).await; + assert!( + result.is_err(), + "a member of the frozen tenant must still be rejected — the rejected \ + authentication above must not have populated tenant_status for the \ + frozen tenant with the moved entity's new tenant's status" + ); +} + // ─── Group-subject mutation vs. concurrent membership change ─────────────── /// Regression test for a review finding: resolving a group-subject From 32371ea55012d8eef07775f80bdf67ab83cccbdb Mon Sep 17 00:00:00 2001 From: Arvindh Date: Fri, 31 Jul 2026 15:55:34 +0530 Subject: [PATCH 09/19] cache documentation Signed-off-by: Arvindh --- product-docs/14-caching.md | 345 +++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 product-docs/14-caching.md diff --git a/product-docs/14-caching.md b/product-docs/14-caching.md new file mode 100644 index 0000000..e0fc97f --- /dev/null +++ b/product-docs/14-caching.md @@ -0,0 +1,345 @@ +# AuthN / AuthZ Cache + +## Status: Active v1 +## Date: 2026-07-31 + +This document describes Atom's Redis-backed cache for authentication and authorization inputs. The source of truth for overall product requirements is [Atom Product Requirements Document](./PRD.md), and authorization terminology is defined in [Atom access model](./11-access-model-simplification.md). + +--- + +## Goal + +Atom is often used as an external Identity Provider. Every request from every downstream service hits Atom's auth path. Without a cache, each request costs: + +- **JWT auth** — a three-way join (`sessions ⨝ entities ⨝ tenants`). +- **API-key auth** — a three-way join (`credentials ⨝ entities ⨝ tenants`) plus a KDF verification. +- **Every authorization decision** — a recursive CTE call (`subject_effective_grants(uuid)`) that flattens direct policies, role assignments, and group membership. + +That is 3–4 Postgres round trips per request, one of them a recursive CTE. Under IdP-scale load this becomes the bottleneck. + +The cache targets these hot reads specifically, keeping Postgres as the single source of truth. + +--- + +## Design Principles + +1. **Cache inputs, never decisions.** The permit/deny outcome is never cached. Only the raw data the PDP consumes is cached, so a policy change takes effect on the next request without needing to invalidate a combinatorial `(subject, action, object)` space. +2. **Correctness over hit rate.** A revoked token, disabled entity, or removed grant must stop working immediately. Every mutation that could affect a cached value invalidates the affected keys through a race-safe barrier. +3. **Fail-safe reads.** If Redis is unreachable, reads fall through to Postgres. Auth still works — just slower. +4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate the cache would risk serving stale grants after a revoke. +5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. + +--- + +## What Is Cached + +Six categories, each keyed by a UUID under the `atom:v1:` namespace. + +| Category | Key format | Payload shape (DTO) | Answers | +|---|---|---|---| +| `Session` | `atom:v1:session:` | `SessionCacheEntry` | Is this JWT's session alive? | +| `EntityStatus` | `atom:v1:entity_status:` | `EntityStatusCacheEntry` | Is the user active? Which tenant? | +| `TenantStatus` | `atom:v1:tenant_status:` | `TenantStatusCacheEntry` | Is the tenant active? | +| `Credential` | `atom:v1:credential:` | `CredentialCacheEntry` | API-key hash, status, expiry (never plaintext). | +| `CredentialCeiling` | `atom:v1:cred_ceiling:` | `CredentialCeiling` | Scoped-token permission cap. | +| `Grants` | `atom:v1:grants:` | `Vec` | The user's full flattened permission list. | + +DTOs are defined in [`src/cache/entries.rs`](../src/cache/entries.rs). Key builders in [`src/cache/keys.rs`](../src/cache/keys.rs). + +### What is *not* cached + +- **Passwords** — never used on the request path. Used only during `/login`, which mints a JWT; the JWT then uses the `Session` cache. +- **Plaintext API-key secrets** — only the hash used to verify them. See [`CredentialCacheEntry`](../src/cache/entries.rs#L49). +- **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. +- **Audit writes** — always go to Postgres. + +--- + +## How AuthN / AuthZ Use the Cache + +The three layers of a request, and which cache each layer hits: + +```mermaid +flowchart TD + Request[Incoming request
Authorization: Bearer ...] + Request --> Which{Token type?} + + Which -->|JWT| JWT[JWT auth] + Which -->|API key| API[API-key auth] + + JWT --> J1[Session cache] + JWT --> J2[EntityStatus cache] + JWT --> J3[TenantStatus cache] + + API --> A1[Credential cache] + API --> A2[EntityStatus cache] + API --> A3[TenantStatus cache] + API --> A4[CredentialCeiling cache
only if scoped] + + J1 & J2 & J3 & A1 & A2 & A3 & A4 --> Authn[AuthContext built] + + Authn --> Authz{Authorization check
needed?} + Authz -->|Yes| G[Grants cache] + G --> PDP[PDP evaluates
allow / deny] + Authz -->|No| Done[Handler runs] + PDP --> Done +``` + +- JWT auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_jwt`. +- API-key auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_api_key`. +- Grants load: [`src/auth.rs`](../src/auth.rs) `AuthContext::effective_grants` and [`src/authz/engine.rs`](../src/authz/engine.rs) inside `load_decision_context`. + +--- + +## Consistency Model + +Every cached entry is a Redis hash with three fields: + +- `v` — an integer version, bumped on every mutation that could affect the entry. +- `dirty` — `"1"` while a mutation is in flight, absent otherwise. +- `p` — the serialized payload, present only when the entry holds a valid value. + +Three atomic Lua scripts implement a per-key **mutation barrier** that closes three otherwise-unavoidable races. + +### The three primitives + +| Primitive | When it runs | What it does | +|---|---|---| +| `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`. Best-effort. | +| `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | + +### Read path + +```mermaid +sequenceDiagram + participant R as Request + participant C as CacheClient + participant P as Postgres + + R->>C: lookup(key) + alt Hit and not dirty + C-->>R: value + else Miss / dirty / Redis down + C-->>R: (Miss, version=N) + R->>P: load(key) + P-->>R: value + R->>C: try_populate(key, version=N, value) + Note over C: Writes only if version still N
and not dirty + end +``` + +### Write path (mutation) + +```mermaid +sequenceDiagram + participant W as Mutation + participant C as CacheClient + participant P as Postgres + + W->>C: begin(keys) + Note over C: bump v, set dirty=1 + alt Redis unreachable + C-->>W: Err service_unavailable + Note over W: Mutation refused + else OK + C-->>W: Ok + W->>P: UPDATE / DELETE ... + P-->>W: committed + W->>C: end(keys) + Note over C: bump v again, clear dirty + end +``` + +### The three races this prevents + +1. **Read-before-mutation.** A reader observes version `N`, starts loading from Postgres; a mutation runs to completion, bumping to `N+2`. The reader's `try_populate` presents `N` — rejected on version mismatch. +2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). +3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. + +The first primitive that fails self-heals: `begin` failing refuses the mutation; `end` failing leaves the entry dirty until barrier-TTL expiry; `try_populate` failing just leaves the entry as a miss for the next reader to reload. + +--- + +## Invalidation Map + +Which mutation invalidates which category: + +| Mutation | Invalidates | Where | +|---|---|---| +| Logout | `Session` | [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Password reset | `Session` (bulk, per active session) | [`src/identity/service.rs`](../src/identity/service.rs) | +| Entity update | `EntityStatus` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Entity activate / deactivate | `EntityStatus` | [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Entity delete | `EntityStatus` + `Session` + `Credential` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Entity restore | `EntityStatus` | [`src/graphql/entities.rs`](../src/graphql/entities.rs) | +| Tenant update | `TenantStatus` | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant delete | `TenantStatus` + child `Session`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant restore | `TenantStatus` + reactivated `Credential`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant create / purge | `Grants` (of acting subject) | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Credential revoke / rotate | `Credential` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +| Credential scope change | `CredentialCeiling` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs) | +| Role assignment (create / delete) | `Grants` for each affected subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Direct policy (create / delete) | `Grants` for the subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Role permission-block change | `Grants` for every assignee of the role | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | +| Group membership change | `Grants` for every member of the group closure | [`src/graphql/groups.rs`](../src/graphql/groups.rs) | + +### Race-safe enumeration + +Group and role invalidations must enumerate **who is affected** (every entity in a group closure, every assignee of a role). If enumeration happens outside a transaction, a concurrent `add_group_member` can slip a new member in *between* enumeration and mutation — that new member's `grants` key is never invalidated, and a stale entry survives until TTL. + +To close this, the enumeration functions lock the relevant rows `FOR UPDATE` inside the caller's transaction: + +- [`lock_group_closures_and_collect_grants_keys`](../src/authz/repo.rs) — locks every group in the closure of the given roots (id-sorted, so it cannot deadlock against itself), then returns the affected `grants` keys. +- [`lock_role_and_collect_grants_keys`](../src/authz/repo.rs) — locks the role row, then locks the closure of every group assigned to it. + +Callers pair these with `begin_all` / `end_all` (see below) rather than `guarded_mutation`, because the enumeration must happen inside the same open transaction as the mutation itself. + +--- + +## API Surface + +### Reading + +```rust +// Cache-aside with a fallback loader. Callers use this at read sites. +crate::cache::cached_or_load( + cache, // Option<&CacheClient> + CacheCategory::Grants, + &cache::keys::grants(subject_id), + || repo::effective_grants_for_subject(pool, subject_id), +).await? +``` + +Defined at [`src/cache/mod.rs`](../src/cache/mod.rs) `cached_or_load`. Tolerant of `cache: None` — falls through to `loader`. + +### Writing (single category, single call site) + +```rust +crate::cache::invalidate::guarded_mutation( + cache, + CacheCategory::Credential, + std::slice::from_ref(&cache::keys::credential(credential_id)), + || async { /* Postgres mutation */ }, +).await? +``` + +Defined at [`src/cache/invalidate.rs`](../src/cache/invalidate.rs) `guarded_mutation`. + +### Writing (multi-category, single call site) + +```rust +crate::cache::invalidate::guarded_multi_mutation( + cache, + &[ + (CacheCategory::TenantStatus, &tenant_keys), + (CacheCategory::Session, &session_keys), + ], + || async { /* mutation touching both */ }, +).await? +``` + +### Writing (mutation already owns a `Transaction`) + +Use `begin_all` / `end_all` when the enumeration of affected keys must happen inside the same open transaction as the mutation (see the race-safe enumeration section): + +```rust +let mut tx = pool.begin().await?; +let grants_keys = lock_group_closures_and_collect_grants_keys(&mut tx, &[group_id]).await?; +let groups = [(CacheCategory::Grants, grants_keys.as_slice())]; + +cache::invalidate::begin_all(cache, &groups).await?; +let outcome = do_mutation_in_tx(&mut tx).await; +let outcome = match outcome { + Ok(()) => tx.commit().await, + Err(e) => Err(e), +}; +cache::invalidate::end_all(cache, &groups).await; +outcome? +``` + +--- + +## Failure Modes + +| Failure | Behaviour | Impact | +|---|---|---| +| Redis unreachable at startup | `main.rs` decides fail-fast vs degrade (see `ATOM_CACHE_REQUIRED`). | Startup fails or logs a warning. | +| Redis unreachable during read | Treated as `Lookup::Unavailable`. Falls through to Postgres loader. | Auth works, slower. | +| Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | The mutation does not commit. | +| Redis unreachable during `end` | Best-effort — logged, not surfaced. Entry stays dirty until barrier TTL. | Entry reloads on next reader; slight perf hit until then. | +| Redis unreachable during `try_populate` | Best-effort — dropped silently. | Next reader still gets a miss and retries. | +| Corrupt payload | Treated as a miss and the entry is deleted so the next read reloads clean. | One extra Postgres round trip. | + +--- + +## Configuration + +Set via environment variables — see [`.env.example`](../.env.example) for the full list. + +| Variable | Meaning | +|---|---| +| `ATOM_CACHE_ENABLED` | Master switch. `false` disables all caching (call sites pass `cache: None`). | +| `ATOM_CACHE_REDIS_URL` | Redis connection URL. | +| `ATOM_CACHE_POOL_MAX_SIZE` | Max Redis connections. | +| `ATOM_CACHE_CONNECT_TIMEOUT_MS` | Startup PING timeout. | +| `ATOM_CACHE_OP_TIMEOUT_MS` | Per-operation timeout. | +| `ATOM_CACHE_TTL_SESSION_SECS` | TTL for `Session` entries. | +| `ATOM_CACHE_TTL_ENTITY_STATUS_SECS` | TTL for `EntityStatus` entries. | +| `ATOM_CACHE_TTL_TENANT_STATUS_SECS` | TTL for `TenantStatus` entries. | +| `ATOM_CACHE_TTL_CREDENTIAL_SECS` | TTL for `Credential` entries. | +| `ATOM_CACHE_TTL_CREDENTIAL_CEILING_SECS` | TTL for `CredentialCeiling` entries. | +| `ATOM_CACHE_TTL_GRANTS_SECS` | TTL for `Grants` entries. | + +Config struct: [`src/config.rs`](../src/config.rs) `CacheConfig` / `CacheTtlConfig`. + +TTLs are the residual staleness bound if invalidation is missed entirely (e.g. barrier TTL expired before `end` completed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. + +--- + +## Metrics + +All cache operations emit metrics through [`src/metrics.rs`](../src/metrics.rs): + +| Metric | Labels | Values | +|---|---|---| +| `atom_cache_lookup_total` | `category`, `outcome` | `hit`, `miss`, `error` | +| `atom_cache_invalidation_total` | `category`, `outcome` | `ok`, `error` | + +Category labels are fixed enum variants, so cardinality is bounded (six categories × three outcomes for lookups, six × two for invalidations). + +--- + +## Extending — Adding a New Cached Category + +The cache client (Redis pool, Lua scripts, barrier, timeouts, metrics) is fully generic. Only the **registry** of categories is centralised. To add a new one: + +1. Add a variant to `CacheCategory` in [`src/cache/mod.rs`](../src/cache/mod.rs), plus cases in `as_str()` and `ttl()`. +2. Add a `_secs: u64` field to `CacheTtlConfig` in [`src/config.rs`](../src/config.rs), and a matching env var. +3. Add a key builder to [`src/cache/keys.rs`](../src/cache/keys.rs). +4. (Optional) add a DTO to [`src/cache/entries.rs`](../src/cache/entries.rs) — the client is generic over any `Serialize + DeserializeOwned` type, so a bespoke DTO is only needed if the DB row shape is not directly usable. +5. At each read site: call `cached_or_load(cache, CacheCategory::, &keys::(id), || loader)`. +6. At each write site that could affect the entry: wrap the mutation in `guarded_mutation` (or `begin_all` / `end_all` if the mutation owns an open `Transaction`). + +The design is deliberately explicit — the enum is not `Other(String)` — because: + +- Bounded label cardinality keeps metrics well-behaved. +- All TTLs are auditable in one file. +- No handler can silently invent a new cache category that collides with an existing one. + +--- + +## Testing + +- **Barrier semantics (Redis-gated unit tests):** [`src/cache/mod.rs`](../src/cache/mod.rs) `#[cfg(test)] mod tests`. Includes tests for both the read-before-mutation race and the read-during-dirty-window race. +- **End-to-end invalidation matrix:** [`tests/m25_cache_invalidation.rs`](../tests/m25_cache_invalidation.rs). Covers every mutation → invalidation pairing listed above. + +Both suites are `#[ignore]` and require `ATOM_TEST_REDIS_URL`; run with `cargo test -- --include-ignored`. + +--- + +## Related Documents + +- [Atom Product Requirements Document](./PRD.md) +- [Atom access model](./11-access-model-simplification.md) +- [Scoped Access Tokens](./13-access-tokens.md) From 47edd809505413e9577fd14d63588f07d7850f01 Mon Sep 17 00:00:00 2001 From: dusan Date: Fri, 31 Jul 2026 12:35:31 +0200 Subject: [PATCH 10/19] Collapse the copy-pasted cache-barrier blocks and instrument populates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add cache::invalidate::guarded_tx_mutation and route all twelve locked-transaction invalidation sites through it. Each had its own hand- written copy of the same sequence — open transaction, lock and enumerate, begin barrier, mutate, commit, end barrier — so each independently re-derived the ordering that makes the barrier correct, and each was its own opportunity to drop the end call on an error path. Stable Rust has no async closures, so the helper takes closures returning a boxed future borrowing the transaction; call sites spell that as |tx| Box::pin(async move { .. }). Fix a call site missed by the previous commit: create_role_assignment_in_tx still called the pool variant of validate_role_assignment while holding lock_live_subject and lock_role on its own transaction. That is the same defect the previous commit fixed elsewhere — the validation could not see the state it was locking, and the second connection risked exhausting the pool. With it routed through the transaction, the pool-based validate_role_assignment and validate_subject_boundary are unreachable and are removed. Record post-miss cache writes as atom_cache_populate_total, labelled applied|stale|error. try_populate was the one cache operation with no metric, and it discarded the script's return value, so a hit rate stuck at zero could not be told apart from every write being rejected by a stuck barrier. Read the auth path's independent keys in one pipelined round trip. lookup_many issues them on a single pooled connection and decode parses each into its own type; a full-hit JWT authentication was three pool acquisitions and three serial round trips, each bounded by op_timeout, before any request work started. begin and end likewise hold one connection across all their chunks instead of re-acquiring per 500 keys while already holding Postgres row locks. Stop cloning the signing key on every refreshSession. LoadedKey holds the raw PKCS8 private-key PEM, and the clone existed only to escape the ActiveKeys read guard; JwtSigner carries the parsed key that signing needs anyway, so the guard is released without duplicating key material. Remove guarded_multi_mutation, which was never called: both multi-category sites hold an open transaction and use begin_all/end_all instead. --- src/auth.rs | 96 ++++++++++-- src/authz/repo.rs | 95 +----------- src/cache/invalidate.rs | 96 ++++++------ src/cache/mod.rs | 199 ++++++++++++++++++------ src/graphql/auth.rs | 14 +- src/graphql/groups.rs | 280 +++++++++++++++++----------------- src/graphql/policies.rs | 329 +++++++++++++++++++++------------------- src/identity/service.rs | 6 +- src/metrics.rs | 16 +- 9 files changed, 626 insertions(+), 505 deletions(-) diff --git a/src/auth.rs b/src/auth.rs index 727fae0..4f15ecc 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -78,6 +78,28 @@ pub struct RequireManage(pub AuthContext); // ─── JWT ────────────────────────────────────────────────────────────────────── +/// The parsed form of a primary key, ready to sign with. Derived from a +/// [`LoadedKey`] via [`JwtSigner::from_key`]. +/// +/// Exists so a caller holding the `ActiveKeys` read guard can take what it +/// needs and release the guard before doing I/O, without cloning `LoadedKey` +/// — which would duplicate the raw PKCS8 private-key PEM on the heap for the +/// duration. This holds only the parsed key that signing requires anyway. +pub struct JwtSigner { + kid: String, + key: EncodingKey, +} + +impl JwtSigner { + pub fn from_key(primary: &LoadedKey) -> Result { + Ok(Self { + kid: primary.kid.clone(), + key: EncodingKey::from_ec_pem(primary.private_key_pem.as_bytes()) + .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}")))?, + }) + } +} + pub fn encode_jwt( entity_id: Uuid, session_id: Uuid, @@ -86,10 +108,30 @@ pub fn encode_jwt( expiry_secs: u64, issuer: &str, audience: &str, +) -> Result { + encode_jwt_with( + entity_id, + session_id, + tenant_id, + &JwtSigner::from_key(primary)?, + expiry_secs, + issuer, + audience, + ) +} + +pub fn encode_jwt_with( + entity_id: Uuid, + session_id: Uuid, + tenant_id: Option, + signer: &JwtSigner, + expiry_secs: u64, + issuer: &str, + audience: &str, ) -> Result { let header = Header { alg: Algorithm::ES256, - kid: Some(primary.kid.clone()), + kid: Some(signer.kid.clone()), ..Header::default() }; @@ -104,10 +146,7 @@ pub fn encode_jwt( exp: now + expiry_secs as usize, }; - let encoding_key = EncodingKey::from_ec_pem(primary.private_key_pem.as_bytes()) - .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}")))?; - - encode(&header, &claims, &encoding_key) + encode(&header, &claims, &signer.key) .map_err(|e| AppError::Internal(anyhow::anyhow!("encode jwt: {e}"))) } @@ -325,19 +364,32 @@ async fn auth_from_jwt(state: &AppState, token: &str) -> Result = vec![&session_key, &entity_key]; + if let Some(key) = &tenant_key { + batch_keys.push(key); + } + let mut raw = cache.lookup_many(&batch_keys).await.into_iter(); + let session_raw = raw.next().unwrap_or_default(); + let entity_raw = raw.next().unwrap_or_default(); + let tenant_raw = raw.next(); + let session_lookup = cache - .lookup::(CacheCategory::Session, &session_key) + .decode::(CacheCategory::Session, &session_key, session_raw) .await; let entity_lookup = cache - .lookup::(CacheCategory::EntityStatus, &entity_key) + .decode::(CacheCategory::EntityStatus, &entity_key, entity_raw) .await; - let tenant_lookup = match &tenant_key { - Some(key) => Some( + let tenant_lookup = match (&tenant_key, tenant_raw) { + (Some(key), Some(raw)) => Some( cache - .lookup::(CacheCategory::TenantStatus, key) + .decode::(CacheCategory::TenantStatus, key, raw) .await, ), - None => None, + _ => None, }; // Full cache hit across every key this token needs: validate entirely in @@ -761,9 +813,20 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result = vec![&entity_key]; + if let Some(key) = &tenant_key { + batch_keys.push(key); + } + let mut raw = cache.lookup_many(&batch_keys).await.into_iter(); + let entity_raw = raw.next().unwrap_or_default(); + let tenant_raw = raw.next(); + if let Lookup::Miss { version } = cache - .lookup::(CacheCategory::EntityStatus, &entity_key) + .decode::(CacheCategory::EntityStatus, &entity_key, entity_raw) .await { let entry = EntityStatusCacheEntry { @@ -774,17 +837,18 @@ async fn auth_from_api_key(state: &AppState, key: &str) -> Result(CacheCategory::TenantStatus, &tenant_key) + .decode::(CacheCategory::TenantStatus, tenant_key, tenant_raw) .await { let entry = TenantStatusCacheEntry { status: tenant_status.clone(), }; cache - .try_populate(CacheCategory::TenantStatus, &tenant_key, version, &entry) + .try_populate(CacheCategory::TenantStatus, tenant_key, version, &entry) .await; } } diff --git a/src/authz/repo.rs b/src/authz/repo.rs index b0f47fd..d1653d5 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -4073,7 +4073,7 @@ pub async fn create_role_assignment_with_audit( ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; let assignment = - create_role_assignment_in_tx(pool, &mut tx, events_enabled, actor_id, req).await?; + create_role_assignment_in_tx(&mut tx, events_enabled, actor_id, req).await?; tx.commit().await.map_err(db_err)?; Ok(assignment) } @@ -4088,7 +4088,6 @@ pub async fn create_role_assignment_with_audit( /// of function re-acquires (never re-validates) those locks, and the caller /// commits. pub(crate) async fn create_role_assignment_in_tx( - pool: &PgPool, tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, @@ -4098,8 +4097,10 @@ pub(crate) async fn create_role_assignment_in_tx( // Lock the role and validate under the lock so a concurrent block-link // mutation cannot add a prohibited block against stale state: it blocks on // this same lock and re-validates against the assignment we are inserting. + // Validation must run on `tx` for that to hold at all — the pool variant + // would neither see the locked state nor respect the locks. lock_role(tx, req.role_id).await?; - validate_role_assignment(pool, &req).await?; + validate_role_assignment_in_tx(tx, &req).await?; let assignment = sqlx::query_as::<_, RoleAssignment>( r#"INSERT INTO role_assignments (tenant_id, subject_kind, subject_id, role_id) @@ -4498,34 +4499,6 @@ pub async fn delete_direct_policy(pool: &PgPool, id: Uuid) -> Result<(), AppErro delete_direct_policy_with_audit(pool, false, None, id).await } -async fn validate_role_assignment( - pool: &PgPool, - req: &CreateRoleAssignment, -) -> Result<(), AppError> { - let role_tenant_id: Option = - sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") - .bind(req.role_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| AppError::bad_request("role assignment references unknown role"))?; - if role_tenant_id != req.tenant_id { - return Err(AppError::bad_request( - "role assignment tenantId must match role tenantId", - )); - } - validate_subject_boundary(pool, req.tenant_id, &req.subject_kind, req.subject_id).await?; - let mut conn = pool.acquire().await.map_err(db_err)?; - crate::guardrails::validate_role_assignment( - &mut conn, - req.tenant_id, - req.subject_kind.clone(), - req.subject_id, - req.role_id, - ) - .await -} - async fn validate_role_assignment_in_tx( tx: &mut Transaction<'_, Postgres>, req: &CreateRoleAssignment, @@ -4634,66 +4607,6 @@ async fn validate_subject_boundary_in_tx( Ok(()) } -async fn validate_subject_boundary( - pool: &PgPool, - tenant_id: Option, - subject_kind: &SubjectKind, - subject_id: Uuid, -) -> Result<(), AppError> { - match subject_kind { - SubjectKind::Entity => { - let entity_tenant_id: Option = sqlx::query_scalar( - "SELECT tenant_id FROM entities WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(subject_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| AppError::bad_request("assignment references unknown entity"))?; - if let Some(tenant_id) = tenant_id { - let member: bool = sqlx::query_scalar( - r#"SELECT EXISTS ( - SELECT 1 FROM tenant_memberships - WHERE tenant_id = $1 AND entity_id = $2 AND status = 'active' - )"#, - ) - .bind(tenant_id) - .bind(subject_id) - .fetch_one(pool) - .await - .map_err(db_err)?; - if entity_tenant_id != Some(tenant_id) && !member { - return Err(AppError::bad_request( - "tenant assignment subject entity must belong to the tenant", - )); - } - } else if entity_tenant_id.is_some() { - return Err(AppError::bad_request( - "platform assignment cannot target tenant-owned entity", - )); - } - } - SubjectKind::Group => { - let group_tenant_id: Option = sqlx::query_scalar( - "SELECT tenant_id FROM principal_groups WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(subject_id) - .fetch_optional(pool) - .await - .map_err(db_err)? - .ok_or_else(|| { - AppError::bad_request("assignment references unknown principal group") - })?; - if group_tenant_id != tenant_id { - return Err(AppError::bad_request( - "assignment subject principal group must be in the same tenant", - )); - } - } - } - Ok(()) -} - pub async fn subject_role_assignments( pool: &PgPool, params: SubjectRoleAssignmentsQuery, diff --git a/src/cache/invalidate.rs b/src/cache/invalidate.rs index 22cb8ea..cf1dc4d 100644 --- a/src/cache/invalidate.rs +++ b/src/cache/invalidate.rs @@ -9,10 +9,18 @@ //! `affected_subject_ids_for_group`) before calling this; the enumeration //! itself is domain SQL and does not belong here. -use std::future::Future; +use std::{future::Future, pin::Pin}; + +use sqlx::{PgPool, Postgres, Transaction}; use super::{CacheCategory, CacheClient}; -use crate::error::AppError; +use crate::error::{db_err, AppError}; + +/// A boxed future borrowing the transaction it runs on — what +/// [`guarded_tx_mutation`]'s closures return. Stable Rust has no async +/// closures, so a closure that awaits while holding `&mut Transaction` has to +/// spell the borrow out by hand; call sites write `|tx| Box::pin(f(tx, ..))`. +pub type TxFuture<'a, T> = Pin> + Send + 'a>>; /// Runs `mutate` guarded by a cache barrier on `keys`. With caching disabled /// (`cache: None`) this is a pure passthrough to `mutate`, byte-identical to @@ -42,56 +50,58 @@ where result } -/// Like [`guarded_mutation`], for a mutation whose effects span more than one -/// cache category in a single Postgres transaction — e.g. tenant restore, -/// which both flips the tenant's own status and reactivates a set of -/// credentials. Establishes a barrier on every `(category, keys)` group -/// before running `mutate`; if a later group's barrier can't be established, -/// the ones already established are cleared immediately (rather than left to -/// self-heal on their barrier TTL) before the mutation is refused. Every -/// established group is cleared after `mutate` regardless of outcome. -pub async fn guarded_multi_mutation( - cache: Option<&CacheClient>, - groups: &[(CacheCategory, &[String])], - mutate: F, +/// [`guarded_mutation`] for a mutation whose affected keys can only be +/// determined *under the locks the mutation itself takes* — every +/// group-closure and role mutation, where the set of affected `grants` keys +/// is the enumerated member set and a concurrent membership change would +/// otherwise slip past the enumeration (see +/// `authz::repo::lock_group_closures_and_collect_member_ids`). +/// +/// Opens the transaction, runs `collect_keys` on it (which locks and +/// enumerates), establishes the barrier on what it returned, runs `mutate` on +/// the same transaction, commits, and clears the barrier — in that order, +/// regardless of outcome. That ordering is the whole point of the helper: the +/// barrier must be established after the lock (so the enumeration is +/// complete) and cleared after the commit (so no reader repopulates from +/// pre-commit state), and every call site previously re-derived it by hand. +/// +/// Callers hold the `None` cache case themselves, since the uncached +/// fallback is a different repo function per call site rather than the same +/// one — usually the non-`_in_tx` variant that opens its own transaction. +pub async fn guarded_tx_mutation( + cache: &CacheClient, + category: CacheCategory, + pool: &PgPool, + collect_keys: K, + mutate: M, ) -> Result where - F: FnOnce() -> Fut, - Fut: Future>, + K: for<'a> FnOnce(&'a mut Transaction<'static, Postgres>) -> TxFuture<'a, Vec>, + M: for<'a> FnOnce(&'a mut Transaction<'static, Postgres>) -> TxFuture<'a, T>, { - let Some(cache) = cache else { - return mutate().await; + let mut tx = pool.begin().await.map_err(db_err)?; + let keys = collect_keys(&mut tx).await?; + cache.begin(category, &keys).await?; + let outcome = mutate(&mut tx).await; + let outcome = match outcome { + Ok(value) => tx.commit().await.map_err(db_err).map(|_| value), + Err(err) => Err(err), }; - - let mut established = Vec::with_capacity(groups.len()); - for &(category, keys) in groups { - match cache.begin(category, keys).await { - Ok(()) => established.push((category, keys)), - Err(err) => { - for (category, keys) in established { - cache.end(category, keys).await; - } - return Err(err); - } - } - } - - let result = mutate().await; - for (category, keys) in established { - cache.end(category, keys).await; - } - result + cache.end(category, &keys).await; + outcome } /// Establishes a barrier on every `(category, keys)` group, in order. Used by /// callers that already hold an open `Transaction` across their own /// lock/enumerate/mutate/commit sequence (so the barrier can't be established -/// via a `FnOnce` closure the way [`guarded_mutation`]/[`guarded_multi_mutation`] -/// do — a closure can't cleanly borrow `&mut Transaction` across an await -/// point on stable Rust). If a later group's barrier can't be established, the -/// ones already established are cleared immediately rather than left to -/// self-heal on their barrier TTL. Pair with [`end_all`], called -/// unconditionally after the mutation regardless of outcome. +/// via a `FnOnce` closure the way [`guarded_mutation`] does). Prefer +/// [`guarded_tx_mutation`] for the single-category case it covers; `begin_all` +/// remains for mutations spanning several categories on one transaction. +/// +/// If a later group's barrier can't be established, the ones already +/// established are cleared immediately rather than left to self-heal on their +/// barrier TTL. Pair with [`end_all`], called unconditionally after the +/// mutation regardless of outcome. pub async fn begin_all( cache: &CacheClient, groups: &[(CacheCategory, &[String])], diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 1f5c6ce..c18e154 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -166,6 +166,44 @@ pub enum Lookup { Unavailable, } +/// One key's unparsed entry from [`CacheClient::lookup_many`]. Opaque by +/// design — [`CacheClient::decode`] is the only way to read it, so the +/// dirty-bit and version handling can't be reimplemented per call site. +#[derive(Debug)] +pub struct RawLookup { + /// `None` when the read itself failed, which `decode` maps to + /// `Lookup::Unavailable`. + version: Option, + /// `None` for an absent *or* dirty entry — both are misses, and a dirty + /// entry's payload must never be served. + payload: Option>, +} + +impl Default for RawLookup { + /// An unavailable entry — the safe reading of a result that never arrived. + fn default() -> Self { + Self::unavailable() + } +} + +impl RawLookup { + fn unavailable() -> Self { + Self { + version: None, + payload: None, + } + } + + fn from_fields(fields: (Option, Option, Option>)) -> Self { + let (version, dirty, payload) = fields; + let is_dirty = dirty.as_deref() == Some("1"); + Self { + version: Some(version.unwrap_or(0)), + payload: payload.filter(|_| !is_dirty), + } + } +} + #[derive(Debug, Error)] pub enum CacheError { #[error("cache operation timed out")] @@ -241,50 +279,90 @@ impl CacheClient { category: CacheCategory, key: &str, ) -> Lookup { + let raw = self + .lookup_many(std::slice::from_ref(&key)) + .await + .pop() + .unwrap_or_else(RawLookup::unavailable); + self.decode(category, key, raw).await + } + + /// Reads every key in one pipelined round trip on a single pooled + /// connection, returning one [`RawLookup`] per key, in order. + /// + /// The auth hot path reads up to three keys before any request work + /// starts; issued one at a time that is three pool acquisitions and three + /// serial round trips, each bounded by `op_timeout`. Use this wherever the + /// keys have no data dependency on each other, then [`Self::decode`] each + /// result into its own type. Metrics are recorded by `decode`, not here, + /// so one batch can span several categories. + /// + /// Always returns exactly `keys.len()` entries; a transport failure yields + /// unavailable entries rather than a short vector. + pub async fn lookup_many(&self, keys: &[&str]) -> Vec { + if keys.is_empty() { + return Vec::new(); + } + let unavailable = || keys.iter().map(|_| RawLookup::unavailable()).collect(); + let mut conn = match self.get_conn().await { Ok(conn) => conn, Err(err) => { - tracing::warn!(category = category.as_str(), error = %err, "cache lookup unavailable"); - metrics::record_cache_lookup(category.as_str(), "error"); - return Lookup::Unavailable; + tracing::warn!(error = %err, "cache lookup unavailable"); + return unavailable(); } }; + let mut pipe = redis::pipe(); + for key in keys { + pipe.cmd("HMGET").arg(*key).arg("v").arg("dirty").arg("p"); + } let result = tokio::time::timeout( self.op_timeout, - redis::cmd("HMGET") - .arg(key) - .arg("v") - .arg("dirty") - .arg("p") - .query_async::<(Option, Option, Option>)>(&mut conn), + pipe.query_async::, Option, Option>)>>(&mut conn), ) .await; - let (version, dirty, payload) = match result { - Ok(Ok(fields)) => fields, + match result { + Ok(Ok(rows)) if rows.len() == keys.len() => { + rows.into_iter().map(RawLookup::from_fields).collect() + } + Ok(Ok(rows)) => { + tracing::warn!( + expected = keys.len(), + got = rows.len(), + "cache pipelined lookup returned an unexpected row count" + ); + unavailable() + } Ok(Err(err)) => { - tracing::warn!(category = category.as_str(), error = %err, "cache lookup failed"); - metrics::record_cache_lookup(category.as_str(), "error"); - return Lookup::Unavailable; + tracing::warn!(error = %err, "cache lookup failed"); + unavailable() } Err(_) => { - tracing::warn!(category = category.as_str(), "cache lookup timed out"); - metrics::record_cache_lookup(category.as_str(), "error"); - return Lookup::Unavailable; + tracing::warn!("cache lookup timed out"); + unavailable() } - }; - - let observed_version = version.unwrap_or(0); - let is_dirty = dirty.as_deref() == Some("1"); + } + } - let Some(payload) = payload.filter(|_| !is_dirty) else { + /// Parses one [`lookup_many`](Self::lookup_many) result into a typed + /// [`Lookup`], recording the read in the per-category metrics. A corrupt + /// payload is deleted and reported as a miss, exactly as in `lookup`. + pub async fn decode( + &self, + category: CacheCategory, + key: &str, + raw: RawLookup, + ) -> Lookup { + let Some(version) = raw.version else { + metrics::record_cache_lookup(category.as_str(), "error"); + return Lookup::Unavailable; + }; + let Some(payload) = raw.payload else { metrics::record_cache_lookup(category.as_str(), "miss"); - return Lookup::Miss { - version: observed_version, - }; + return Lookup::Miss { version }; }; - match serde_json::from_slice::(&payload) { Ok(value) => { metrics::record_cache_lookup(category.as_str(), "hit"); @@ -294,9 +372,7 @@ impl CacheClient { tracing::warn!(category = category.as_str(), error = %err, "cache payload corrupt; discarding"); self.best_effort_delete(key).await; metrics::record_cache_lookup(category.as_str(), "miss"); - Lookup::Miss { - version: observed_version, - } + Lookup::Miss { version } } } } @@ -322,11 +398,15 @@ impl CacheClient { category = category.as_str(), "cache payload serialize failed" ); + metrics::record_cache_populate(category.as_str(), "error"); return; }; let mut conn = match self.get_conn().await { Ok(conn) => conn, - Err(_) => return, + Err(_) => { + metrics::record_cache_populate(category.as_str(), "error"); + return; + } }; let ttl = category.ttl(&self.ttl); let outcome = tokio::time::timeout( @@ -339,10 +419,27 @@ impl CacheClient { .invoke_async::(&mut conn), ) .await; - if let Err(err) = outcome { - tracing::warn!(category = category.as_str(), error = %err, "cache populate timed out"); - } else if let Ok(Err(err)) = outcome { - tracing::warn!(category = category.as_str(), error = %err, "cache populate failed"); + // The script distinguishes `applied` from `stale` (barrier dirty, or + // the version moved since the caller's `lookup`). Both are normal, but + // only the split tells an operator whether a zero hit rate means "cold" + // or "every write is being rejected" — so it is recorded, not dropped. + match outcome { + Ok(Ok(result)) => { + let outcome = if result == "applied" { + "applied" + } else { + "stale" + }; + metrics::record_cache_populate(category.as_str(), outcome); + } + Ok(Err(err)) => { + tracing::warn!(category = category.as_str(), error = %err, "cache populate failed"); + metrics::record_cache_populate(category.as_str(), "error"); + } + Err(_) => { + tracing::warn!(category = category.as_str(), "cache populate timed out"); + metrics::record_cache_populate(category.as_str(), "error"); + } } } @@ -380,15 +477,16 @@ impl CacheClient { return Ok(()); } let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); + // One connection for every chunk, not one per chunk: a bulk + // invalidation re-acquiring from the pool per 500 keys competes with + // the request path for connections while already holding Postgres row + // locks. + let mut conn = self.get_conn().await.map_err(|err| { + tracing::warn!(category = category.as_str(), error = %err, "cache begin: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + AppError::service_unavailable("cache unavailable; refusing security-sensitive mutation") + })?; for chunk in keys.chunks(BULK_CHUNK_SIZE) { - let mut conn = self.get_conn().await.map_err(|err| { - tracing::warn!(category = category.as_str(), error = %err, "cache begin: connection unavailable"); - metrics::record_cache_invalidation(category.as_str(), "error"); - AppError::service_unavailable( - "cache unavailable; refusing security-sensitive mutation", - ) - })?; - let mut invocation = self.begin_script.prepare_invoke(); for key in chunk { invocation.key(key); @@ -431,15 +529,16 @@ impl CacheClient { return; } let barrier_ttl = barrier_ttl(category.ttl(&self.ttl)); + // One connection for every chunk — see `begin`. + let mut conn = match self.get_conn().await { + Ok(conn) => conn, + Err(err) => { + tracing::warn!(category = category.as_str(), error = %err, "cache end: connection unavailable"); + metrics::record_cache_invalidation(category.as_str(), "error"); + return; + } + }; for chunk in keys.chunks(BULK_CHUNK_SIZE) { - let mut conn = match self.get_conn().await { - Ok(conn) => conn, - Err(err) => { - tracing::warn!(category = category.as_str(), error = %err, "cache end: connection unavailable"); - metrics::record_cache_invalidation(category.as_str(), "error"); - continue; - } - }; let mut invocation = self.end_script.prepare_invoke(); for key in chunk { invocation.key(key); diff --git a/src/graphql/auth.rs b/src/graphql/auth.rs index ed1f4f2..1f22335 100644 --- a/src/graphql/auth.rs +++ b/src/graphql/auth.rs @@ -147,9 +147,15 @@ impl AuthMutation { )) })?; let state = ctx.data::()?; - let keys = state.keys.read().await; - let primary_key = keys.primary.clone(); - drop(keys); + // Derive the signer under the read guard and release it immediately: + // cloning `LoadedKey` to escape the guard would put a copy of the raw + // PKCS8 private-key PEM on the heap for every refresh, dropped without + // zeroization. Holding the guard across the mutation instead would + // block key rotation for the length of a DB transaction. + let signer = { + let keys = state.keys.read().await; + crate::auth::JwtSigner::from_key(&keys.primary).map_err(gql_error)? + }; // Extends `expires_at` in place for the same session_id; without // invalidating, a stale cached (shorter) expiry could cause a @@ -163,7 +169,7 @@ impl AuthMutation { service::refresh_session( &state.pool, &state.config, - &primary_key, + &signer, auth.entity_id, session_id, ) diff --git a/src/graphql/groups.rs b/src/graphql/groups.rs index db42ac1..d3a1e8c 100644 --- a/src/graphql/groups.rs +++ b/src/graphql/groups.rs @@ -410,34 +410,33 @@ impl GroupMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::update_group_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - update, - "group.update", - details.clone(), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let details = details.clone(); + Box::pin(async move { + repo::update_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + update, + "group.update", + details, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(value) => tx - .commit() - .await - .map_err(crate::error::db_err) - .map(|_| value), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome + .await } else { repo::update_group_with_audit( &state.pool, @@ -504,28 +503,30 @@ impl GroupMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::set_group_parent_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - parent_id, + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::set_group_parent_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + parent_id, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; repo::get_group(&state.pool, id).await } .await; @@ -579,27 +580,29 @@ impl GroupMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::remove_group_parent_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::remove_group_parent_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; Ok(tenant_id) } .await; @@ -662,28 +665,30 @@ impl GroupMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::delete_group_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::delete_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; Ok(tenant_id) } .await; @@ -731,28 +736,30 @@ impl GroupMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::restore_group_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + repo::restore_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; // Mirrors `restore_group_with_audit`'s own post-commit audit // write (fire-and-forget, after the mutation durably commits) — // see `audit::commit_with_audit`'s doc comment. Only needed on @@ -996,34 +1003,33 @@ impl GroupMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_group_closures_and_collect_grants_keys(&mut tx, &[id]).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = repo::update_group_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - update, - event, - details.clone(), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[id]).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let details = details.clone(); + Box::pin(async move { + repo::update_group_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + update, + event, + details, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(value) => tx - .commit() - .await - .map_err(crate::error::db_err) - .map(|_| value), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome + .await } .await; if let Err(ref err) = result { diff --git a/src/graphql/policies.rs b/src/graphql/policies.rs index ab3b75c..b3bc250 100644 --- a/src/graphql/policies.rs +++ b/src/graphql/policies.rs @@ -458,28 +458,31 @@ impl PolicyMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = - authz_repo::lock_role_and_collect_grants_keys(&mut tx, role_id).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::replace_role_permission_block_links_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - role_id, - &permission_block_ids, + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_role_and_collect_grants_keys(tx, role_id).await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + let permission_block_ids = permission_block_ids.clone(); + Box::pin(async move { + authz_repo::replace_role_permission_block_links_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + role_id, + &permission_block_ids, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; Ok(tenant_id) } .await; @@ -537,27 +540,30 @@ impl PolicyMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = authz_repo::lock_role_and_collect_grants_keys(&mut tx, id).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::delete_role_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin( + async move { authz_repo::lock_role_and_collect_grants_keys(tx, id).await }, + ) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_role_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; Ok(tenant_id) } .await; @@ -604,27 +610,30 @@ impl PolicyMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = authz_repo::lock_role_and_collect_grants_keys(&mut tx, id).await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::restore_role_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - Some(auth.entity_id), + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin( + async move { authz_repo::lock_role_and_collect_grants_keys(tx, id).await }, + ) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::restore_role_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + Some(auth.entity_id), + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; + .await?; // Mirrors `restore_role_with_audit`'s own post-commit audit // write (fire-and-forget, after the mutation durably commits) — // see `audit::commit_with_audit`'s doc comment. Only needed on @@ -1154,7 +1163,6 @@ impl PolicyMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; // Locks the role *before* `subject_id`'s group closure — // matching `replaceRolePermissionBlocks`/`deleteRole`/ // `restoreRole`'s lock order exactly, since locking the @@ -1162,34 +1170,32 @@ impl PolicyMutation { // `create_role_assignment_in_tx`) can deadlock against // those paths. See // `authz::repo::lock_role_then_group_closure_and_collect_grants_keys`. - let grants_keys = - authz_repo::lock_role_then_group_closure_and_collect_grants_keys( - &mut tx, role_id, subject_id, - ) - .await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::create_role_assignment_in_tx( + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, &state.pool, - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - req, + |tx| { + Box::pin(async move { + authz_repo::lock_role_then_group_closure_and_collect_grants_keys( + tx, role_id, subject_id, + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::create_role_assignment_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + req, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(value) => tx - .commit() - .await - .map_err(crate::error::db_err) - .map(|_| value), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome + .await } } } @@ -1258,30 +1264,33 @@ impl PolicyMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( - &mut tx, - &[assignment.subject_id], + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[assignment.subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_role_assignment_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, ) .await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::delete_role_assignment_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; } } Ok(tenant_id) @@ -1361,34 +1370,33 @@ impl PolicyMutation { ) .await; }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( - &mut tx, - &[subject_id], - ) - .await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::create_direct_policy_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - req, + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::create_direct_policy_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + req, + ) + .await + }) + }, ) - .await; - let outcome = match outcome { - Ok(value) => tx - .commit() - .await - .map_err(crate::error::db_err) - .map(|_| value), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome + .await } } } @@ -1457,30 +1465,33 @@ impl PolicyMutation { .await?; return Ok(tenant_id); }; - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let grants_keys = authz_repo::lock_group_closures_and_collect_grants_keys( - &mut tx, - &[policy.subject_id], + crate::cache::invalidate::guarded_tx_mutation( + cache, + crate::cache::CacheCategory::Grants, + &state.pool, + |tx| { + Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys( + tx, + &[policy.subject_id], + ) + .await + }) + }, + |tx| { + let events_enabled = state.config.events.enabled(); + Box::pin(async move { + authz_repo::delete_direct_policy_in_tx( + tx, + events_enabled, + Some(auth.entity_id), + id, + ) + .await + }) + }, ) .await?; - cache - .begin(crate::cache::CacheCategory::Grants, &grants_keys) - .await?; - let outcome = authz_repo::delete_direct_policy_in_tx( - &mut tx, - state.config.events.enabled(), - Some(auth.entity_id), - id, - ) - .await; - let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), - Err(err) => Err(err), - }; - cache - .end(crate::cache::CacheCategory::Grants, &grants_keys) - .await; - outcome?; } } Ok(tenant_id) diff --git a/src/identity/service.rs b/src/identity/service.rs index 3fcd37f..9aa19db 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -1037,7 +1037,7 @@ pub async fn oauth_exchange( pub async fn refresh_session( pool: &PgPool, cfg: &Config, - primary_key: &LoadedKey, + signer: &crate::auth::JwtSigner, entity_id: Uuid, session_id: Uuid, ) -> Result { @@ -1049,11 +1049,11 @@ pub async fn refresh_session( let session = super::repo::refresh_session_in_tx(&mut tx, session_id, entity_id, cfg.jwt_expiry_secs) .await?; - let token = encode_jwt( + let token = crate::auth::encode_jwt_with( entity_id, session.id, tenant_id, - primary_key, + signer, cfg.jwt_expiry_secs, &cfg.jwt_issuer, &cfg.jwt_audience, diff --git a/src/metrics.rs b/src/metrics.rs index 539c0be..1e6019e 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -43,6 +43,11 @@ pub const CACHE_LOOKUP: &str = "atom_cache_lookup_total"; /// Counter of cache invalidation/barrier operations, labelled by `category` /// and `outcome` (ok|error). pub const CACHE_INVALIDATION: &str = "atom_cache_invalidation_total"; +/// Counter of post-miss cache writes, labelled by `category` and `outcome` +/// (applied|stale|error). A hit rate stuck at zero is ambiguous without this: +/// `stale` distinguishes "every write is being rejected because a barrier is +/// stuck dirty or the version keeps moving" from a merely cold cache. +pub const CACHE_POPULATE: &str = "atom_cache_populate_total"; #[cfg(feature = "metrics")] mod backend { @@ -118,6 +123,11 @@ mod backend { metrics::counter!(CACHE_INVALIDATION, "category" => category, "outcome" => outcome) .increment(1); } + + pub fn record_cache_populate(category: &'static str, outcome: &'static str) { + metrics::counter!(CACHE_POPULATE, "category" => category, "outcome" => outcome) + .increment(1); + } } #[cfg(not(feature = "metrics"))] @@ -150,10 +160,12 @@ mod backend { pub fn record_cache_lookup(_category: &'static str, _outcome: &'static str) {} #[inline] pub fn record_cache_invalidation(_category: &'static str, _outcome: &'static str) {} + #[inline] + pub fn record_cache_populate(_category: &'static str, _outcome: &'static str) {} } pub use backend::{ enabled, init, record_audit_db_suppressed, record_audit_failure, record_cache_invalidation, - record_cache_lookup, record_decision, record_outbox_exhausted, record_outbox_publish_failure, - record_rate_limit_rejection, render, + record_cache_lookup, record_cache_populate, record_decision, record_outbox_exhausted, + record_outbox_publish_failure, record_rate_limit_rejection, render, }; From 69171ce4aa58d7b6e624385917e104d104c85cdc Mon Sep 17 00:00:00 2001 From: Arvindh Date: Fri, 31 Jul 2026 16:31:20 +0530 Subject: [PATCH 11/19] update cache documentation Signed-off-by: Arvindh --- docs/content/docs/architecture/caching.mdx | 206 +++++++++++++++++++++ docs/content/docs/architecture/meta.json | 2 +- product-docs/14-caching.md | 55 ++++-- 3 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 docs/content/docs/architecture/caching.mdx diff --git a/docs/content/docs/architecture/caching.mdx b/docs/content/docs/architecture/caching.mdx new file mode 100644 index 0000000..24e1916 --- /dev/null +++ b/docs/content/docs/architecture/caching.mdx @@ -0,0 +1,206 @@ +--- +title: Caching +description: Atom's Redis-backed cache for authentication and authorization inputs — what is cached, how invalidation stays correct, and how it fails safe. +--- + +Atom is designed to sit on the auth path of every downstream service. Without a cache, each request costs three or four Postgres round trips, one of which is a recursive CTE. This page explains the Redis cache that sits in front of those reads: what it stores, when it invalidates, and why it can be trusted with security-sensitive decisions. + +## Goal + +For every request arriving at Atom: + +- **JWT auth** needs a three-way join across `sessions`, `entities`, and `tenants`. +- **API-key auth** needs the same shape across `credentials`, `entities`, and `tenants`, plus a KDF verification. +- **Every authorization decision** needs `subject_effective_grants(uuid)` — a recursive CTE that flattens direct policies, role assignments, and group membership. + +The cache targets these hot reads. Postgres remains the single source of truth; the cache is optional at every call site. + +## Design Principles + +1. **Cache inputs, never decisions.** The permit/deny outcome is never cached, so a policy change takes effect on the next request without touching a combinatorial `(subject, action, object)` space. +2. **Correctness over hit rate.** A revoked token, disabled entity, or removed grant must stop working immediately. Every mutation that could affect a cached value invalidates the affected keys through a race-safe barrier. +3. **Fail-safe reads.** If Redis is unreachable, reads fall through to Postgres. Auth still works — just slower. +4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate would risk serving stale grants after a revoke. +5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. + +## What Is Cached + +Six categories, each keyed by a UUID under the `atom:v1:` namespace. + +| Category | Key format | Answers | +|---|---|---| +| `Session` | `atom:v1:session:` | Is this JWT's session alive? | +| `EntityStatus` | `atom:v1:entity_status:` | Is the user active? Which tenant? | +| `TenantStatus` | `atom:v1:tenant_status:` | Is the tenant active? | +| `Credential` | `atom:v1:credential:` | API-key hash, status, expiry (never plaintext). | +| `CredentialCeiling` | `atom:v1:cred_ceiling:` | Scoped-token permission cap. | +| `Grants` | `atom:v1:grants:` | The user's full flattened permission list. | + +### What is *not* cached + +- **Passwords** — never used on the request path. Used only during login, which mints a JWT; the JWT then uses the `Session` cache. +- **Plaintext API-key secrets** — only the hash used to verify them. +- **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. +- **Audit writes** — always go to Postgres. + +## Request Flow + +Authorization Bearer ...] + Request --> Which{Token type?} + Which -->|JWT| JWT[JWT auth] + Which -->|API key| API[API-key auth] + JWT --> J1[Session cache] + JWT --> J2[EntityStatus cache] + JWT --> J3[TenantStatus cache] + API --> A1[Credential cache] + API --> A2[EntityStatus cache] + API --> A3[TenantStatus cache] + API --> A4[CredentialCeiling cache
only if scoped] + J1 --> Authn[AuthContext built] + J2 --> Authn + J3 --> Authn + A1 --> Authn + A2 --> Authn + A3 --> Authn + A4 --> Authn + Authn --> Authz{Authorization check?} + Authz -->|Yes| G[Grants cache] + G --> PDP[PDP evaluates
allow / deny] + Authz -->|No| Done[Handler runs] + PDP --> Done +`} /> + +The auth hot path reads its independent keys in **one pipelined round trip on a single pooled connection**. Issued one at a time, three lookups would mean three pool acquisitions and three serial round trips, each bounded by the operation timeout, before any request work started. + +## Consistency Model + +Every cached entry is a Redis hash with three fields: + +- `v` — an integer version, bumped on every mutation that could affect the entry. +- `dirty` — `"1"` while a mutation is in flight, absent otherwise. +- `p` — the serialized payload, present only when the entry holds a valid value. + +Three atomic Lua scripts implement a per-key **mutation barrier**. + +| Primitive | When it runs | What it does | +|---|---|---| +| `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, re-applies the entry's expiry. Best-effort. | +| `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | + +### Read path + +>C: lookup(key) + alt Hit and not dirty + C-->>R: value + else Miss / dirty / Redis down + C-->>R: (Miss, version=N) + R->>P: load(key) + P-->>R: value + R->>C: try_populate(key, version=N, value) + Note over C: Writes only if version still N
and not dirty + end +`} /> + +### Write path + +>C: begin(keys) + Note over C: bump v, set dirty=1 + alt Redis unreachable + C-->>W: Err service_unavailable + Note over W: Mutation refused + else OK + C-->>W: Ok + W->>P: UPDATE / DELETE ... + P-->>W: committed + W->>C: end(keys) + Note over C: bump v again, clear dirty + end +`} /> + +### The races this prevents + +1. **Read-before-mutation.** A reader observes version `N`, starts loading from Postgres; a mutation runs to completion, bumping to `N+2`. The reader's `try_populate` presents `N` — rejected on version mismatch. +2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). +3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. +4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — populates only apply when the observed version's key matches the key the payload describes. + +## Invalidation Map + +| Mutation | Invalidates | +|---|---| +| Logout | `Session` | +| Password reset | `Session` (bulk, per active session) | +| Entity update / activate / deactivate / restore | `EntityStatus` | +| Entity delete (REST or GraphQL) | `EntityStatus` + `Session` + `Credential` | +| Tenant update | `TenantStatus` | +| Tenant delete | `TenantStatus` + child `Session`s | +| Tenant restore | `TenantStatus` + reactivated `Credential`s | +| Tenant create / purge | `Grants` (of acting subject) | +| Credential revoke / rotate | `Credential` | +| Credential scope change | `CredentialCeiling` | +| Role assignment (create / delete) | `Grants` for each affected subject | +| Direct policy (create / delete) | `Grants` for the subject | +| Role permission-block change | `Grants` for every assignee of the role | +| Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | + +### Race-safe enumeration + +Group and role invalidations must enumerate *who is affected* — every entity in a group closure, every assignee of a role. If enumeration happens outside a transaction, a concurrent `add_group_member` can slip a new member in between enumeration and mutation. That new member's `grants` key is never invalidated, and a stale entry survives until TTL. + +The enumeration functions therefore lock the relevant rows `FOR UPDATE` inside the caller's transaction, in id-sorted order so they cannot deadlock against themselves. The `guarded_tx_mutation` helper wires that ordering — open transaction, lock and enumerate, `begin` barrier, mutate, commit, `end` barrier — in one place so every call site does it identically. + +## Failure Modes + +| Failure | Behaviour | Impact | +|---|---|---| +| Redis unreachable at startup | Startup fails fast or logs a warning, per config. | Operator-controlled. | +| Redis unreachable during read | Treated as unavailable; falls through to Postgres. | Auth works, slower. | +| Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | Mutation does not commit. | +| Redis unreachable during `end` | Best-effort; entry stays dirty until barrier TTL. | Entry reloads on next reader. | +| Redis unreachable during `try_populate` | Best-effort; dropped silently. | Next reader still gets a miss and retries. | +| Corrupt payload | Treated as a miss; the entry is deleted so the next read reloads clean. | One extra Postgres round trip. | + +## Configuration + +All knobs live under the `ATOM_CACHE_*` prefix: + +- `ATOM_CACHE_ENABLED` — master switch. +- `ATOM_CACHE_REDIS_URL` — connection URL. +- `ATOM_CACHE_POOL_MAX_SIZE` — max Redis connections. +- `ATOM_CACHE_CONNECT_TIMEOUT_MS` — startup PING timeout. +- `ATOM_CACHE_OP_TIMEOUT_MS` — per-operation timeout. +- `ATOM_CACHE_TTL__SECS` — one per category (session, entity_status, tenant_status, credential, credential_ceiling, grants). + +TTLs are the residual staleness bound if invalidation is missed entirely (e.g. `end` never completed and the barrier TTL elapsed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. + +## Metrics + +| Metric | Labels | Values | +|---|---|---| +| `atom_cache_lookup_total` | `category`, `outcome` | `hit`, `miss`, `error` | +| `atom_cache_invalidation_total` | `category`, `outcome` | `ok`, `error` | +| `atom_cache_populate_total` | `category`, `outcome` | `applied`, `stale`, `error` | + +Category labels are fixed enum variants, so cardinality is bounded. + +The populate metric splits `applied` (write took) from `stale` (rejected by the barrier: dirty entry, or the version moved since the caller's `lookup`). A hit rate stuck at zero can then be told apart from every write being rejected by a stuck barrier — otherwise indistinguishable. + +## Extending + +The cache client (Redis pool, Lua scripts, barrier, timeouts, metrics) is fully generic over any serde type. Only the **registry** of categories is centralised in the codebase — one enum, one TTL config struct, one file of key builders — so that: + +- Metrics label cardinality stays bounded. +- All TTLs are auditable in one file. +- No handler can silently invent a new cache category that collides with an existing one. + +Adding a new cached category is a five-step, one-file-at-a-time change: extend the category enum, add a TTL field, add a key builder, wrap read sites in the cache-aside helper, and wrap write sites in the invalidation helpers. diff --git a/docs/content/docs/architecture/meta.json b/docs/content/docs/architecture/meta.json index 6b4b02f..7450eab 100644 --- a/docs/content/docs/architecture/meta.json +++ b/docs/content/docs/architecture/meta.json @@ -1,4 +1,4 @@ { "title": "Architecture", - "pages": ["index", "data-model"] + "pages": ["index", "data-model", "caching"] } diff --git a/product-docs/14-caching.md b/product-docs/14-caching.md index e0fc97f..1b10151 100644 --- a/product-docs/14-caching.md +++ b/product-docs/14-caching.md @@ -89,6 +89,8 @@ flowchart TD - API-key auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_api_key`. - Grants load: [`src/auth.rs`](../src/auth.rs) `AuthContext::effective_grants` and [`src/authz/engine.rs`](../src/authz/engine.rs) inside `load_decision_context`. +The auth hot path reads its independent keys (session/entity/tenant for JWT, credential/entity/tenant for API-key) in **one pipelined round trip on a single pooled connection** via `CacheClient::lookup_many` + `CacheClient::decode` — see [`src/cache/mod.rs`](../src/cache/mod.rs). Issued one at a time it would be three pool acquisitions and three serial round trips, each bounded by `op_timeout`, before any request work started. + --- ## Consistency Model @@ -151,11 +153,12 @@ sequenceDiagram end ``` -### The three races this prevents +### The races this prevents 1. **Read-before-mutation.** A reader observes version `N`, starts loading from Postgres; a mutation runs to completion, bumping to `N+2`. The reader's `try_populate` presents `N` — rejected on version mismatch. 2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). -3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. +3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. `end` re-applies `PEXPIRE` on the entry, since `HINCRBY` would otherwise recreate an already-expired key and leave it immortal. +4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — e.g. the JWT miss loader joins tenants through `entities.tenant_id`, so it returns the entity's *current* tenant's status, which is not necessarily the tenant the token's `tid` claim points to when the token outlived a tenant move. Populates now write only when the observed version's key matches the key the payload describes. The first primitive that fails self-heals: `begin` failing refuses the mutation; `end` failing leaves the entry dirty until barrier-TTL expiry; `try_populate` failing just leaves the entry as a miss for the next reader to reload. @@ -182,7 +185,9 @@ Which mutation invalidates which category: | Role assignment (create / delete) | `Grants` for each affected subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | | Direct policy (create / delete) | `Grants` for the subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | | Role permission-block change | `Grants` for every assignee of the role | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | -| Group membership change | `Grants` for every member of the group closure | [`src/graphql/groups.rs`](../src/graphql/groups.rs) | +| Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | [`src/graphql/groups.rs`](../src/graphql/groups.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | + +The REST paths (`delete_entity`, `add_group_member`, `remove_group_member`) invalidate through the same helpers as the GraphQL resolvers — entity deletion is consolidated into a single service entry point at [`src/identity/service.rs`](../src/identity/service.rs), and both group-member handlers wrap their mutation in `guarded_mutation`. ### Race-safe enumeration @@ -193,7 +198,7 @@ To close this, the enumeration functions lock the relevant rows `FOR UPDATE` ins - [`lock_group_closures_and_collect_grants_keys`](../src/authz/repo.rs) — locks every group in the closure of the given roots (id-sorted, so it cannot deadlock against itself), then returns the affected `grants` keys. - [`lock_role_and_collect_grants_keys`](../src/authz/repo.rs) — locks the role row, then locks the closure of every group assigned to it. -Callers pair these with `begin_all` / `end_all` (see below) rather than `guarded_mutation`, because the enumeration must happen inside the same open transaction as the mutation itself. +Callers pair these with `guarded_tx_mutation` (see below) rather than `guarded_mutation`, because the enumeration must happen inside the same open transaction as the mutation itself. --- @@ -226,30 +231,43 @@ crate::cache::invalidate::guarded_mutation( Defined at [`src/cache/invalidate.rs`](../src/cache/invalidate.rs) `guarded_mutation`. -### Writing (multi-category, single call site) +### Writing (single-category, mutation owns a `Transaction`) + +Use `guarded_tx_mutation` when the affected keys can only be enumerated **under the locks the mutation itself takes** — every group-closure and role mutation. It opens the transaction, runs `collect_keys` (which locks and enumerates), establishes the barrier on that key set, runs `mutate` on the same transaction, commits, and clears the barrier — in that fixed order, regardless of outcome. Stable Rust has no async closures, so the closures return a boxed future borrowing the transaction: ```rust -crate::cache::invalidate::guarded_multi_mutation( +crate::cache::invalidate::guarded_tx_mutation( cache, - &[ - (CacheCategory::TenantStatus, &tenant_keys), - (CacheCategory::Session, &session_keys), - ], - || async { /* mutation touching both */ }, + CacheCategory::Grants, + &state.pool, + |tx| Box::pin(async move { + authz_repo::lock_group_closures_and_collect_grants_keys(tx, &[group_id]).await + }), + |tx| Box::pin(async move { do_mutation_in_tx(tx).await }), ).await? ``` -### Writing (mutation already owns a `Transaction`) +Defined at [`src/cache/invalidate.rs`](../src/cache/invalidate.rs) `guarded_tx_mutation`. Callers hold the `cache: None` fallback themselves because the uncached path is usually a different repo function (typically the non-`_in_tx` variant that opens its own transaction). + +### Writing (multi-category, mutation owns a `Transaction`) -Use `begin_all` / `end_all` when the enumeration of affected keys must happen inside the same open transaction as the mutation (see the race-safe enumeration section): +Use `begin_all` / `end_all` for mutations spanning several categories on one open transaction (e.g. `deleteEntity` invalidates `EntityStatus` + `Session` + `Credential`): ```rust let mut tx = pool.begin().await?; -let grants_keys = lock_group_closures_and_collect_grants_keys(&mut tx, &[group_id]).await?; -let groups = [(CacheCategory::Grants, grants_keys.as_slice())]; +let (session_ids, credential_ids) = + repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by).await?; +let session_keys: Vec = session_ids.iter().map(|id| keys::session(*id)).collect(); +let credential_keys: Vec = credential_ids.iter().map(|id| keys::credential(*id)).collect(); +let entity_status_keys = [keys::entity_status(id)]; +let groups = [ + (CacheCategory::EntityStatus, entity_status_keys.as_slice()), + (CacheCategory::Session, session_keys.as_slice()), + (CacheCategory::Credential, credential_keys.as_slice()), +]; cache::invalidate::begin_all(cache, &groups).await?; -let outcome = do_mutation_in_tx(&mut tx).await; +let outcome = repo::finish_entity_deletion_in_tx(&mut tx, id).await; let outcome = match outcome { Ok(()) => tx.commit().await, Err(e) => Err(e), @@ -305,8 +323,11 @@ All cache operations emit metrics through [`src/metrics.rs`](../src/metrics.rs): |---|---|---| | `atom_cache_lookup_total` | `category`, `outcome` | `hit`, `miss`, `error` | | `atom_cache_invalidation_total` | `category`, `outcome` | `ok`, `error` | +| `atom_cache_populate_total` | `category`, `outcome` | `applied`, `stale`, `error` | + +Category labels are fixed enum variants, so cardinality is bounded. -Category labels are fixed enum variants, so cardinality is bounded (six categories × three outcomes for lookups, six × two for invalidations). +The populate metric splits `applied` (the write took) from `stale` (rejected by the barrier: dirty entry, or the version moved since the caller's `lookup`). A hit rate stuck at zero can then be told apart from every write being rejected by a stuck barrier — otherwise indistinguishable. --- From 491b758642ad474a628e5c7668ea48452a467e57 Mon Sep 17 00:00:00 2001 From: dusan Date: Fri, 31 Jul 2026 13:15:15 +0200 Subject: [PATCH 12/19] Keep the cache barrier intact when Redis is down or a payload is corrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An enabled cache that cannot reach Redis at startup no longer degrades to `None`. `None` means "caching is not configured", and every mutation guard becomes a pass-through on that basis — so a replica that booted during a Redis outage went on mutating grants, sessions and credentials without invalidating entries other replicas were still serving, leaving a revoke here authorized there. Unreachable Redis is a runtime condition, not a configuration one: `CacheClient::build` now separates pool construction (fatal on a bad URL) from `probe` (retryable), and startup keeps the client either way. The client's own behavior already covers the outage — reads fall through to Postgres as misses, and `begin` fails, so security-sensitive mutations are refused until Redis returns. ATOM_CACHE_FAIL_FAST_ON_STARTUP still decides whether to abort instead of booting into that refusing state. Discarding a corrupt payload no longer deletes the whole hash. `DEL` took `v` and `dirty` with it, so a cleanup landing after a concurrent `begin` destroyed that mutation's barrier: the next reader saw an absent key, observed version 0, loaded pre-commit state, and populated it successfully. The new discard script clears only `p`, guarded by the same version/dirty check `try_populate` uses. `end` also clears `p` defensively, which is what its contract already claimed. createTenant invalidates the creator's grants. It bootstraps a tenant-admin role, assignment and membership for the creator in the same transaction, growing their grant set, while the capability gate immediately above warms that exact key — so a creator without platform-wide manage could not administer the tenant they had just created until the grants TTL lapsed. --- src/cache/mod.rs | 152 +++++++++++++++++++++++++++----- src/graphql/tenants.rs | 35 +++++--- src/main.rs | 41 ++++++--- tests/m25_cache_invalidation.rs | 97 ++++++++++++++++++++ 4 files changed, 278 insertions(+), 47 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index c18e154..863489e 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -22,8 +22,8 @@ //! barrier itself with an expiry so a lost `end` call self-heals rather than //! leaving the entry dirty forever. //! - `end` — called after the mutation (success or failure). Bumps the -//! version *again* and clears `dirty`. Never restores a payload — the next -//! reader does a clean reload. The second version bump (beyond the one +//! version *again*, clears `dirty`, and clears any payload — the next reader +//! does a clean reload. The second version bump (beyond the one //! `begin` already did) is what closes the dirty-window race below — it is //! not merely resetting a flag. //! - `try_populate` — called by a cache-miss read after it finishes loading @@ -61,7 +61,6 @@ pub mod keys; use std::{future::Future, time::Duration}; use deadpool_redis::{Config as PoolConfig, Pool, Runtime}; -use redis::AsyncCommands; use serde::{de::DeserializeOwned, Serialize}; use thiserror::Error; @@ -91,11 +90,19 @@ return 1 // with no TTL would never be reclaimed. An `end` that lands after its own // barrier expired (a long mutation, or a bulk invalidation chunking through // many keys) would otherwise leak an immortal hash per key. +// +// `HDEL p` is defensive rather than load-bearing: `begin` already cleared the +// payload, and `try_populate` refuses to write while dirty. It costs one call +// and guarantees that whatever happened to the key in between — including a +// reader repopulating it against a barrier that was destroyed out from under +// this mutation — the entry is left without a payload for the next reader to +// reload cleanly, which is what `end`'s contract has always claimed. const END_SCRIPT_SRC: &str = r#" local ttl_ms = ARGV[1] for i, key in ipairs(KEYS) do redis.call('HINCRBY', key, 'v', 1) redis.call('HSET', key, 'dirty', '0') + redis.call('HDEL', key, 'p') redis.call('PEXPIRE', key, ttl_ms) end return 1 @@ -114,6 +121,25 @@ redis.call('PEXPIRE', KEYS[1], ARGV[3]) return 'applied' "#; +// Discards a corrupt payload without touching the barrier fields. Guarded by +// exactly the same version/dirty check as `try_populate`, and for the same +// reason: an unconditional `DEL` of the whole hash would take `v` and `dirty` +// with it, destroying an in-flight mutation's barrier. The next reader would +// then see an absent key, observe version 0, load pre-commit state, and +// populate it successfully — the barrier's whole purpose, defeated by a +// cleanup path. +const DISCARD_SCRIPT_SRC: &str = r#" +local v = redis.call('HGET', KEYS[1], 'v') +if v == false then v = '0' end +local dirty = redis.call('HGET', KEYS[1], 'dirty') +if dirty == false then dirty = '0' end +if dirty == '1' or v ~= ARGV[1] then + return 'skipped' +end +redis.call('HDEL', KEYS[1], 'p') +return 'discarded' +"#; + /// Fixed, low-cardinality label for cache metrics and log lines. Never an ID, /// action name, or arbitrary string — see `src/metrics.rs`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -222,13 +248,17 @@ pub struct CacheClient { begin_script: redis::Script, end_script: redis::Script, try_populate_script: redis::Script, + discard_script: redis::Script, } impl CacheClient { - /// Connects and verifies reachability with a single `PING`, bounded by - /// `cfg.connect_timeout`. Callers decide what to do with a connect - /// failure (fail fast vs. degrade) — see `main.rs`. - pub async fn connect(cfg: &CacheConfig) -> anyhow::Result { + /// Builds the client and its connection pool *without* contacting Redis. + /// + /// An error here is a configuration error — an unparseable URL, an invalid + /// pool size — never a transient outage, so callers should treat it as + /// fatal. Reachability is a separate, retryable concern: see + /// [`Self::probe`] and `main::init_cache`. + pub fn build(cfg: &CacheConfig) -> anyhow::Result { let pool_cfg = PoolConfig::from_url(&cfg.redis_url); let mut pool_builder = pool_cfg .builder() @@ -239,20 +269,31 @@ impl CacheClient { .build() .map_err(|e| anyhow::anyhow!("failed to build cache pool: {e}"))?; - let client = Self { + Ok(Self { pool, op_timeout: Duration::from_millis(cfg.op_timeout_ms), ttl: cfg.ttl, begin_script: redis::Script::new(BEGIN_SCRIPT_SRC), end_script: redis::Script::new(END_SCRIPT_SRC), try_populate_script: redis::Script::new(TRY_POPULATE_SCRIPT_SRC), - }; + discard_script: redis::Script::new(DISCARD_SCRIPT_SRC), + }) + } - tokio::time::timeout(Duration::from_millis(cfg.connect_timeout_ms), client.ping()) + /// One-shot reachability check with a single `PING`, bounded by + /// `connect_timeout_ms`. + pub async fn probe(&self, connect_timeout_ms: u64) -> anyhow::Result<()> { + tokio::time::timeout(Duration::from_millis(connect_timeout_ms), self.ping()) .await .map_err(|_| anyhow::anyhow!("cache connect timed out"))? - .map_err(|e| anyhow::anyhow!("cache connect failed: {e}"))?; + .map_err(|e| anyhow::anyhow!("cache connect failed: {e}")) + } + /// [`build`](Self::build) plus a [`probe`](Self::probe) — fails unless + /// Redis is reachable right now. + pub async fn connect(cfg: &CacheConfig) -> anyhow::Result { + let client = Self::build(cfg)?; + client.probe(cfg.connect_timeout_ms).await?; Ok(client) } @@ -370,17 +411,29 @@ impl CacheClient { } Err(err) => { tracing::warn!(category = category.as_str(), error = %err, "cache payload corrupt; discarding"); - self.best_effort_delete(key).await; + self.discard_payload(key, version).await; metrics::record_cache_lookup(category.as_str(), "miss"); Lookup::Miss { version } } } } - async fn best_effort_delete(&self, key: &str) { - if let Ok(mut conn) = self.get_conn().await { - let _: Result<(), redis::RedisError> = conn.del(key).await; - } + /// Clears a corrupt payload field, leaving the barrier (`v`/`dirty`) + /// intact, and only while the entry is still at `observed_version` and not + /// dirty. Best-effort: a failure just leaves the corrupt payload for the + /// next reader to trip over and re-attempt. + async fn discard_payload(&self, key: &str, observed_version: i64) { + let Ok(mut conn) = self.get_conn().await else { + return; + }; + let _ = tokio::time::timeout( + self.op_timeout, + self.discard_script + .key(key) + .arg(observed_version) + .invoke_async::(&mut conn), + ) + .await; } /// Best-effort conditional write following a cache-miss load. Discarded @@ -689,7 +742,7 @@ mod tests { #[tokio::test] #[ignore] - async fn corrupt_payload_is_treated_as_a_miss_and_deleted() { + async fn corrupt_payload_is_discarded_without_destroying_the_barrier() { let client = test_client().await; let key = unique_key("corrupt"); @@ -711,14 +764,69 @@ mod tests { other => panic!("expected corrupt payload to be a miss, got {other:?}"), } - // The corrupt entry must have been best-effort deleted so the next - // lookup doesn't repeat the same deserialize failure. - let exists: bool = redis::cmd("EXISTS") + // The corrupt payload must be gone so the next lookup doesn't repeat + // the same deserialize failure — but *only* the payload. Deleting the + // whole hash would take `v`/`dirty` with it, and a concurrent + // mutation's barrier along with them. + let (version, payload): (Option, Option>) = redis::cmd("HMGET") + .arg(&key) + .arg("v") + .arg("p") + .query_async(&mut conn) + .await + .expect("read back barrier fields"); + assert!( + payload.is_none(), + "corrupt payload should have been cleared" + ); + assert_eq!( + version, + Some(1), + "the version must survive: it is what a concurrent mutation's barrier rests on" + ); + } + + /// The reason [`DISCARD_SCRIPT_SRC`] is version-guarded: a corrupt-payload + /// cleanup racing a mutation must not clear the barrier that mutation just + /// established, or a reader could repopulate pre-commit state over it. + #[tokio::test] + #[ignore] + async fn corrupt_payload_cleanup_leaves_a_concurrent_barrier_intact() { + let client = test_client().await; + let key = unique_key("corrupt-vs-barrier"); + let keys = vec![key.clone()]; + + let mut conn = client.get_conn().await.expect("conn"); + let _: () = redis::cmd("HSET") .arg(&key) + .arg("v") + .arg(1) + .arg("p") + .arg("not valid json") + .query_async(&mut conn) + .await + .expect("seed corrupt payload"); + + // A mutation opens its barrier *after* the reader observed version 1. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("begin"); + + // The reader's cleanup now fires against its stale observed version. + client.discard_payload(&key, 1).await; + + let dirty: Option = redis::cmd("HGET") + .arg(&key) + .arg("dirty") .query_async(&mut conn) .await - .expect("exists check"); - assert!(!exists, "corrupt entry should have been deleted"); + .expect("read dirty"); + assert_eq!( + dirty.as_deref(), + Some("1"), + "the in-flight mutation's barrier must survive a late corrupt-payload cleanup" + ); } #[tokio::test] diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 4908623..1f79d93 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -289,18 +289,31 @@ impl TenantMutation { &[("manage", Scope::Platform), ("create", Scope::Platform)], ) .await?; - tenant_repo::create_tenant_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_model::CreateTenant { - id, - name: input.name, - alias: input.alias, - tags: input.tags.unwrap_or_default(), - attributes: input.attributes.unwrap_or(serde_json::Value::Null), + // `create_tenant` bootstraps a tenant-admin role, role assignment + // and membership for the creator in the same transaction, so it + // grows the creator's own grant set. The capability gate directly + // above has just warmed that exact `grants` entry, so without this + // barrier a creator who isn't already a platform admin cannot + // manage the tenant they just created until the grants TTL lapses. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Grants, + std::slice::from_ref(&crate::cache::keys::grants(auth.entity_id)), + || { + tenant_repo::create_tenant_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_model::CreateTenant { + id, + name: input.name, + alias: input.alias, + tags: input.tags.unwrap_or_default(), + attributes: input.attributes.unwrap_or(serde_json::Value::Null), + }, + Some(auth.entity_id), + ) }, - Some(auth.entity_id), ) .await } diff --git a/src/main.rs b/src/main.rs index be23534..f323602 100644 --- a/src/main.rs +++ b/src/main.rs @@ -100,27 +100,40 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -/// Connects the Redis-backed cache when enabled. A connect failure honors -/// `ATOM_CACHE_FAIL_FAST_ON_STARTUP`: abort like an unreachable Postgres -/// would, or log and continue cache-free (the recommended default — caching -/// is a performance optimization, not a correctness dependency for reads). +/// Builds the Redis-backed cache when enabled. +/// +/// `None` means "caching is not configured", and every mutation guard becomes +/// a pure pass-through on that basis. So an *enabled* cache that merely can't +/// reach Redis right now must never degrade to `None`: this process would go +/// on mutating grants, sessions, and credentials without invalidating entries +/// that other replicas are still serving, and a revoke here would stay +/// authorized there. Unreachable Redis is a runtime condition, not a +/// configuration one — the client is retained either way, and its own +/// behavior covers the outage: reads fall through to Postgres as misses, +/// while `begin` fails and so refuses security-sensitive mutations until +/// Redis returns (see `src/cache/mod.rs` and `src/cache/invalidate.rs`). +/// +/// A *build* failure is fatal regardless: an unparseable URL cannot recover. +/// `ATOM_CACHE_FAIL_FAST_ON_STARTUP` then decides whether an unreachable +/// Redis should also abort startup, rather than boot into the refusing state. async fn init_cache(cfg: &config::CacheConfig) -> anyhow::Result> { if !cfg.enabled { return Ok(None); } - match cache::CacheClient::connect(cfg).await { - Ok(client) => { - tracing::info!("cache enabled; connected to Redis"); - Ok(Some(client)) - } + let client = cache::CacheClient::build(cfg).context("cache configuration is invalid")?; + match client.probe(cfg.connect_timeout_ms).await { + Ok(()) => tracing::info!("cache enabled; connected to Redis"), Err(err) if cfg.fail_fast_on_startup => { - Err(err.context("cache connect failed and ATOM_CACHE_FAIL_FAST_ON_STARTUP=true")) - } - Err(err) => { - tracing::error!("cache connect failed, continuing without cache: {err}"); - Ok(None) + return Err( + err.context("cache connect failed and ATOM_CACHE_FAIL_FAST_ON_STARTUP=true") + ); } + Err(err) => tracing::error!( + "cache enabled but Redis is unreachable: {err}. Reads fall through to Postgres; \ + security-sensitive mutations are refused until Redis recovers." + ), } + Ok(Some(client)) } fn init_tracing(logging: &config::LoggingConfig) -> anyhow::Result<()> { diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 32ca268..8f1df4f 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -1356,6 +1356,103 @@ async fn a_stale_jwt_from_before_a_tenant_move_cannot_poison_the_old_tenants_sta ); } +/// A platform-scope allow block granting `create`, so a non-admin subject can +/// be given exactly the capability `createTenant`'s gate requires. +async fn make_platform_create_block(pool: &PgPool) -> Uuid { + let action_id: Uuid = + sqlx::query_scalar("SELECT id FROM actions WHERE name = 'create' LIMIT 1") + .fetch_one(pool) + .await + .expect("create action"); + authz_repo::create_permission_block( + pool, + CreatePermissionBlock { + tenant_id: None, + scope_mode: "platform".into(), + object_kind: None, + object_type: None, + object_id: None, + group_id: None, + effect: Effect::Allow, + conditions: json!({}), + action_ids: vec![action_id], + }, + ) + .await + .expect("create permission block") + .id +} + +/// Regression test for a review finding: `createTenant` bootstraps a +/// tenant-admin role, role assignment and membership for the creator in the +/// same transaction — growing the creator's own grant set — but ran with no +/// `grants` invalidation. The capability gate immediately above it warms that +/// exact key, so a creator without platform-wide `manage` could not administer +/// the tenant they had just created until the grants TTL lapsed. +#[tokio::test] +#[ignore] +async fn tenant_creation_immediately_grants_the_creator_tenant_admin() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + // Deliberately *not* a platform admin: with platform-wide `manage` the + // creator would pass the post-creation check no matter what the cache + // held, and the test would prove nothing. + let creator = active_entity(&p, "service").await; + let block_id = make_platform_create_block(&p).await; + authz_repo::create_direct_policy( + &p, + CreateDirectPolicy { + tenant_id: None, + subject_kind: SubjectKind::Entity, + subject_id: creator, + permission_block_id: block_id, + }, + ) + .await + .expect("create direct policy"); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + creator, + cache.clone(), + r#"mutation { createTenant(input: { name: "cache-test-bootstrap" }) { id } }"#, + )) + .await; + assert!( + resp.errors.is_empty(), + "createTenant failed: {:?}", + resp.errors + ); + let tenant_id: Uuid = resp + .data + .into_json() + .expect("response is json") + .pointer("/createTenant/id") + .and_then(|id| id.as_str()) + .expect("createTenant returned an id") + .parse() + .expect("parse created tenant id"); + + // The creator's grants were warmed by `createTenant`'s own capability + // gate, moments before the bootstrap added the tenant-admin assignment. + let req = AuthzRequest { + subject_id: creator, + action: "manage".into(), + resource_id: None, + object_kind: Some("tenant".into()), + object_id: Some(tenant_id), + context: json!({}), + }; + let auth = auth_context(creator, cache.clone()); + let decision = engine::evaluate(&p, &req, &auth).await.expect("evaluate"); + assert!( + decision.allowed, + "the creator must be able to manage the tenant they just created on the very next \ + request, not after the grants TTL expires" + ); +} + // ─── Group-subject mutation vs. concurrent membership change ─────────────── /// Regression test for a review finding: resolving a group-subject From f8c1da815236383733b158a3035a61476b63984a Mon Sep 17 00:00:00 2001 From: dusan Date: Fri, 31 Jul 2026 13:23:24 +0200 Subject: [PATCH 13/19] Update docs Signed-off-by: dusan --- docs/content/docs/architecture/caching.mdx | 27 ++++++++++---- product-docs/14-caching.md | 43 ++++++++++++++++------ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/docs/content/docs/architecture/caching.mdx b/docs/content/docs/architecture/caching.mdx index 24e1916..3d840c7 100644 --- a/docs/content/docs/architecture/caching.mdx +++ b/docs/content/docs/architecture/caching.mdx @@ -23,6 +23,8 @@ The cache targets these hot reads. Postgres remains the single source of truth; 4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate would risk serving stale grants after a revoke. 5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. +`cache: None` means *caching is not configured*, and it is the one state in which every mutation guard degrades to a pass-through. An **enabled** cache that merely cannot reach Redis is a different state and never collapses into `None` — otherwise a replica that booted during an outage would mutate grants, sessions, and credentials without invalidating entries its peers were still serving, and a revoke on one replica would stay authorized on another. Unreachable Redis is a runtime condition: the client is retained, reads degrade to misses, and `begin` fails so security-sensitive mutations are refused until Redis returns. + ## What Is Cached Six categories, each keyed by a UUID under the `atom:v1:` namespace. @@ -42,6 +44,7 @@ Six categories, each keyed by a UUID under the `atom:v1:` namespace. - **Plaintext API-key secrets** — only the hash used to verify them. - **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. - **Audit writes** — always go to Postgres. +- **Tombstones.** The entity and tenant entries carry no `deleted_at`. Both miss loaders already filter `deleted_at IS NULL`, so a cached copy would be empty by construction and any check against it a no-op that merely *looked* like a tombstone check. Denying a subsequently soft-deleted entity or tenant is the delete path's invalidation duty. ## Request Flow @@ -71,7 +74,9 @@ Six categories, each keyed by a UUID under the `atom:v1:` namespace. PDP --> Done `} /> -The auth hot path reads its independent keys in **one pipelined round trip on a single pooled connection**. Issued one at a time, three lookups would mean three pool acquisitions and three serial round trips, each bounded by the operation timeout, before any request work started. +The auth hot path batches the keys it can into **one pipelined round trip on a single pooled connection**. Issued one at a time, three lookups would mean three pool acquisitions and three serial round trips, each bounded by the operation timeout, before any request work started. + +Only keys with no data dependency on each other can share a round trip. For JWT auth, session, entity, and tenant are all known up front — the tenant key comes from the token's `tid` claim — so all three go together. For API-key auth the credential key is read first and alone because it gates everything after it, and the tenant key is derived from the entity's current tenant rather than from the credential, so it cannot be batched with it. ## Consistency Model @@ -81,13 +86,14 @@ Every cached entry is a Redis hash with three fields: - `dirty` — `"1"` while a mutation is in flight, absent otherwise. - `p` — the serialized payload, present only when the entry holds a valid value. -Three atomic Lua scripts implement a per-key **mutation barrier**. +Four atomic Lua scripts implement a per-key **mutation barrier**. | Primitive | When it runs | What it does | |---|---|---| | `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | -| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, re-applies the entry's expiry. Best-effort. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, clears `p`, and re-applies the entry's expiry. Best-effort. | | `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | +| `discard` | After a reader fails to deserialize a payload | Clears **only** `p`, and only if `dirty=0` and `v` still equals what that reader observed. Never touches the barrier fields. | ### Read path @@ -123,7 +129,7 @@ Three atomic Lua scripts implement a per-key **mutation barrier**. W->>P: UPDATE / DELETE ... P-->>W: committed W->>C: end(keys) - Note over C: bump v again, clear dirty + Note over C: bump v again, clear dirty,
clear p, re-apply expiry end `} /> @@ -133,6 +139,7 @@ Three atomic Lua scripts implement a per-key **mutation barrier**. 2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). 3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. 4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — populates only apply when the observed version's key matches the key the payload describes. +5. **Cleanup destroying a live barrier.** Discarding a corrupt payload must not take `v` and `dirty` with it. Deleting the whole hash would erase a concurrent mutation's barrier: the next reader would find an absent key, observe version `0`, load pre-commit state, and populate it successfully. `discard` is version-guarded and clears only `p`; `end` also clears `p` defensively. ## Invalidation Map @@ -145,7 +152,7 @@ Three atomic Lua scripts implement a per-key **mutation barrier**. | Tenant update | `TenantStatus` | | Tenant delete | `TenantStatus` + child `Session`s | | Tenant restore | `TenantStatus` + reactivated `Credential`s | -| Tenant create / purge | `Grants` (of acting subject) | +| Tenant create | `Grants` (of the creator) | | Credential revoke / rotate | `Credential` | | Credential scope change | `CredentialCeiling` | | Role assignment (create / delete) | `Grants` for each affected subject | @@ -153,6 +160,8 @@ Three atomic Lua scripts implement a per-key **mutation barrier**. | Role permission-block change | `Grants` for every assignee of the role | | Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | +Tenant creation is on this list because `create_tenant` bootstraps a tenant-admin role, role assignment, and membership for the creator in the same transaction — it grows the creator's own grant set, and the capability gate immediately above it has just warmed that exact key. `purgeTenant` performs no invalidation: it is reachable only for an already-soft-deleted tenant, whose soft delete invalidated `TenantStatus` and the members' sessions, so the tenant is already denied at the lifecycle check that runs before grant matching. + ### Race-safe enumeration Group and role invalidations must enumerate *who is affected* — every entity in a group closure, every assignee of a role. If enumeration happens outside a transaction, a concurrent `add_group_member` can slip a new member in between enumeration and mutation. That new member's `grants` key is never invalidated, and a stale entry survives until TTL. @@ -163,12 +172,13 @@ The enumeration functions therefore lock the relevant rows `FOR UPDATE` inside t | Failure | Behaviour | Impact | |---|---|---| -| Redis unreachable at startup | Startup fails fast or logs a warning, per config. | Operator-controlled. | +| Redis unreachable at startup | The client is retained, never downgraded to `cache: None`. `ATOM_CACHE_FAIL_FAST_ON_STARTUP` decides whether to abort instead. | Reads fall through to Postgres; security-sensitive mutations are refused until Redis recovers. | +| Cache config invalid at startup | Fatal regardless of the fail-fast flag — an unparseable URL cannot recover by retrying. | Startup fails. | | Redis unreachable during read | Treated as unavailable; falls through to Postgres. | Auth works, slower. | | Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | Mutation does not commit. | | Redis unreachable during `end` | Best-effort; entry stays dirty until barrier TTL. | Entry reloads on next reader. | | Redis unreachable during `try_populate` | Best-effort; dropped silently. | Next reader still gets a miss and retries. | -| Corrupt payload | Treated as a miss; the entry is deleted so the next read reloads clean. | One extra Postgres round trip. | +| Corrupt payload | Treated as a miss; only the payload field is cleared, version-guarded, so a concurrent mutation's barrier survives. | One extra Postgres round trip. | ## Configuration @@ -177,9 +187,10 @@ All knobs live under the `ATOM_CACHE_*` prefix: - `ATOM_CACHE_ENABLED` — master switch. - `ATOM_CACHE_REDIS_URL` — connection URL. - `ATOM_CACHE_POOL_MAX_SIZE` — max Redis connections. +- `ATOM_CACHE_FAIL_FAST_ON_STARTUP` — `true` aborts startup when Redis is unreachable. Default `false`. - `ATOM_CACHE_CONNECT_TIMEOUT_MS` — startup PING timeout. - `ATOM_CACHE_OP_TIMEOUT_MS` — per-operation timeout. -- `ATOM_CACHE_TTL__SECS` — one per category (session, entity_status, tenant_status, credential, credential_ceiling, grants). +- `ATOM_CACHE_TTL__SECS` — one per category (session, entity_status, tenant_status, credential, credential_ceiling, grants). Each must be greater than zero and no more than 86400 seconds (24h); the barrier expiry is derived as `entry_ttl * 5` and has to stay representable. TTLs are the residual staleness bound if invalidation is missed entirely (e.g. `end` never completed and the barrier TTL elapsed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. diff --git a/product-docs/14-caching.md b/product-docs/14-caching.md index 1b10151..ed64463 100644 --- a/product-docs/14-caching.md +++ b/product-docs/14-caching.md @@ -29,6 +29,8 @@ The cache targets these hot reads specifically, keeping Postgres as the single s 4. **Fail-refused writes.** If Redis is unreachable during a security-sensitive Postgres mutation, the mutation is refused. Committing without being able to invalidate the cache would risk serving stale grants after a revoke. 5. **Optional at every layer.** Every call site tolerates `cache: None`. Removing Redis is a config change, not a code change. +`cache: None` means *caching is not configured*, and it is the one state in which every mutation guard degrades to a pass-through. An **enabled** cache that merely cannot reach Redis is a different state and never collapses into `None` — otherwise a replica that booted during an outage would mutate grants, sessions, and credentials without invalidating entries its peers were still serving, and a revoke on one replica would stay authorized on another. Unreachable Redis is a runtime condition: the client is retained, reads degrade to misses, and `begin` fails so security-sensitive mutations are refused until Redis returns. + --- ## What Is Cached @@ -46,10 +48,12 @@ Six categories, each keyed by a UUID under the `atom:v1:` namespace. DTOs are defined in [`src/cache/entries.rs`](../src/cache/entries.rs). Key builders in [`src/cache/keys.rs`](../src/cache/keys.rs). +The entity and tenant DTOs deliberately carry **no `deleted_at` field**. Both miss loaders already filter `deleted_at IS NULL`, so an entry can only ever be populated from a live row — a cached tombstone column would be `None` by construction and any check against it a no-op that merely *looked* like a tombstone check. Denying a subsequently soft-deleted entity or tenant is the delete path's invalidation duty, not the entry's. + ### What is *not* cached - **Passwords** — never used on the request path. Used only during `/login`, which mints a JWT; the JWT then uses the `Session` cache. -- **Plaintext API-key secrets** — only the hash used to verify them. See [`CredentialCacheEntry`](../src/cache/entries.rs#L49). +- **Plaintext API-key secrets** — only the hash used to verify them. See `CredentialCacheEntry` in [`src/cache/entries.rs`](../src/cache/entries.rs). - **The authorization decision itself** — the PDP evaluates conditions per request against fresh (or freshly cached) grants. - **Audit writes** — always go to Postgres. @@ -89,7 +93,12 @@ flowchart TD - API-key auth: [`src/auth.rs`](../src/auth.rs) around `auth_from_api_key`. - Grants load: [`src/auth.rs`](../src/auth.rs) `AuthContext::effective_grants` and [`src/authz/engine.rs`](../src/authz/engine.rs) inside `load_decision_context`. -The auth hot path reads its independent keys (session/entity/tenant for JWT, credential/entity/tenant for API-key) in **one pipelined round trip on a single pooled connection** via `CacheClient::lookup_many` + `CacheClient::decode` — see [`src/cache/mod.rs`](../src/cache/mod.rs). Issued one at a time it would be three pool acquisitions and three serial round trips, each bounded by `op_timeout`, before any request work started. +The auth hot path batches the keys it can into **one pipelined round trip on a single pooled connection**, via `CacheClient::lookup_many` + `CacheClient::decode` — see [`src/cache/mod.rs`](../src/cache/mod.rs). Issued one at a time these would be three pool acquisitions and three serial round trips, each bounded by `op_timeout`, before any request work started. + +Only keys with no data dependency on each other can share a round trip: + +- **JWT** — session, entity, and tenant are all known up front, since the tenant key comes from the token's `tid` claim. All three go in one round trip. +- **API-key** — the credential key is read first and alone, because it gates everything after it; on a credential hit the entity read likewise precedes the tenant key, which is derived from the *entity's* current tenant rather than from the credential (see the note on `CredentialCacheEntry` above). On the cold path, where both are already known from the loaded row, the entity and tenant reads are batched. --- @@ -101,15 +110,16 @@ Every cached entry is a Redis hash with three fields: - `dirty` — `"1"` while a mutation is in flight, absent otherwise. - `p` — the serialized payload, present only when the entry holds a valid value. -Three atomic Lua scripts implement a per-key **mutation barrier** that closes three otherwise-unavoidable races. +Four atomic Lua scripts implement a per-key **mutation barrier** that closes races a plain cache-aside plus post-commit `DEL` cannot. ### The three primitives | Primitive | When it runs | What it does | |---|---|---| | `begin` | Before a security-sensitive Postgres mutation | Bumps `v`, sets `dirty=1`, clears `p`. Fails the mutation if Redis is unreachable. | -| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`. Best-effort. | +| `end` | After the mutation (success or failure) | Bumps `v` again, clears `dirty`, clears `p`, and re-applies the entry's expiry. Best-effort. | | `try_populate` | After a cache-miss reader finishes loading from Postgres | Writes the payload only if `dirty=0` **and** `v` still equals what the reader observed pre-load; otherwise discards silently. | +| `discard` | After a reader fails to deserialize a payload | Clears **only** `p`, and only if `dirty=0` and `v` still equals what that reader observed. Never touches the barrier fields. | ### Read path @@ -149,7 +159,7 @@ sequenceDiagram W->>P: UPDATE / DELETE ... P-->>W: committed W->>C: end(keys) - Note over C: bump v again, clear dirty + Note over C: bump v again, clear dirty,
clear p, re-apply expiry end ``` @@ -159,6 +169,7 @@ sequenceDiagram 2. **Read-during-dirty-window.** A reader lands while `dirty=1`, observes the post-`begin` version. `end`'s **second** version bump ensures that observed version is stale by the time `try_populate` runs — rejected either by the dirty check (if still dirty) or the version check (if `end` already ran). 3. **Lost-invalidation.** If `end` never runs (crash), the barrier TTL causes the whole entry to expire rather than being stuck dirty forever. `end` re-applies `PEXPIRE` on the entry, since `HINCRBY` would otherwise recreate an already-expired key and leave it immortal. 4. **Cross-key poisoning during populate.** A miss loader whose returned payload describes a *different* key than the one being populated must never write across keys — e.g. the JWT miss loader joins tenants through `entities.tenant_id`, so it returns the entity's *current* tenant's status, which is not necessarily the tenant the token's `tid` claim points to when the token outlived a tenant move. Populates now write only when the observed version's key matches the key the payload describes. +5. **Cleanup destroying a live barrier.** Discarding a corrupt payload must not take `v` and `dirty` with it. Deleting the whole hash would erase a concurrent mutation's barrier: the next reader would find an absent key, observe version `0`, load pre-commit state, and populate it successfully — the barrier defeated by a cleanup path. `discard` is version-guarded and clears only `p`; `end` also clears `p` defensively, so whatever happened to the key mid-mutation, the next reader reloads cleanly. The first primitive that fails self-heals: `begin` failing refuses the mutation; `end` failing leaves the entry dirty until barrier-TTL expiry; `try_populate` failing just leaves the entry as a miss for the next reader to reload. @@ -179,7 +190,7 @@ Which mutation invalidates which category: | Tenant update | `TenantStatus` | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | | Tenant delete | `TenantStatus` + child `Session`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | | Tenant restore | `TenantStatus` + reactivated `Credential`s | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | -| Tenant create / purge | `Grants` (of acting subject) | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | +| Tenant create | `Grants` (of the creator) | [`src/graphql/tenants.rs`](../src/graphql/tenants.rs) | | Credential revoke / rotate | `Credential` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | | Credential scope change | `CredentialCeiling` | [`src/graphql/credentials.rs`](../src/graphql/credentials.rs) | | Role assignment (create / delete) | `Grants` for each affected subject | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | @@ -187,6 +198,10 @@ Which mutation invalidates which category: | Role permission-block change | `Grants` for every assignee of the role | [`src/graphql/policies.rs`](../src/graphql/policies.rs) | | Group membership change (REST or GraphQL) | `Grants` for every member of the group closure | [`src/graphql/groups.rs`](../src/graphql/groups.rs), [`src/identity/handlers.rs`](../src/identity/handlers.rs) | +Tenant creation is on this list because `create_tenant` bootstraps a tenant-admin role, role assignment, and membership for the creator in the same transaction — it grows the creator's own grant set, and the capability gate immediately above it has just warmed that exact key. + +`purgeTenant` performs **no** cache invalidation. It is reachable only for an already-soft-deleted tenant, and the soft delete invalidated `TenantStatus` and the members' sessions, so the tenant is already denied at the lifecycle check that runs before grant matching. Residual `Grants` entries naming the purged tenant are therefore inert rather than dangerous — but this rests on the soft delete having run first, and is worth revisiting if purge ever becomes reachable directly. + The REST paths (`delete_entity`, `add_group_member`, `remove_group_member`) invalidate through the same helpers as the GraphQL resolvers — entity deletion is consolidated into a single service entry point at [`src/identity/service.rs`](../src/identity/service.rs), and both group-member handlers wrap their mutation in `guarded_mutation`. ### Race-safe enumeration @@ -282,12 +297,13 @@ outcome? | Failure | Behaviour | Impact | |---|---|---| -| Redis unreachable at startup | `main.rs` decides fail-fast vs degrade (see `ATOM_CACHE_REQUIRED`). | Startup fails or logs a warning. | +| Redis unreachable at startup | The client is retained, never downgraded to `cache: None`. `ATOM_CACHE_FAIL_FAST_ON_STARTUP` decides whether to abort startup instead of booting into the refusing state below. | Reads fall through to Postgres; security-sensitive mutations are refused until Redis recovers. | +| Cache config invalid at startup | Fatal regardless of `ATOM_CACHE_FAIL_FAST_ON_STARTUP` — an unparseable `ATOM_CACHE_REDIS_URL` cannot recover by retrying. | Startup fails. | | Redis unreachable during read | Treated as `Lookup::Unavailable`. Falls through to Postgres loader. | Auth works, slower. | | Redis unreachable during `begin` | Mutation refused with `503 service_unavailable`. | The mutation does not commit. | | Redis unreachable during `end` | Best-effort — logged, not surfaced. Entry stays dirty until barrier TTL. | Entry reloads on next reader; slight perf hit until then. | | Redis unreachable during `try_populate` | Best-effort — dropped silently. | Next reader still gets a miss and retries. | -| Corrupt payload | Treated as a miss and the entry is deleted so the next read reloads clean. | One extra Postgres round trip. | +| Corrupt payload | Treated as a miss; only the payload field is cleared, version-guarded, so a concurrent mutation's barrier survives. | One extra Postgres round trip. | --- @@ -300,6 +316,7 @@ Set via environment variables — see [`.env.example`](../.env.example) for the | `ATOM_CACHE_ENABLED` | Master switch. `false` disables all caching (call sites pass `cache: None`). | | `ATOM_CACHE_REDIS_URL` | Redis connection URL. | | `ATOM_CACHE_POOL_MAX_SIZE` | Max Redis connections. | +| `ATOM_CACHE_FAIL_FAST_ON_STARTUP` | `true` aborts startup when Redis is unreachable. Default `false` — boot and refuse security-sensitive mutations until Redis recovers. | | `ATOM_CACHE_CONNECT_TIMEOUT_MS` | Startup PING timeout. | | `ATOM_CACHE_OP_TIMEOUT_MS` | Per-operation timeout. | | `ATOM_CACHE_TTL_SESSION_SECS` | TTL for `Session` entries. | @@ -311,6 +328,8 @@ Set via environment variables — see [`.env.example`](../.env.example) for the Config struct: [`src/config.rs`](../src/config.rs) `CacheConfig` / `CacheTtlConfig`. +Every `ATOM_CACHE_TTL_*` value must be greater than zero and no more than **86400 seconds (24h)**, rejected at startup otherwise. The upper bound exists because the barrier expiry is derived as `entry_ttl * 5`, which has to stay representable — without the bound, a nonsensically large TTL would boot and serve reads happily, then fail on the first security-sensitive mutation. + TTLs are the residual staleness bound if invalidation is missed entirely (e.g. barrier TTL expired before `end` completed). They should be short enough that a missed invalidation is a bounded outage, not an indefinite one. --- @@ -340,7 +359,7 @@ The cache client (Redis pool, Lua scripts, barrier, timeouts, metrics) is fully 3. Add a key builder to [`src/cache/keys.rs`](../src/cache/keys.rs). 4. (Optional) add a DTO to [`src/cache/entries.rs`](../src/cache/entries.rs) — the client is generic over any `Serialize + DeserializeOwned` type, so a bespoke DTO is only needed if the DB row shape is not directly usable. 5. At each read site: call `cached_or_load(cache, CacheCategory::, &keys::(id), || loader)`. -6. At each write site that could affect the entry: wrap the mutation in `guarded_mutation` (or `begin_all` / `end_all` if the mutation owns an open `Transaction`). +6. At each write site that could affect the entry: wrap the mutation in `guarded_mutation` — or `guarded_tx_mutation` if the affected keys can only be enumerated under the mutation's own locks, or `begin_all` / `end_all` if the mutation spans several categories on one open `Transaction`. The design is deliberately explicit — the enum is not `Other(String)` — because: @@ -352,11 +371,13 @@ The design is deliberately explicit — the enum is not `Other(String)` — beca ## Testing -- **Barrier semantics (Redis-gated unit tests):** [`src/cache/mod.rs`](../src/cache/mod.rs) `#[cfg(test)] mod tests`. Includes tests for both the read-before-mutation race and the read-during-dirty-window race. -- **End-to-end invalidation matrix:** [`tests/m25_cache_invalidation.rs`](../tests/m25_cache_invalidation.rs). Covers every mutation → invalidation pairing listed above. +- **Barrier semantics (Redis-gated unit tests):** [`src/cache/mod.rs`](../src/cache/mod.rs) `#[cfg(test)] mod tests`. Covers the read-before-mutation race, the read-during-dirty-window race, and that a corrupt-payload cleanup leaves a concurrent barrier intact. +- **End-to-end invalidation matrix:** [`tests/m25_cache_invalidation.rs`](../tests/m25_cache_invalidation.rs). Covers every mutation → invalidation pairing listed above, plus the cross-key poisoning and enumeration races. Both suites are `#[ignore]` and require `ATOM_TEST_REDIS_URL`; run with `cargo test -- --include-ignored`. +Redis must be **flushed between test binaries**, alongside the per-binary database recreate — see `run_one` in [`.github/workflows/rust.yml`](../.github/workflows/rust.yml). The suite caches under keys derived from the fixed seeded admin id, so without a flush the admin's grant expansion outlives the database it was derived from and a later binary authorizes against a tenant graph that no longer exists. + --- ## Related Documents From b9b4a97f477e326279e722931cbc48ed24d1836c Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Sat, 1 Aug 2026 16:42:02 +0300 Subject: [PATCH 14/19] Switch cache payload encoding from JSON to MessagePack Faster and smaller than JSON while staying self-describing, so EffectiveGrant.conditions (arbitrary serde_json::Value) round-trips with no wrapper needed. --- Cargo.lock | 20 ++++++++++++++++++++ Cargo.toml | 1 + src/cache/mod.rs | 12 ++++++------ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9068ba..630a371 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -452,6 +452,7 @@ dependencies = [ "rcgen", "redis", "ring", + "rmp-serde", "serde", "serde_json", "sqlx", @@ -3509,6 +3510,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rsa" version = "0.9.10" diff --git a/Cargo.toml b/Cargo.toml index 5124b94..4553abc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ lapin = { version = "4.10.0", default-features = false, features = [ ] } redis = { version = "1", default-features = false, features = ["script"] } deadpool-redis = { version = "0.23", default-features = false, features = ["rt_tokio_1"] } +rmp-serde = "1" [features] # Metrics are on by default. Disable at compile time for maximum-performance diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 863489e..4a7a247 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -404,7 +404,7 @@ impl CacheClient { metrics::record_cache_lookup(category.as_str(), "miss"); return Lookup::Miss { version }; }; - match serde_json::from_slice::(&payload) { + match rmp_serde::from_slice::(&payload) { Ok(value) => { metrics::record_cache_lookup(category.as_str(), "hit"); Lookup::Hit(value) @@ -446,7 +446,7 @@ impl CacheClient { expected_version: i64, value: &T, ) { - let Ok(payload) = serde_json::to_vec(value) else { + let Ok(payload) = rmp_serde::to_vec(value) else { tracing::warn!( category = category.as_str(), "cache payload serialize failed" @@ -746,15 +746,15 @@ mod tests { let client = test_client().await; let key = unique_key("corrupt"); - // Write a payload that isn't valid JSON for `Payload` directly into - // the hash, bypassing `try_populate`, to simulate corruption. + // Write a payload that isn't a validly-encoded `Payload` directly + // into the hash, bypassing `try_populate`, to simulate corruption. let mut conn = client.get_conn().await.expect("conn"); let _: () = redis::cmd("HSET") .arg(&key) .arg("v") .arg(1) .arg("p") - .arg("not valid json") + .arg("not a valid payload") .query_async(&mut conn) .await .expect("seed corrupt payload"); @@ -802,7 +802,7 @@ mod tests { .arg("v") .arg(1) .arg("p") - .arg("not valid json") + .arg("not a valid payload") .query_async(&mut conn) .await .expect("seed corrupt payload"); From e80f1f7bf3e9931f6454a73e7ffc931ebc5d2208 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Sat, 1 Aug 2026 20:06:24 +0300 Subject: [PATCH 15/19] cargo fmt after rebase conflict resolution --- src/authz/repo.rs | 3 +-- src/identity/repo.rs | 11 ++++++----- src/main.rs | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/authz/repo.rs b/src/authz/repo.rs index d1653d5..b813165 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -4072,8 +4072,7 @@ pub async fn create_role_assignment_with_audit( req: CreateRoleAssignment, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; - let assignment = - create_role_assignment_in_tx(&mut tx, events_enabled, actor_id, req).await?; + let assignment = create_role_assignment_in_tx(&mut tx, events_enabled, actor_id, req).await?; tx.commit().await.map_err(db_err)?; Ok(assignment) } diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 3306c67..0b3d54c 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -841,11 +841,12 @@ pub async fn finish_entity_deletion_in_tx( .await .map_err(db_err)?; - let tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1") - .bind(id) - .fetch_one(&mut **tx) - .await - .map_err(db_err)?; + let tenant_id: Option = + sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1") + .bind(id) + .fetch_one(&mut **tx) + .await + .map_err(db_err)?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, diff --git a/src/main.rs b/src/main.rs index f323602..d71a1ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,8 +46,7 @@ async fn main() -> anyhow::Result<()> { let cache = init_cache(&cfg.cache).await?; - let mut state = - state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer, cache); + let mut state = state::AppState::new(pool, cfg.clone(), active_keys, certificate_issuer, cache); if cfg.events.enabled() { let publisher = events::publisher::AmqpPublisher::connect(&cfg.events) .await From a0fabea2c3f40705ba1caf2753e5cddf1f1956bd Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Mon, 3 Aug 2026 10:55:13 +0300 Subject: [PATCH 16/19] Close the logout cache-barrier gap before the revoke commits logout cleared the session cache barrier before its revoke transaction committed, letting a concurrent authentication repopulate a stale "still valid" entry. Route both handlers through guarded_mutation so the barrier only clears after the commit. Also drops the unused, cache-unsafe revoke_session wrapper. --- src/graphql/auth.rs | 88 +++++++++++------------ src/identity/handlers.rs | 72 +++++++++---------- src/identity/repo.rs | 7 -- tests/m25_cache_invalidation.rs | 121 ++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 88 deletions(-) diff --git a/src/graphql/auth.rs b/src/graphql/auth.rs index 1f22335..dff9026 100644 --- a/src/graphql/auth.rs +++ b/src/graphql/auth.rs @@ -88,53 +88,55 @@ impl AuthMutation { let auth = require_auth(ctx)?; let state = ctx.data::()?; - let mut tx = state - .pool - .begin() - .await - .map_err(|e| gql_error(crate::error::db_err(e)))?; - if let Some(session_id) = auth.session_id { - // The revoke must stay inside `tx` so it commits atomically with - // the audit event/outbox row below, but it also needs the cache - // barrier around it so a concurrent reader can't repopulate a - // stale (not-yet-revoked) session entry — see `src/cache/mod.rs`. - let session_key = crate::cache::keys::session(session_id); - if let Some(cache) = state.cache.as_deref() { - cache - .begin( - crate::cache::CacheCategory::Session, - std::slice::from_ref(&session_key), - ) + let event = audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: auth.tenant_id, + target_kind: Some("entity"), + target_id: Some(auth.entity_id), + event: "auth.logout", + outcome: AuditOutcome::Allow, + details: serde_json::json!({}), + }; + + match auth.session_id { + Some(session_id) => { + // The barrier must span the revoke *and* its commit, not just + // the revoke: releasing it before `commit_with_audit`'s + // internal `tx.commit()` lets a concurrent reader load the + // still-uncommitted (pre-revoke) row from Postgres and + // repopulate the cache with it, right after the barrier + // that was supposed to block exactly that — see + // `src/cache/mod.rs`. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + repo::revoke_session_in_tx(&mut tx, session_id).await?; + audit::commit_with_audit( + &state.pool, + tx, + state.config.events.enabled(), + &event, + ) + .await + }, + ) + .await + .map_err(gql_error)?; + } + None => { + let tx = state + .pool + .begin() + .await + .map_err(|e| gql_error(crate::error::db_err(e)))?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) .await .map_err(gql_error)?; } - let result = repo::revoke_session_in_tx(&mut tx, session_id).await; - if let Some(cache) = state.cache.as_deref() { - cache - .end( - crate::cache::CacheCategory::Session, - std::slice::from_ref(&session_key), - ) - .await; - } - result.map_err(gql_error)?; } - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: auth.tenant_id, - target_kind: Some("entity"), - target_id: Some(auth.entity_id), - event: "auth.logout", - outcome: AuditOutcome::Allow, - details: serde_json::json!({}), - }, - ) - .await - .map_err(gql_error)?; Ok(true) } diff --git a/src/identity/handlers.rs b/src/identity/handlers.rs index d74493d..e2d8c6a 100644 --- a/src/identity/handlers.rs +++ b/src/identity/handlers.rs @@ -242,47 +242,43 @@ pub async fn logout( State(state): State, auth: AuthContext, ) -> Result { - let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - if let Some(session_id) = auth.session_id { - // See `src/graphql/auth.rs`'s `logout` for why this is inlined - // rather than going through `guarded_mutation`: the revoke must stay - // inside `tx` for outbox atomicity, but still needs the cache - // barrier around it. - let session_key = crate::cache::keys::session(session_id); - if let Some(cache) = state.cache.as_deref() { - cache - .begin( - crate::cache::CacheCategory::Session, - std::slice::from_ref(&session_key), - ) - .await?; + let event = audit::AuditEvent { + actor_entity_id: Some(auth.entity_id), + tenant_id: auth.tenant_id, + target_kind: Some("entity"), + target_id: Some(auth.entity_id), + event: "auth.logout", + outcome: AuditOutcome::Allow, + details: serde_json::json!({}), + }; + + match auth.session_id { + Some(session_id) => { + // The barrier must span the revoke *and* its commit, not just + // the revoke: releasing it before `commit_with_audit`'s internal + // `tx.commit()` lets a concurrent reader load the still- + // uncommitted (pre-revoke) row from Postgres and repopulate the + // cache with it, right after the barrier that was supposed to + // block exactly that — see `src/cache/mod.rs`. + crate::cache::invalidate::guarded_mutation( + state.cache.as_deref(), + crate::cache::CacheCategory::Session, + std::slice::from_ref(&crate::cache::keys::session(session_id)), + || async { + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + repo::revoke_session_in_tx(&mut tx, session_id).await?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) + .await + }, + ) + .await?; } - let result = repo::revoke_session_in_tx(&mut tx, session_id).await; - if let Some(cache) = state.cache.as_deref() { - cache - .end( - crate::cache::CacheCategory::Session, - std::slice::from_ref(&session_key), - ) - .await; + None => { + let tx = state.pool.begin().await.map_err(crate::error::db_err)?; + audit::commit_with_audit(&state.pool, tx, state.config.events.enabled(), &event) + .await?; } - result?; } - audit::commit_with_audit( - &state.pool, - tx, - state.config.events.enabled(), - &audit::AuditEvent { - actor_entity_id: Some(auth.entity_id), - tenant_id: auth.tenant_id, - target_kind: Some("entity"), - target_id: Some(auth.entity_id), - event: "auth.logout", - outcome: AuditOutcome::Allow, - details: serde_json::json!({}), - }, - ) - .await?; let mut response = Json(serde_json::json!({"authenticated": false})).into_response(); response.headers_mut().append( header::SET_COOKIE, diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 0b3d54c..06e4d00 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -1108,13 +1108,6 @@ pub async fn get_session(pool: &PgPool, id: Uuid) -> Result { }) } -pub async fn revoke_session(pool: &PgPool, id: Uuid) -> Result<(), AppError> { - let mut tx = pool.begin().await.map_err(db_err)?; - revoke_session_in_tx(&mut tx, id).await?; - tx.commit().await.map_err(db_err)?; - Ok(()) -} - /// The caller owns the commit, so a logout can bind the session revocation and /// its `auth.logout` event into one transaction. pub async fn revoke_session_in_tx( diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 8f1df4f..0857b50 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -512,6 +512,127 @@ async fn session_revoke_immediately_rejects_the_next_authentication() { ); } +/// Regression test for a review finding: `logout` used to clear the session's +/// cache barrier (`cache.end`) before `audit::commit_with_audit`'s internal +/// `tx.commit()` actually landed the revoke. In that window a concurrent +/// authentication would see a clean (non-dirty) barrier and a Postgres row +/// that (via MVCC, on a separate connection) still read as not revoked, and +/// would repopulate the cache with a "still valid" entry that then survived +/// for the full session TTL — defeating immediate revocation for exactly the +/// operation most likely to be relied on for it. +/// +/// A black-box race against real `authenticate_token` calls turned out not to +/// reliably land in the gap — it's sub-millisecond, and a full auth call has +/// too much of its own latency to consistently hit it. Instead this polls the +/// barrier's raw `dirty` flag and a direct `revoked_at` read, on a real OS +/// thread running in parallel with `logout` (`flavor = "multi_thread"`, not +/// cooperative async interleaving), so it observes the transition rather than +/// racing to beat it. The invariant: once `dirty` is observed set for this +/// key, it must never be observed clear again while `revoked_at` is still +/// null — that combination is only reachable if the barrier was released +/// before the commit landed. Confirmed this fails against the pre-fix +/// ordering (reverted locally) and passes against the fix. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] +async fn logout_cannot_leave_a_stale_valid_session_cached_during_the_revoke() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let entity_id = active_entity(&p, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + None, + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session/entity-status cache. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + + let logout_auth = AuthContext { + session_id: Some(session_id), + ..auth_context(entity_id, cache.clone()) + }; + let schema = build_schema(state.clone()); + let session_key = atom::cache::keys::session(session_id); + + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop2 = stop.clone(); + let p_poll = p.clone(); + let key_poll = session_key.clone(); + let poller = tokio::spawn(async move { + let mut conn = raw_redis_conn().await; + let mut saw_dirty = false; + loop { + let dirty: Option = redis::cmd("HGET") + .arg(&key_poll) + .arg("dirty") + .query_async(&mut conn) + .await + .unwrap_or(None); + let is_dirty = dirty.as_deref() == Some("1"); + if is_dirty { + saw_dirty = true; + } + if saw_dirty && !is_dirty { + let revoked_at: Option> = + sqlx::query_scalar("SELECT revoked_at FROM sessions WHERE id = $1") + .bind(session_id) + .fetch_one(&p_poll) + .await + .expect("read revoked_at"); + // Barrier just went clean; if the row isn't committed as + // revoked yet, that's the bug. If it's already revoked, the + // ordering was correct — nothing more to catch. + return revoked_at.is_none(); + } + if stop2.load(std::sync::atomic::Ordering::Relaxed) { + return false; + } + } + }); + + let logout_result = schema + .execute(Request::new("mutation { logout }").data(logout_auth)) + .await; + assert!( + logout_result.errors.is_empty(), + "logout failed: {:?}", + logout_result.errors + ); + + stop.store(true, std::sync::atomic::Ordering::Relaxed); + let violation = poller.await.expect("join poller"); + assert!( + !violation, + "observed the session cache barrier clear (dirty -> clean) while the session row still \ + read as un-revoked — a concurrent reader landing at that exact moment would repopulate \ + the cache with a stale \"still valid\" entry, exactly the bug this regresses" + ); + + let result = auth::authenticate_token(&state, &token).await; + assert!( + result.is_err(), + "revoked session must be rejected on the very next authentication, not after a TTL" + ); +} + #[tokio::test] #[ignore] async fn entity_deactivation_immediately_rejects_an_existing_valid_session() { From 29a6563e7ef4cacb9b5aac143d864e08d1a5ae75 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Mon, 3 Aug 2026 11:30:53 +0300 Subject: [PATCH 17/19] Establish entity/tenant delete cache barriers before the status flip deactivate_entity_and_collect_revocation_ids_in_tx and its tenant counterpart flipped status in the same step used to lock and enumerate, so the cache barrier only went up after the row was already inactive. A concurrent request in that window could take a full cache hit on the stale active entity/session/credential entries and keep running past the delete's commit. Split each into a lock-and-enumerate step (run before the barrier) and a deactivate-and-finish step (run after). Also fixes entity.delete's audit tenant_id: the post-commit read used get_entity, which filters deleted_at IS NULL and always missed the just-deleted row. The tenant_id is now captured inside the transaction during the status-flip UPDATE and returned to the caller instead. --- src/graphql/tenants.rs | 20 +++--- src/identity/repo.rs | 119 +++++++++++++++++++------------- src/identity/service.rs | 35 ++++++---- src/tenants/repo.rs | 115 +++++++++++++++++------------- tests/m25_cache_invalidation.rs | 42 +++++++---- 5 files changed, 197 insertions(+), 134 deletions(-) diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 1f79d93..446d974 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -414,8 +414,10 @@ impl TenantMutation { // treatment — `restore_tenant`'s own invalidation (see // `reactivate_tenant_and_collect_credential_ids_in_tx`) already // covers that side by the time a restore could matter. See - // `deactivate_tenant_and_collect_session_ids_in_tx` for why - // session ids are enumerated inside the same locked transaction. + // `lock_tenant_and_collect_session_ids_in_tx` for why session ids + // are enumerated inside the same locked transaction, and why that + // lock-and-enumerate step must run before the barrier below is + // established, not after the tenant is flipped to `deleted`. let Some(cache) = state.cache.as_deref() else { tenant_repo::soft_delete_tenant_with_audit( &state.pool, @@ -428,13 +430,8 @@ impl TenantMutation { return Ok(()); }; let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; - let (_tenant, session_ids) = - tenant_repo::deactivate_tenant_and_collect_session_ids_in_tx( - &mut tx, - tenant_id, - Some(auth.entity_id), - ) - .await?; + let session_ids = + tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id).await?; let session_keys: Vec = session_ids .iter() .map(|id| crate::cache::keys::session(*id)) @@ -448,15 +445,16 @@ impl TenantMutation { (crate::cache::CacheCategory::Session, &session_keys), ]; crate::cache::invalidate::begin_all(cache, &groups).await?; - let outcome = tenant_repo::finish_tenant_soft_delete_in_tx( + let outcome = tenant_repo::deactivate_and_finish_tenant_soft_delete_in_tx( &mut tx, state.config.events.enabled(), Some(auth.entity_id), + Some(auth.entity_id), tenant_id, ) .await; let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(crate::error::db_err), + Ok(_tenant) => tx.commit().await.map_err(crate::error::db_err), Err(err) => Err(err), }; crate::cache::invalidate::end_all(cache, &groups).await; diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 06e4d00..0932f7d 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -731,39 +731,38 @@ pub async fn entity_active_access_token_ids( .map_err(db_err) } -/// Flips the entity to inactive/tombstoned and, in the *same* transaction, -/// enumerates the exact session and access-token credential ids this delete -/// is about to revoke. Callers invalidate `atom:v1:session:*` / -/// `atom:v1:credential:*` cache entries for these before calling -/// [`finish_entity_deletion_in_tx`]. +/// Locks the entity row and, in the *same* transaction, enumerates the exact +/// session and access-token credential ids a subsequent delete is about to +/// revoke — *before* the entity is touched. Callers establish the cache +/// barrier on these ids (plus the entity's own `entity_status` key) before +/// calling [`deactivate_and_finish_entity_deletion_in_tx`]: starting the +/// barrier only after the status flip would leave a window where a +/// concurrent request can still take a full cache hit on the pre-delete +/// entity/session/credential entries and keep running past the point the +/// delete commits. /// -/// The status-flip `UPDATE` below takes an exclusive row lock on the entity — -/// the same lock `create_session`/`create_access_token` take (via +/// The `SELECT ... FOR UPDATE` below takes the same exclusive row lock on the +/// entity that `create_session`/`create_access_token` take (via /// `lock_active_entity`) before inserting. So a session or access token -/// created concurrently for this entity either committed before this `UPDATE` -/// acquired the lock (and is therefore visible to the enumeration below, -/// which runs after it, in the same transaction) or is blocked until this -/// transaction commits (and then fails, since the entity is no longer -/// active). Enumerating via a plain pre-transaction pool query — the previous -/// shape of this code — could miss a session/credential created in that -/// window, leaving its cache entry uninvalidated indefinitely. See +/// created concurrently for this entity either committed before this lock was +/// acquired (and is therefore visible to the enumeration below, which runs +/// after it, in the same transaction) or is blocked until this transaction +/// commits (and then fails, since the entity is no longer active by then). +/// Enumerating via a plain pre-transaction pool query — the previous shape of +/// this code — could miss a session/credential created in that window, +/// leaving its cache entry uninvalidated indefinitely. See /// `src/cache/mod.rs`'s consistency model. -pub async fn deactivate_entity_and_collect_revocation_ids_in_tx( +pub async fn lock_entity_and_collect_revocation_ids_in_tx( tx: &mut Transaction<'_, Postgres>, id: Uuid, - deleted_by: Option, ) -> Result<(Vec, Vec), AppError> { - let result = sqlx::query( - "UPDATE entities - SET status = 'inactive', deleted_at = now(), deleted_by = $2, updated_at = now() - WHERE id = $1 AND deleted_at IS NULL", - ) - .bind(id) - .bind(deleted_by) - .execute(&mut **tx) - .await - .map_err(db_err)?; - if result.rows_affected() == 0 { + let locked = + sqlx::query("SELECT id FROM entities WHERE id = $1 AND deleted_at IS NULL FOR UPDATE") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + if locked.is_none() { return Err(AppError::not_found(format!("entity {id} not found"))); } @@ -789,17 +788,37 @@ pub async fn deactivate_entity_and_collect_revocation_ids_in_tx( } /// Finishes the entity soft-delete started by -/// [`deactivate_entity_and_collect_revocation_ids_in_tx`] in the same -/// transaction: revokes active credentials (all kinds) and sessions, and -/// tombstones the email. Does not commit — the caller commits after this -/// succeeds, once the cache barrier established on the enumerated ids covers -/// the whole transaction. -pub async fn finish_entity_deletion_in_tx( +/// [`lock_entity_and_collect_revocation_ids_in_tx`] in the same transaction: +/// flips the entity to inactive/tombstoned, revokes active credentials (all +/// kinds) and sessions, and tombstones the email. Does not commit — the +/// caller commits after this succeeds, once the cache barrier established on +/// the enumerated ids covers the whole transaction. +/// +/// Returns the entity's `tenant_id`, captured here (before the tombstone) +/// rather than left for the caller to re-derive post-commit: `get_entity` +/// filters `deleted_at IS NULL`, so a re-read after this commits always +/// misses the row and silently drops the tenant from the delete's audit +/// trail. +pub async fn deactivate_and_finish_entity_deletion_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, + deleted_by: Option, id: Uuid, -) -> Result<(), AppError> { +) -> Result, AppError> { + let tenant_id: Option = sqlx::query_scalar( + "UPDATE entities + SET status = 'inactive', deleted_at = now(), deleted_by = $2, updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING tenant_id", + ) + .bind(id) + .bind(deleted_by) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .ok_or_else(|| AppError::not_found(format!("entity {id} not found")))?; + let revoked_certificates: i64 = sqlx::query_scalar( r#"WITH revoked AS ( UPDATE credentials @@ -841,12 +860,6 @@ pub async fn finish_entity_deletion_in_tx( .await .map_err(db_err)?; - let tenant_id: Option = - sqlx::query_scalar("SELECT tenant_id FROM entities WHERE id = $1") - .bind(id) - .fetch_one(&mut **tx) - .await - .map_err(db_err)?; let meta = crate::audit::AuditMeta { actor_entity_id: actor_id, tenant_id, @@ -855,7 +868,7 @@ pub async fn finish_entity_deletion_in_tx( event: "entity.delete", }; crate::audit::observe_in_tx(tx, events_enabled, &meta, &serde_json::json!({})).await?; - Ok(()) + Ok(tenant_id) } pub async fn delete_entity( @@ -873,10 +886,10 @@ pub async fn delete_entity( /// explicit. /// /// Used directly only when no cache is configured; the cache-aware path -/// (`graphql::entities::delete_entity`) calls -/// [`deactivate_entity_and_collect_revocation_ids_in_tx`] and -/// [`finish_entity_deletion_in_tx`] itself so it can establish the cache -/// barrier between them. +/// (`identity::service::delete_entity`) calls +/// [`lock_entity_and_collect_revocation_ids_in_tx`] and +/// [`deactivate_and_finish_entity_deletion_in_tx`] itself so it can establish +/// the cache barrier between them. pub async fn delete_entity_with_audit( pool: &PgPool, events_enabled: bool, @@ -885,14 +898,22 @@ pub async fn delete_entity_with_audit( deleted_by: Option, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by).await?; - finish_entity_deletion_in_tx(&mut tx, events_enabled, actor_id, id).await?; + lock_entity_and_collect_revocation_ids_in_tx(&mut tx, id).await?; + let tenant_id = deactivate_and_finish_entity_deletion_in_tx( + &mut tx, + events_enabled, + actor_id, + deleted_by, + id, + ) + .await?; tx.commit().await.map_err(db_err)?; // The audit_logs row is deliberately written after commit (fire-and-forget, // never blocks an already-valid delete) — see `audit::commit_with_audit`'s // doc comment. The outbox row, by contrast, went in atomically with the - // mutation above via `observe_in_tx`. - let tenant_id = get_entity(pool, id).await.ok().and_then(|e| e.tenant_id); + // mutation above via `observe_in_tx`. `tenant_id` is captured inside the + // transaction above (before the tombstone) rather than re-read here — a + // post-commit `get_entity` would always miss the now-deleted row. crate::audit::write( pool, false, diff --git a/src/identity/service.rs b/src/identity/service.rs index 9aa19db..13c9b6a 100644 --- a/src/identity/service.rs +++ b/src/identity/service.rs @@ -2291,7 +2291,7 @@ fn make_shared_key(cred_id: Uuid) -> String { /// invalidation, a stale cached session or credential becomes a full hit /// again the moment a later restore repopulates `entity_status` as active — /// despite staying revoked in Postgres. See -/// [`super::repo::deactivate_entity_and_collect_revocation_ids_in_tx`] for why the +/// [`super::repo::lock_entity_and_collect_revocation_ids_in_tx`] for why the /// ids are enumerated inside the same locked transaction. pub async fn delete_entity( pool: &PgPool, @@ -2312,9 +2312,16 @@ pub async fn delete_entity( }; let mut tx = pool.begin().await.map_err(db_err)?; + // Lock and enumerate *before* establishing the barrier: the entity is + // still fully active at this point, so a concurrent cache read has + // nothing dirty to react to yet. Only once the barrier below is up does + // a concurrent read stop trusting the (about-to-be-stale) cache entries + // and fall through to Postgres — flipping status first would leave a + // window where a concurrent request keeps taking full cache hits on the + // pre-delete entity/session/credential entries and runs past the point + // this delete commits. let (session_ids, credential_ids) = - super::repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, id, deleted_by) - .await?; + super::repo::lock_entity_and_collect_revocation_ids_in_tx(&mut tx, id).await?; let session_keys: Vec = session_ids .iter() .copied() @@ -2335,23 +2342,27 @@ pub async fn delete_entity( (crate::cache::CacheCategory::Credential, &credential_keys), ]; crate::cache::invalidate::begin_all(cache, &groups).await?; - let outcome = - super::repo::finish_entity_deletion_in_tx(&mut tx, events_enabled, deleted_by, id).await; + let outcome = super::repo::deactivate_and_finish_entity_deletion_in_tx( + &mut tx, + events_enabled, + deleted_by, + deleted_by, + id, + ) + .await; let outcome = match outcome { - Ok(()) => tx.commit().await.map_err(db_err), + Ok(tenant_id) => tx.commit().await.map_err(db_err).map(|_| tenant_id), Err(err) => Err(err), }; crate::cache::invalidate::end_all(cache, &groups).await; - outcome?; + let tenant_id = outcome?; // Mirrors `delete_entity_with_audit`'s own post-commit audit write // (fire-and-forget, after the mutation durably commits) — see // `audit::commit_with_audit`'s doc comment. Only needed on this locked // path; the cache-disabled fallback above already gets it from - // `delete_entity_with_audit` itself. - let tenant_id = super::repo::get_entity(pool, id) - .await - .ok() - .and_then(|e| e.tenant_id); + // `delete_entity_with_audit` itself. `tenant_id` comes from the captured + // value inside the transaction — a post-commit `get_entity` would always + // miss the now-deleted row. crate::audit::write( pool, false, diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index dffed4f..f98d291 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -547,43 +547,41 @@ pub async fn update_tenant( update_tenant_with_audit(pool, false, None, id, req, updated_by).await } -/// Flips the tenant to `deleted` and, in the *same* transaction, enumerates -/// the exact active session ids of its member entities that -/// [`finish_tenant_soft_delete_in_tx`] is about to revoke. Callers invalidate -/// `atom:v1:session:*` cache entries for these before calling it. +/// Locks the tenant row and, in the *same* transaction, enumerates the exact +/// active session ids of its member entities that a subsequent delete is +/// about to revoke — *before* the tenant is touched. Callers establish the +/// cache barrier on these ids (plus the tenant's own `tenant_status` key) +/// before calling [`deactivate_and_finish_tenant_soft_delete_in_tx`]: starting +/// the barrier only after the status flip would leave a window where a +/// concurrent request can still take a full cache hit on the pre-delete +/// tenant-status/session entries and keep running past the point the delete +/// commits. /// -/// The status-flip `UPDATE` below takes an exclusive row lock on the tenant — -/// the same lock `lock_active_tenant`/`lock_optional_active_tenant` take +/// The `SELECT ... FOR UPDATE` below takes the same exclusive row lock on the +/// tenant that `lock_active_tenant`/`lock_optional_active_tenant` take /// (transitively, via `lock_active_entity`) before any session or credential /// can be created for an entity in this tenant. So a session created -/// concurrently for a member entity either committed before this `UPDATE` -/// acquired the lock (and is therefore visible to the enumeration below, -/// which runs after it, in the same transaction) or is blocked until this -/// transaction commits (and then fails, since the tenant is no longer -/// active). Enumerating via a plain pre-transaction pool query — the previous -/// shape of this code — could miss a session created in that window, leaving -/// its cache entry uninvalidated indefinitely. See `src/cache/mod.rs`'s +/// concurrently for a member entity either committed before this lock was +/// acquired (and is therefore visible to the enumeration below, which runs +/// after it, in the same transaction) or is blocked until this transaction +/// commits (and then fails, since the tenant is no longer active by then). +/// Enumerating via a plain pre-transaction pool query — the previous shape of +/// this code — could miss a session created in that window, leaving its +/// cache entry uninvalidated indefinitely. See `src/cache/mod.rs`'s /// consistency model. -pub async fn deactivate_tenant_and_collect_session_ids_in_tx( +pub async fn lock_tenant_and_collect_session_ids_in_tx( tx: &mut Transaction<'_, Postgres>, id: Uuid, - deleted_by: Option, -) -> Result<(Tenant, Vec), AppError> { - let tenant = sqlx::query_as::<_, Tenant>(&format!( - r#"UPDATE tenants - SET status = 'deleted', deleted_at = now(), deleted_by = $2, - updated_by = $2, updated_at = now() - WHERE id = $1 AND deleted_at IS NULL - RETURNING {TENANT_COLS}"#, - )) - .bind(id) - .bind(deleted_by) - .fetch_one(&mut **tx) - .await - .map_err(|e| match e { - sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), - other => AppError::Database(other), - })?; +) -> Result, AppError> { + let locked = + sqlx::query("SELECT id FROM tenants WHERE id = $1 AND deleted_at IS NULL FOR UPDATE") + .bind(id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)?; + if locked.is_none() { + return Err(AppError::not_found(format!("tenant {id} not found"))); + } let session_ids: Vec = sqlx::query_scalar( r#"SELECT id FROM sessions @@ -595,21 +593,38 @@ pub async fn deactivate_tenant_and_collect_session_ids_in_tx( .await .map_err(db_err)?; - Ok((tenant, session_ids)) + Ok(session_ids) } /// Finishes the tenant soft-delete started by -/// [`deactivate_tenant_and_collect_session_ids_in_tx`] in the same -/// transaction: revokes every active credential and session belonging to the -/// tenant's entities. Does not commit — the caller commits after this -/// succeeds, once the cache barrier established on the enumerated session ids -/// covers the whole transaction. -pub async fn finish_tenant_soft_delete_in_tx( +/// [`lock_tenant_and_collect_session_ids_in_tx`] in the same transaction: +/// flips the tenant to `deleted`, and revokes every active credential and +/// session belonging to the tenant's entities. Does not commit — the caller +/// commits after this succeeds, once the cache barrier established on the +/// enumerated session ids covers the whole transaction. +pub async fn deactivate_and_finish_tenant_soft_delete_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, + deleted_by: Option, id: Uuid, -) -> Result<(), AppError> { +) -> Result { + let tenant = sqlx::query_as::<_, Tenant>(&format!( + r#"UPDATE tenants + SET status = 'deleted', deleted_at = now(), deleted_by = $2, + updated_by = $2, updated_at = now() + WHERE id = $1 AND deleted_at IS NULL + RETURNING {TENANT_COLS}"#, + )) + .bind(id) + .bind(deleted_by) + .fetch_one(&mut **tx) + .await + .map_err(|e| match e { + sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), + other => AppError::Database(other), + })?; + // Stamp the tenant-delete marker on every revoked credential (not just // certificates) so restore_tenant can reverse exactly the revocations this // delete caused, without disturbing credentials revoked earlier for other @@ -657,7 +672,7 @@ pub async fn finish_tenant_soft_delete_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant) } /// Soft-delete a tenant: mark `status = deleted`, stamp the tombstone, and @@ -674,9 +689,9 @@ pub async fn soft_delete_tenant( /// Used directly only when no cache is configured; the cache-aware path /// (`graphql::tenants::delete_tenant`) calls -/// [`deactivate_tenant_and_collect_session_ids_in_tx`] and -/// [`finish_tenant_soft_delete_in_tx`] itself so it can establish the cache -/// barrier between them. +/// [`lock_tenant_and_collect_session_ids_in_tx`] and +/// [`deactivate_and_finish_tenant_soft_delete_in_tx`] itself so it can +/// establish the cache barrier between them. pub async fn soft_delete_tenant_with_audit( pool: &PgPool, events_enabled: bool, @@ -685,9 +700,15 @@ pub async fn soft_delete_tenant_with_audit( deleted_by: Option, ) -> Result { let mut tx = pool.begin().await.map_err(db_err)?; - let (tenant, _session_ids) = - deactivate_tenant_and_collect_session_ids_in_tx(&mut tx, id, deleted_by).await?; - finish_tenant_soft_delete_in_tx(&mut tx, events_enabled, actor_id, id).await?; + lock_tenant_and_collect_session_ids_in_tx(&mut tx, id).await?; + let tenant = deactivate_and_finish_tenant_soft_delete_in_tx( + &mut tx, + events_enabled, + actor_id, + deleted_by, + id, + ) + .await?; tx.commit().await.map_err(db_err)?; Ok(tenant) } @@ -725,7 +746,7 @@ pub async fn soft_delete_tenant_with_audit( /// so the enumeration that follows it (in the same transaction) sees a /// consistent snapshot with respect to any concurrent restore/purge of the /// same tenant — mirroring the same lock-then-enumerate shape used by -/// [`deactivate_tenant_and_collect_session_ids_in_tx`], even though (unlike +/// [`lock_tenant_and_collect_session_ids_in_tx`], even though (unlike /// that case) no *new* matching credential can appear here: only /// `soft_delete_tenant` ever stamps the `tenant_deleted` revocation reason, /// and it cannot run again against an already-deleted tenant. diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 0857b50..2b3d740 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -1173,10 +1173,9 @@ async fn entity_delete_then_restore_immediately_rejects_a_pre_existing_access_to /// the_group_subject_lock` below), just previously unfixed for /// sessions/credentials. /// -/// Fixed by moving the enumeration inside the same transaction as the -/// status-flip `UPDATE` -/// (`identity::repo::deactivate_entity_and_collect_revocation_ids_in_tx`), -/// which takes an exclusive lock on the entity row — the same lock +/// Fixed by moving the enumeration inside the same transaction as the lock +/// (`identity::repo::lock_entity_and_collect_revocation_ids_in_tx`), which +/// takes an exclusive lock on the entity row — the same lock /// `create_session`/`create_access_token` take (via `lock_active_entity`) /// before inserting. As with the group-membership fix, an end-to-end test /// racing a real `create_session` against a real `deleteEntity` mutation @@ -1195,9 +1194,9 @@ async fn concurrent_session_creation_cannot_evade_the_entity_delete_enumeration( let mut tx = p.begin().await.expect("begin tx"); let (session_ids, credential_ids) = - identity_repo::deactivate_entity_and_collect_revocation_ids_in_tx(&mut tx, entity, None) + identity_repo::lock_entity_and_collect_revocation_ids_in_tx(&mut tx, entity) .await - .expect("lock, deactivate, and enumerate"); + .expect("lock and enumerate"); assert!( session_ids.is_empty() && credential_ids.is_empty(), "no sessions/credentials exist yet for this fresh entity" @@ -1218,6 +1217,12 @@ async fn concurrent_session_creation_cannot_evade_the_entity_delete_enumeration( not commit a session the enumeration has already missed" ); + // Mirrors the production call site's order: the status-flip runs after + // the lock-and-enumerate step (which the barrier, in production, is + // established between), still inside the same transaction as the lock. + identity_repo::deactivate_and_finish_entity_deletion_in_tx(&mut tx, false, None, None, entity) + .await + .expect("deactivate and finish"); tx.commit().await.expect("commit lock-holding tx"); // Once committed, the entity is deactivated (the same effect @@ -1239,11 +1244,11 @@ async fn concurrent_session_creation_cannot_evade_the_entity_delete_enumeration( /// transaction/lock was taken, so a session created for a member entity in /// the window between enumeration and the tenant's own bulk revoke was never /// included in the cache barrier. Fixed by moving the enumeration inside the -/// same transaction as the tenant's status-flip `UPDATE` -/// (`tenants::repo::deactivate_tenant_and_collect_session_ids_in_tx`), which -/// takes an exclusive lock on the tenant row — the same lock -/// `lock_active_entity` takes (via `lock_optional_active_tenant`) before any -/// session/credential can be created for *any* entity in the tenant. +/// same transaction as the lock +/// (`tenants::repo::lock_tenant_and_collect_session_ids_in_tx`), which takes +/// an exclusive lock on the tenant row — the same lock `lock_active_entity` +/// takes (via `lock_optional_active_tenant`) before any session/credential +/// can be created for *any* entity in the tenant. #[tokio::test] #[ignore] async fn concurrent_session_creation_cannot_evade_the_tenant_delete_enumeration() { @@ -1252,10 +1257,9 @@ async fn concurrent_session_creation_cannot_evade_the_tenant_delete_enumeration( let entity = active_entity_in_tenant(&p, tenant_id, "service").await; let mut tx = p.begin().await.expect("begin tx"); - let (_tenant, session_ids) = - tenant_repo::deactivate_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id, None) - .await - .expect("lock, deactivate, and enumerate"); + let session_ids = tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id) + .await + .expect("lock and enumerate"); assert!( session_ids.is_empty(), "no sessions exist yet for this fresh tenant's member" @@ -1272,6 +1276,14 @@ async fn concurrent_session_creation_cannot_evade_the_tenant_delete_enumeration( delete's enumeration holds, not commit a session the enumeration has already missed" ); + // Mirrors the production call site's order: the status-flip runs after + // the lock-and-enumerate step (which the barrier, in production, is + // established between), still inside the same transaction as the lock. + tenant_repo::deactivate_and_finish_tenant_soft_delete_in_tx( + &mut tx, false, None, None, tenant_id, + ) + .await + .expect("deactivate and finish"); tx.commit().await.expect("commit lock-holding tx"); // Once committed, the tenant is deleted (so `lock_active_entity`'s own From 86d90992528cc63013f9f215a543b924c89fc760 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Mon, 3 Aug 2026 12:34:47 +0300 Subject: [PATCH 18/19] Make the dirty barrier nesting-aware and restore missing observe-path logs Two overlapping mutations on the same cache key could have the first end clear the barrier while the second was still in flight, letting a reader repopulate stale data with no correction if the second mutation's own end was ever delayed or lost. dirty is now a nesting counter (incremented by begin, decremented by end) instead of a 0/1 flag, so the barrier only reads clean once every overlapping mutation has ended. Also restores the post-commit stdout observability log that several _with_audit wrappers and their GraphQL cache-enabled resolver branches dropped when they were split into _in_tx/_with_audit pairs: role, role_assignment, direct_policy, tenant, and group mutations were enqueuing their outbox event but never logging success. Delete-style _in_tx functions now return the tenant_id they capture pre-commit so the caller can log accurately without a re-read that would miss the now-deleted row. --- src/authz/repo.rs | 117 +++++++++++++++++---- src/cache/mod.rs | 224 ++++++++++++++++++++++++++++++---------- src/graphql/groups.rs | 70 ++++++++++++- src/graphql/policies.rs | 83 +++++++++++++++ src/graphql/tenants.rs | 8 ++ src/identity/repo.rs | 74 ++++++++++--- src/tenants/repo.rs | 10 ++ 7 files changed, 502 insertions(+), 84 deletions(-) diff --git a/src/authz/repo.rs b/src/authz/repo.rs index b813165..a056b55 100644 --- a/src/authz/repo.rs +++ b/src/authz/repo.rs @@ -1230,7 +1230,7 @@ pub async fn replace_role_permission_block_links_with_audit( permission_block_ids: &[Uuid], ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - replace_role_permission_block_links_in_tx( + let tenant_id = replace_role_permission_block_links_in_tx( &mut tx, events_enabled, actor_id, @@ -1238,7 +1238,21 @@ pub async fn replace_role_permission_block_links_with_audit( permission_block_ids, ) .await?; - tx.commit().await.map_err(db_err) + tx.commit().await.map_err(db_err)?; + // `_in_tx` already enqueued the outbox row via `observe_in_tx` before + // returning — this is the post-commit stdout observability log + // `commit_with_observation` would otherwise provide. + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role", + target_id: Some(role_id), + event: "role.permission_blocks.replace", + }, + &serde_json::json!({ "permission_block_ids": permission_block_ids }), + ); + Ok(()) } /// Body of [`replace_role_permission_block_links`]; caller contract per @@ -1250,13 +1264,17 @@ pub async fn replace_role_permission_block_links_with_audit( /// below is only meaningful under the role lock this transaction holds, and a /// second connection acquired mid-transaction is a pool-exhaustion deadlock /// under concurrency. +/// Returns the role's `tenant_id`, captured here rather than left for the +/// caller to re-derive post-commit — cheaper than an extra query, and safe +/// for any future case where the role itself stops existing by the time the +/// caller wants to log. pub(crate) async fn replace_role_permission_block_links_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, role_id: Uuid, permission_block_ids: &[Uuid], -) -> Result<(), AppError> { +) -> Result, AppError> { let role_tenant_id: Option = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(role_id) @@ -1326,7 +1344,8 @@ pub(crate) async fn replace_role_permission_block_links_in_tx( }, &serde_json::json!({ "permission_block_ids": unique_block_ids }), ) - .await + .await?; + Ok(role_tenant_id) } async fn insert_role_permission_block( @@ -2827,20 +2846,34 @@ pub async fn delete_role_with_audit( deleted_by: Option, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - delete_role_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; - tx.commit().await.map_err(db_err) + let tenant_id = delete_role_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role", + target_id: Some(id), + event: "role.delete", + }, + &serde_json::json!({}), + ); + Ok(()) } /// Body of [`delete_role`]; caller contract per /// [`create_role_assignment_in_tx`] — the resolver must already hold the -/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. +/// role lock via [`lock_role_and_collect_grants_keys`] on this `tx`. Returns +/// the role's `tenant_id`, captured here rather than left for the caller to +/// re-derive post-commit — a re-read after this commits would always miss +/// the now-deleted row. pub(crate) async fn delete_role_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, deleted_by: Option, -) -> Result<(), AppError> { +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM roles WHERE id = $1 AND deleted_at IS NULL") .bind(id) @@ -2871,7 +2904,7 @@ pub(crate) async fn delete_role_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant_id) } pub async fn delete_role( @@ -4074,6 +4107,16 @@ pub async fn create_role_assignment_with_audit( let mut tx = pool.begin().await.map_err(db_err)?; let assignment = create_role_assignment_in_tx(&mut tx, events_enabled, actor_id, req).await?; tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: assignment.tenant_id, + target_kind: "role_assignment", + target_id: Some(assignment.id), + event: "role_assignment.create", + }, + &serde_json::json!({}), + ); Ok(assignment) } @@ -4255,19 +4298,32 @@ pub async fn delete_role_assignment_with_audit( id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - delete_role_assignment_in_tx(&mut tx, events_enabled, actor_id, id).await?; - tx.commit().await.map_err(db_err) + let tenant_id = delete_role_assignment_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }, + &serde_json::json!({}), + ); + Ok(()) } /// Body of [`delete_role_assignment`]; caller contract per /// [`create_role_assignment_in_tx`] — the group-subject resolver path must -/// already hold the subject's group closure lock on this `tx`. +/// already hold the subject's group closure lock on this `tx`. Returns the +/// assignment's `tenant_id`, captured here rather than left for the caller +/// to re-derive post-commit — the row is gone by then. pub(crate) async fn delete_role_assignment_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, -) -> Result<(), AppError> { +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM role_assignments WHERE id = $1") .bind(id) @@ -4300,7 +4356,7 @@ pub(crate) async fn delete_role_assignment_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant_id) } pub async fn create_direct_policy_with_audit( @@ -4312,6 +4368,16 @@ pub async fn create_direct_policy_with_audit( let mut tx = pool.begin().await.map_err(db_err)?; let policy = create_direct_policy_in_tx(&mut tx, events_enabled, actor_id, req).await?; tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: policy.tenant_id, + target_kind: "direct_policy", + target_id: Some(policy.id), + event: "direct_policy.create", + }, + &serde_json::json!({}), + ); Ok(policy) } @@ -4446,18 +4512,31 @@ pub async fn delete_direct_policy_with_audit( id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - delete_direct_policy_in_tx(&mut tx, events_enabled, actor_id, id).await?; - tx.commit().await.map_err(db_err) + let tenant_id = delete_direct_policy_in_tx(&mut tx, events_enabled, actor_id, id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "direct_policy", + target_id: Some(id), + event: "direct_policy.delete", + }, + &serde_json::json!({}), + ); + Ok(()) } /// Body of [`delete_direct_policy`]; caller contract per -/// [`create_role_assignment_in_tx`]. +/// [`create_role_assignment_in_tx`]. Returns the policy's `tenant_id`, +/// captured here rather than left for the caller to re-derive post-commit — +/// the row is gone by then. pub(crate) async fn delete_direct_policy_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, -) -> Result<(), AppError> { +) -> Result, AppError> { let policy_tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM direct_policies WHERE id = $1") .bind(id) @@ -4491,7 +4570,7 @@ pub(crate) async fn delete_direct_policy_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant_id) } pub async fn delete_direct_policy(pool: &PgPool, id: Uuid) -> Result<(), AppError> { diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 4a7a247..dc29ccb 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -5,47 +5,64 @@ //! # Consistency model //! //! Every cached entry is a Redis hash with three fields: `v` (an integer -//! version, bumped on every mutation that can affect the entry), `dirty` -//! (`"1"` while a mutation is in flight, absent otherwise), and `p` (the -//! serialized payload, present only when the entry holds a valid value). +//! version, bumped on every mutation that can affect the entry), `dirty` (an +//! integer *nesting counter* — above zero while one or more mutations are in +//! flight, zero or absent otherwise), and `p` (the serialized payload, +//! present only when the entry holds a valid value). //! //! Three primitives, each a small atomic Lua script, implement a per-key -//! mutation barrier that prevents three races that a plain cache-aside + +//! mutation barrier that prevents four races that a plain cache-aside + //! post-commit `DEL` cannot: a read started *before* a mutation repopulating //! the cache with stale data after the mutation's invalidation ran, a read //! started *during* the mutation's dirty window doing the same once the -//! mutation finishes, and a lost invalidation silently resurrecting a -//! revoked value: +//! mutation finishes, a lost invalidation silently resurrecting a revoked +//! value, and — the reason `dirty` is a counter rather than a flag — two +//! overlapping mutations on the same key finishing at different times: //! -//! - `begin` — called before a security-sensitive Postgres mutation. Bumps the -//! version, marks the entry dirty, clears any payload, and bounds the -//! barrier itself with an expiry so a lost `end` call self-heals rather than -//! leaving the entry dirty forever. +//! - `begin` — called before a security-sensitive Postgres mutation. +//! Increments the version and the dirty counter, clears any payload, and +//! bounds the barrier itself with an expiry so a lost `end` call self-heals +//! rather than leaving the entry dirty forever. //! - `end` — called after the mutation (success or failure). Bumps the -//! version *again*, clears `dirty`, and clears any payload — the next reader -//! does a clean reload. The second version bump (beyond the one -//! `begin` already did) is what closes the dirty-window race below — it is -//! not merely resetting a flag. +//! version *again*, decrements the dirty counter, and clears any payload — +//! the next reader does a clean reload. The second version bump (beyond the +//! one `begin` already did) is what closes the dirty-window race below — it +//! is not merely decrementing a counter. //! - `try_populate` — called by a cache-miss read after it finishes loading -//! from Postgres. Writes the payload only if the entry is not dirty and its -//! version still matches what the reader observed before it started +//! from Postgres. Writes the payload only if the dirty counter is zero and +//! the version still matches what the reader observed before it started //! loading; otherwise the write is silently discarded. //! -//! The dirty-window race this second bump defeats (found by external review, -//! 2026-07-29 — the original `end` only cleared `dirty`, without -//! re-bumping the version): a reader's `lookup` can land *while* a mutation -//! is mid-flight (`dirty == "1"`), observe the *post-`begin`* version, then -//! proceed to load from Postgres — possibly reading the pre-mutation state, -//! since the mutation's own Postgres write may not have committed yet. If -//! `end` only cleared `dirty` without moving the version again, that -//! reader's later `try_populate` call, run after `end`, would find the -//! version *unchanged* since the moment it was observed and would succeed — -//! re-caching a stale value for the mutation's category for a full TTL, -//! exactly during a revoke/policy-change race. Bumping the version in `end` -//! too means any version a reader could have observed during the dirty -//! window is guaranteed stale by the time `end` finishes, so `try_populate` -//! always rejects it — whether it runs while still dirty (rejected by the -//! `dirty` check) or after `end` (rejected by the version check). +//! The dirty-window race the version's second bump defeats (found by external +//! review, 2026-07-29 — the original `end` only cleared `dirty`, without +//! re-bumping the version): a reader's `lookup` can land *while* a mutation is +//! mid-flight (`dirty > 0`), observe the *post-`begin`* version, then proceed +//! to load from Postgres — possibly reading the pre-mutation state, since the +//! mutation's own Postgres write may not have committed yet. If `end` only +//! cleared `dirty` without moving the version again, that reader's later +//! `try_populate` call, run after `end`, would find the version *unchanged* +//! since the moment it was observed and would succeed — re-caching a stale +//! value for the mutation's category for a full TTL, exactly during a +//! revoke/policy-change race. Bumping the version in `end` too means any +//! version a reader could have observed during the dirty window is +//! guaranteed stale by the time `end` finishes, so `try_populate` always +//! rejects it — whether it runs while still dirty (rejected by the `dirty` +//! check) or after `end` (rejected by the version check). +//! +//! The overlapping-mutations race a counter (rather than a `0`/`1` flag) +//! defeats (also found by external review, 2026-07-29): if two +//! security-sensitive mutations both touch the same key — say, two role +//! changes affecting the same subject's `grants` entry — and a boolean `dirty` +//! flag is unconditionally cleared by whichever mutation's `end` runs first, +//! a reader landing after that first `end` but before the *second* mutation's +//! commit sees a clean, non-dirty entry and can `try_populate` a payload +//! loaded from the pre-second-mutation database state. That entry is wrong +//! the instant the second mutation commits, and if the second mutation's own +//! `end` is ever delayed or lost, nothing else corrects it before the TTL. +//! Making `dirty` a nesting counter — incremented by every `begin`, +//! decremented by every `end` — means the barrier only reads as clean once +//! *every* overlapping mutation on the key has called `end`, so a reader can +//! never land in the gap between one mutation's `end` and another's commit. //! //! Reads never depend on Redis being reachable: any error (timeout, //! connection failure, corrupt payload) is treated as a miss and falls @@ -78,7 +95,7 @@ const BEGIN_SCRIPT_SRC: &str = r#" local ttl_ms = ARGV[1] for i, key in ipairs(KEYS) do redis.call('HINCRBY', key, 'v', 1) - redis.call('HSET', key, 'dirty', '1') + redis.call('HINCRBY', key, 'dirty', 1) redis.call('HDEL', key, 'p') redis.call('PEXPIRE', key, ttl_ms) end @@ -91,6 +108,14 @@ return 1 // barrier expired (a long mutation, or a bulk invalidation chunking through // many keys) would otherwise leak an immortal hash per key. // +// `dirty` is a nesting counter, not a flag: `begin` increments it, `end` +// decrements it, and only 0 means "no mutation is still in flight" — see the +// module docs for why two overlapping mutations on the same key need this. +// The clamp back to 0 below guards against `dirty` drifting negative if `end` +// is ever called without a matching `begin` (should not happen, but a +// negative counter would otherwise require one *extra* `begin` to dig back +// out of before the barrier could ever be seen as dirty again). +// // `HDEL p` is defensive rather than load-bearing: `begin` already cleared the // payload, and `try_populate` refuses to write while dirty. It costs one call // and guarantees that whatever happened to the key in between — including a @@ -101,7 +126,10 @@ const END_SCRIPT_SRC: &str = r#" local ttl_ms = ARGV[1] for i, key in ipairs(KEYS) do redis.call('HINCRBY', key, 'v', 1) - redis.call('HSET', key, 'dirty', '0') + local remaining = redis.call('HINCRBY', key, 'dirty', -1) + if remaining < 0 then + redis.call('HSET', key, 'dirty', 0) + end redis.call('HDEL', key, 'p') redis.call('PEXPIRE', key, ttl_ms) end @@ -111,9 +139,8 @@ return 1 const TRY_POPULATE_SCRIPT_SRC: &str = r#" local v = redis.call('HGET', KEYS[1], 'v') if v == false then v = '0' end -local dirty = redis.call('HGET', KEYS[1], 'dirty') -if dirty == false then dirty = '0' end -if dirty == '1' or v ~= ARGV[1] then +local dirty = tonumber(redis.call('HGET', KEYS[1], 'dirty')) or 0 +if dirty > 0 or v ~= ARGV[1] then return 'stale' end redis.call('HSET', KEYS[1], 'p', ARGV[2]) @@ -131,9 +158,8 @@ return 'applied' const DISCARD_SCRIPT_SRC: &str = r#" local v = redis.call('HGET', KEYS[1], 'v') if v == false then v = '0' end -local dirty = redis.call('HGET', KEYS[1], 'dirty') -if dirty == false then dirty = '0' end -if dirty == '1' or v ~= ARGV[1] then +local dirty = tonumber(redis.call('HGET', KEYS[1], 'dirty')) or 0 +if dirty > 0 or v ~= ARGV[1] then return 'skipped' end redis.call('HDEL', KEYS[1], 'p') @@ -220,9 +246,12 @@ impl RawLookup { } } - fn from_fields(fields: (Option, Option, Option>)) -> Self { + fn from_fields(fields: (Option, Option, Option>)) -> Self { let (version, dirty, payload) = fields; - let is_dirty = dirty.as_deref() == Some("1"); + // `dirty` is a nesting counter (see `BEGIN_SCRIPT_SRC`/`END_SCRIPT_SRC`): + // any value above zero means at least one overlapping mutation on this + // key is still in flight. + let is_dirty = dirty.unwrap_or(0) > 0; Self { version: Some(version.unwrap_or(0)), payload: payload.filter(|_| !is_dirty), @@ -360,7 +389,7 @@ impl CacheClient { } let result = tokio::time::timeout( self.op_timeout, - pipe.query_async::, Option, Option>)>>(&mut conn), + pipe.query_async::, Option, Option>)>>(&mut conn), ) .await; @@ -521,10 +550,14 @@ impl CacheClient { } } - /// Marks `keys` dirty before a security-sensitive Postgres mutation. - /// **Fails the caller** if the barrier cannot be established (Redis - /// unreachable/timeout) — see module docs and `src/cache/invalidate.rs`. - /// A no-op that always succeeds when `keys` is empty. + /// Increments `keys`' dirty counter before a security-sensitive Postgres + /// mutation — safe to call while another mutation on the same key is + /// already in flight, since the counter (not a flag) is what lets `end` + /// tell "this mutation is done" apart from "every overlapping mutation on + /// this key is done" (see module docs). **Fails the caller** if the + /// barrier cannot be established (Redis unreachable/timeout) — see module + /// docs and `src/cache/invalidate.rs`. A no-op that always succeeds when + /// `keys` is empty. pub async fn begin(&self, category: CacheCategory, keys: &[String]) -> Result<(), AppError> { if keys.is_empty() { return Ok(()); @@ -571,12 +604,14 @@ impl CacheClient { Ok(()) } - /// Bumps the version and clears the dirty marker on `keys` after the - /// mutation (success or failure). Always best-effort — never fails the - /// caller. Left dirty entries self-heal once the barrier TTL set by - /// `begin` expires. The version bump (not just the dirty clear) is what - /// stops a reader whose `lookup` landed during the dirty window from - /// repopulating a stale value afterward — see the module docs. + /// Bumps the version and decrements the dirty counter on `keys` after the + /// mutation (success or failure) — the entry only reads as clean once + /// every overlapping `begin` on the key has been matched by an `end`. + /// Always best-effort — never fails the caller. Left dirty entries + /// self-heal once the barrier TTL set by `begin` expires. The version + /// bump (not just the dirty decrement) is what stops a reader whose + /// `lookup` landed during the dirty window from repopulating a stale + /// value afterward — see the module docs. pub async fn end(&self, category: CacheCategory, keys: &[String]) { if keys.is_empty() { return; @@ -1099,4 +1134,89 @@ mod tests { Lookup::Unavailable => panic!("cache should be reachable in this test"), } } + + /// Regression test for a review finding: the two tests above only cover a + /// single mutation on a key. When *two* security-sensitive mutations + /// overlap on the same key — e.g. two role changes affecting the same + /// subject's `grants` entry — a `dirty` field that is a flag rather than + /// a nesting counter is unconditionally cleared by whichever mutation's + /// `end` runs first, even though the second mutation is still in flight. + /// A reader landing in that gap sees a clean, non-dirty entry and can + /// `try_populate` a payload loaded before the second mutation's own + /// commit — wrong the instant that commit lands, and never corrected if + /// the second mutation's own `end` is ever delayed or lost. + #[tokio::test] + #[ignore] + async fn dirty_barrier_stays_up_until_every_overlapping_mutation_ends() { + let client = test_client().await; + let key = unique_key("overlapping-mutations"); + let keys = vec![key.clone()]; + + // Two mutations both touch this key concurrently — M1 begins first, + // then M2 begins while M1 is still in flight. + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("M1 begin"); + client + .begin(CacheCategory::Grants, &keys) + .await + .expect("M2 begin"); + + // M1 finishes first. With a flag (not a counter), this would clear + // `dirty` outright even though M2 hasn't committed yet. + client.end(CacheCategory::Grants, &keys).await; + + // A reader lands in the gap between M1's `end` and M2's commit. The + // barrier must still read as dirty — M2 is still in flight — so this + // must be a miss, not a hit, and the version it observes here must + // never successfully populate the cache. + let gap_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => panic!( + "expected a miss while M2 is still in flight (M1's end must not have cleared \ + the barrier), got {other:?}" + ), + }; + let stale_value = Payload { + value: "STALE — read while a second overlapping mutation was still in flight".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, gap_version, &stale_value) + .await; + match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Hit(got) => assert_ne!( + got, stale_value, + "a reader landing between M1's `end` and M2's commit was able to poison the \ + cache — this means `dirty` is being treated as a flag instead of a nesting \ + counter, so the first of two overlapping mutations to finish clears the \ + barrier the second one still needs" + ), + Lookup::Miss { .. } => {} + Lookup::Unavailable => panic!("cache should be reachable in this test"), + } + + // M2 finishes. Only now should the barrier be fully clear. + client.end(CacheCategory::Grants, &keys).await; + let clean_version = match client.lookup::(CacheCategory::Grants, &key).await { + Lookup::Miss { version } => version, + other => { + panic!("expected a clean miss once every overlapping mutation ended, got {other:?}") + } + }; + let fresh_value = Payload { + value: "fresh, post-M2 value".into(), + }; + client + .try_populate(CacheCategory::Grants, &key, clean_version, &fresh_value) + .await; + assert!( + matches!( + client.lookup::(CacheCategory::Grants, &key).await, + Lookup::Hit(got) if got == fresh_value + ), + "once every overlapping mutation has called `end`, the barrier must clear and a \ + fresh populate must succeed" + ); + } } diff --git a/src/graphql/groups.rs b/src/graphql/groups.rs index d3a1e8c..c6338dc 100644 --- a/src/graphql/groups.rs +++ b/src/graphql/groups.rs @@ -437,6 +437,21 @@ impl GroupMutation { }, ) .await + .inspect(|group| { + // Only needed on this locked path — the non-status-change + // branch below already gets it from + // `update_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.update", + }, + &details, + ); + }) } else { repo::update_group_with_audit( &state.pool, @@ -527,7 +542,21 @@ impl GroupMutation { }, ) .await?; - repo::get_group(&state.pool, id).await + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `set_group_parent_with_audit` + // itself. + repo::get_group(&state.pool, id).await.inspect(|group| { + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.parent.set", + }, + &serde_json::json!({ "parent_id": parent_id }), + ); + }) } .await; if let Err(ref err) = result { @@ -603,6 +632,19 @@ impl GroupMutation { }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `remove_group_parent_with_audit` + // itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.parent.remove", + }, + &serde_json::json!({}), + ); Ok(tenant_id) } .await; @@ -689,6 +731,18 @@ impl GroupMutation { }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `delete_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.delete", + }, + &details, + ); Ok(tenant_id) } .await; @@ -1030,6 +1084,20 @@ impl GroupMutation { }, ) .await + .inspect(|group| { + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `update_group_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event, + }, + &details, + ); + }) } .await; if let Err(ref err) = result { diff --git a/src/graphql/policies.rs b/src/graphql/policies.rs index b3bc250..092ec63 100644 --- a/src/graphql/policies.rs +++ b/src/graphql/policies.rs @@ -483,6 +483,20 @@ impl PolicyMutation { }, ) .await?; + // `_in_tx` already enqueued the outbox row before returning — this + // is the post-commit stdout observability log. Only needed on + // this locked path: the cache-disabled branch above already gets + // it from `replace_role_permission_block_links_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role", + target_id: Some(role_id), + event: "role.permission_blocks.replace", + }, + &serde_json::json!({ "permission_block_ids": permission_block_ids }), + ); Ok(tenant_id) } .await; @@ -564,6 +578,18 @@ impl PolicyMutation { }, ) .await?; + // Only needed on this locked path — the cache-disabled branch + // above already gets it from `delete_role_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role", + target_id: Some(id), + event: "role.delete", + }, + &details, + ); Ok(tenant_id) } .await; @@ -1196,6 +1222,22 @@ impl PolicyMutation { }, ) .await + .inspect(|assignment| { + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `create_role_assignment_with_audit` + // itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: assignment.tenant_id, + target_kind: "role_assignment", + target_id: Some(assignment.id), + event: "role_assignment.create", + }, + &details, + ); + }) } } } @@ -1291,6 +1333,19 @@ impl PolicyMutation { }, ) .await?; + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `delete_role_assignment_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "role_assignment", + target_id: Some(id), + event: "role_assignment.delete", + }, + &details, + ); } } Ok(tenant_id) @@ -1397,6 +1452,21 @@ impl PolicyMutation { }, ) .await + .inspect(|policy| { + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `create_direct_policy_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: policy.tenant_id, + target_kind: "direct_policy", + target_id: Some(policy.id), + event: "direct_policy.create", + }, + &details, + ); + }) } } } @@ -1492,6 +1562,19 @@ impl PolicyMutation { }, ) .await?; + // Only needed on this locked path — the entity-subject + // branch and the cache-disabled fallback above already + // get it from `delete_direct_policy_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id, + target_kind: "direct_policy", + target_id: Some(id), + event: "direct_policy.delete", + }, + &details, + ); } } Ok(tenant_id) diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 446d974..9a894f4 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -458,6 +458,14 @@ impl TenantMutation { Err(err) => Err(err), }; crate::cache::invalidate::end_all(cache, &groups).await; + // `deactivate_and_finish_tenant_soft_delete_in_tx` already + // enqueued the outbox row before returning — this is the + // post-commit stdout observability log. Only needed on this + // locked path; the cache-disabled branch above already gets it + // from `soft_delete_tenant_with_audit` itself. + if outcome.is_ok() { + audit::log_observe_allow(&meta, &details); + } outcome } .await; diff --git a/src/identity/repo.rs b/src/identity/repo.rs index 0932f7d..8fda210 100644 --- a/src/identity/repo.rs +++ b/src/identity/repo.rs @@ -1444,10 +1444,24 @@ pub async fn update_group_with_audit( id, req, event_name, - audit_details, + audit_details.clone(), ) .await?; tx.commit().await.map_err(db_err)?; + // `update_group_in_tx` already enqueued the outbox row via `observe_in_tx` + // before returning — this is the post-commit stdout observability log + // `commit_with_observation` would otherwise provide; calling that helper + // instead would enqueue the outbox row a second time. + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(id), + event: event_name, + }, + &audit_details, + ); Ok(group) } @@ -1469,7 +1483,18 @@ pub async fn set_group_parent_with_audit( let mut tx = pool.begin().await.map_err(db_err)?; set_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id, parent_id).await?; tx.commit().await.map_err(db_err)?; - get_group(pool, child_id).await + let group = get_group(pool, child_id).await?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: group.tenant_id, + target_kind: "group", + target_id: Some(child_id), + event: "group.parent.set", + }, + &serde_json::json!({ "parent_id": parent_id }), + ); + Ok(group) } /// Body of [`set_group_parent`] (minus its post-commit `get_group` read); @@ -1651,18 +1676,30 @@ pub async fn remove_group_parent_with_audit( child_id: Uuid, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - remove_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id).await?; - tx.commit().await.map_err(db_err) + let tenant_id = remove_group_parent_in_tx(&mut tx, events_enabled, actor_id, child_id).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "group", + target_id: Some(child_id), + event: "group.parent.remove", + }, + &serde_json::json!({}), + ); + Ok(()) } /// Body of [`remove_group_parent`]; caller contract per -/// [`set_group_parent_in_tx`]. +/// [`set_group_parent_in_tx`]. Returns the group's `tenant_id` for the +/// caller's post-commit observability log. pub(crate) async fn remove_group_parent_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, child_id: Uuid, -) -> Result<(), AppError> { +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(child_id) @@ -1709,7 +1746,7 @@ pub(crate) async fn remove_group_parent_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant_id) } pub async fn list_child_groups( @@ -1742,8 +1779,19 @@ pub async fn delete_group_with_audit( deleted_by: Option, ) -> Result<(), AppError> { let mut tx = pool.begin().await.map_err(db_err)?; - delete_group_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; - tx.commit().await.map_err(db_err) + let tenant_id = delete_group_in_tx(&mut tx, events_enabled, actor_id, id, deleted_by).await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id, + target_kind: "group", + target_id: Some(id), + event: "group.delete", + }, + &serde_json::json!({}), + ); + Ok(()) } /// Body of [`delete_group`]; caller contract per @@ -1751,14 +1799,16 @@ pub async fn delete_group_with_audit( /// hold this group's closure lock via /// `authz::repo::lock_group_closures_and_collect_grants_keys` on this `tx` — /// a soft delete only sets `deleted_at`, leaving `group_hierarchy` untouched, -/// so the closure is unaffected by this mutation's own effect. +/// so the closure is unaffected by this mutation's own effect. Returns the +/// group's `tenant_id` for the caller's post-commit observability log — a +/// re-read after this commits would always miss the now-deleted row. pub(crate) async fn delete_group_in_tx( tx: &mut Transaction<'_, Postgres>, events_enabled: bool, actor_id: Option, id: Uuid, deleted_by: Option, -) -> Result<(), AppError> { +) -> Result, AppError> { let tenant_id: Option> = sqlx::query_scalar("SELECT tenant_id FROM groups WHERE id = $1 AND deleted_at IS NULL") .bind(id) @@ -1800,7 +1850,7 @@ pub(crate) async fn delete_group_in_tx( }; let details = serde_json::json!({}); crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; - Ok(()) + Ok(tenant_id) } pub async fn delete_group( diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index f98d291..747f18c 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -710,6 +710,16 @@ pub async fn soft_delete_tenant_with_audit( ) .await?; tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: "tenant", + target_id: Some(id), + event: "tenant.delete", + }, + &serde_json::json!({}), + ); Ok(tenant) } From 7576eb273162c99089bae6f47409ce739bfe5306 Mon Sep 17 00:00:00 2001 From: ianmuchyri Date: Mon, 3 Aug 2026 13:01:09 +0300 Subject: [PATCH 19/19] Invalidate session cache entries when a tenant is disabled or frozen change_tenant_status_with_audit bulk-revokes the tenant's members' sessions when transitioning away from Active, but the resolver only ever barriered tenant_status. A stale session cache entry left in place this way survives disable/freeze undetected (tenant_status alone denies auth while the tenant stays that way) and resurfaces the instant the tenant is re-enabled and tenant_status is a hit again. Split change_tenant_status_with_audit into an _in_tx/_with_audit pair so the resolver can lock the tenant and enumerate affected session ids before establishing the barrier on both categories, mirroring deleteTenant. The Active (re-enable) transition never revokes anything and keeps its original single-key tenant_status barrier. --- src/graphql/tenants.rs | 103 +++++++++++++++++++++++++++----- src/tenants/repo.rs | 43 +++++++++++-- tests/m25_cache_invalidation.rs | 83 +++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 19 deletions(-) diff --git a/src/graphql/tenants.rs b/src/graphql/tenants.rs index 9a894f4..2c95f0e 100644 --- a/src/graphql/tenants.rs +++ b/src/graphql/tenants.rs @@ -904,22 +904,95 @@ async fn change_tenant_status(ctx: &Context<'_>, id: ID, status: TenantStatus) - // load_decision_context`), so a stale tenant-membership-implicit // grant inside a cached `grants` entry is harmless as long as this // key itself invalidates. - crate::cache::invalidate::guarded_mutation( - state.cache.as_deref(), - crate::cache::CacheCategory::TenantStatus, - std::slice::from_ref(&crate::cache::keys::tenant_status(tenant_id)), - || { - tenant_repo::change_tenant_status_with_audit( - &state.pool, - state.config.events.enabled(), - Some(auth.entity_id), - tenant_id, - status, - event, - ) - }, + let Some(cache) = state.cache.as_deref() else { + return tenant_repo::change_tenant_status_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, + ) + .await; + }; + if status == TenantStatus::Active { + // Re-enabling never revokes anything (see + // `change_tenant_status_in_tx`), so `tenant_status` is the only + // key that needs a barrier here. + return crate::cache::invalidate::guarded_mutation( + Some(cache), + crate::cache::CacheCategory::TenantStatus, + std::slice::from_ref(&crate::cache::keys::tenant_status(tenant_id)), + || { + tenant_repo::change_tenant_status_with_audit( + &state.pool, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, + ) + }, + ) + .await; + } + // Disabling/freezing also bulk-revokes the tenant's members' + // sessions (see `change_tenant_status_in_tx`) — lock the tenant and + // enumerate those session ids first, establish the barrier on both + // categories, then mutate. Mirrors `deleteTenant` exactly, and for + // the same reason: a session cache entry this call is about to + // revoke must never be left reachable as a stale hit, or it survives + // (with `revoked_at = None`) until the tenant is re-enabled and its + // own fresh `tenant_status` hit stops masking the stale session. + let mut tx = state.pool.begin().await.map_err(crate::error::db_err)?; + let session_ids = + tenant_repo::lock_tenant_and_collect_session_ids_in_tx(&mut tx, tenant_id).await?; + let session_keys: Vec = session_ids + .iter() + .map(|id| crate::cache::keys::session(*id)) + .collect(); + let tenant_status_keys = [crate::cache::keys::tenant_status(tenant_id)]; + let groups: [(crate::cache::CacheCategory, &[String]); 2] = [ + ( + crate::cache::CacheCategory::TenantStatus, + &tenant_status_keys, + ), + (crate::cache::CacheCategory::Session, &session_keys), + ]; + crate::cache::invalidate::begin_all(cache, &groups).await?; + let outcome = tenant_repo::change_tenant_status_in_tx( + &mut tx, + state.config.events.enabled(), + Some(auth.entity_id), + tenant_id, + status, + event, ) - .await + .await; + let outcome = match outcome { + Ok(tenant) => tx + .commit() + .await + .map_err(crate::error::db_err) + .map(|_| tenant), + Err(err) => Err(err), + }; + crate::cache::invalidate::end_all(cache, &groups).await; + let tenant = outcome?; + // Only needed on this locked path — the `Active` branch above and + // the cache-disabled fallback both already get it from + // `change_tenant_status_with_audit` itself. + audit::log_observe_allow( + &audit::AuditMeta { + actor_entity_id: Some(auth.entity_id), + tenant_id: Some(tenant_id), + target_kind: "tenant", + target_id: Some(tenant_id), + event, + }, + &serde_json::json!({ "status": status_detail.clone() }), + ); + Ok(tenant) } .await; if let Err(ref err) = result { diff --git a/src/tenants/repo.rs b/src/tenants/repo.rs index 747f18c..2975a09 100644 --- a/src/tenants/repo.rs +++ b/src/tenants/repo.rs @@ -965,13 +965,48 @@ pub async fn change_tenant_status_with_audit( id: Uuid, status: TenantStatus, event_name: &str, +) -> Result { + let mut tx = pool.begin().await.map_err(db_err)?; + let tenant = + change_tenant_status_in_tx(&mut tx, events_enabled, actor_id, id, status, event_name) + .await?; + tx.commit().await.map_err(db_err)?; + crate::audit::log_observe_allow( + &crate::audit::AuditMeta { + actor_entity_id: actor_id, + tenant_id: Some(id), + target_kind: "tenant", + target_id: Some(id), + event: event_name, + }, + &serde_json::json!({ "status": tenant.status }), + ); + Ok(tenant) +} + +/// Body of [`change_tenant_status_with_audit`]. When `status != Active`, this +/// also bulk-revokes the tenant's members' sessions — a cache-aware caller +/// must already have enumerated those session ids via +/// [`lock_tenant_and_collect_session_ids_in_tx`] and established the barrier +/// on them (alongside `tenant_status`) before calling this, the same way +/// `deleteTenant` does: a barrier put up only after this commits would leave +/// a window where a concurrent read still takes a full cache hit on a +/// session this call is about to revoke, which then survives (with +/// `revoked_at = None`) until the tenant is re-enabled and its own +/// `tenant_status` cache entry stops masking the stale session. +pub(crate) async fn change_tenant_status_in_tx( + tx: &mut Transaction<'_, Postgres>, + events_enabled: bool, + actor_id: Option, + id: Uuid, + status: TenantStatus, + event_name: &str, ) -> Result { if status == TenantStatus::Deleted { return Err(AppError::bad_request( "use delete tenant to apply the soft-delete lifecycle", )); } - let mut tx = pool.begin().await.map_err(db_err)?; let tenant = sqlx::query_as::<_, Tenant>(&format!( r#"UPDATE tenants SET status = $2, updated_by = $3, updated_at = now() @@ -981,7 +1016,7 @@ pub async fn change_tenant_status_with_audit( .bind(id) .bind(&status) .bind(actor_id) - .fetch_one(&mut *tx) + .fetch_one(&mut **tx) .await .map_err(|e| match e { sqlx::Error::RowNotFound => AppError::not_found(format!("tenant {id} not found")), @@ -995,7 +1030,7 @@ pub async fn change_tenant_status_with_audit( AND entity_id IN (SELECT id FROM entities WHERE tenant_id = $1)", ) .bind(id) - .execute(&mut *tx) + .execute(&mut **tx) .await .map_err(db_err)?; } @@ -1008,7 +1043,7 @@ pub async fn change_tenant_status_with_audit( event: event_name, }; let details = serde_json::json!({ "status": tenant.status }); - crate::audit::commit_with_observation(tx, events_enabled, &meta, &details).await?; + crate::audit::observe_in_tx(tx, events_enabled, &meta, &details).await?; Ok(tenant) } diff --git a/tests/m25_cache_invalidation.rs b/tests/m25_cache_invalidation.rs index 2b3d740..6461fc9 100644 --- a/tests/m25_cache_invalidation.rs +++ b/tests/m25_cache_invalidation.rs @@ -822,6 +822,89 @@ async fn tenant_delete_immediately_rejects_an_existing_session_of_a_member() { ); } +/// Regression test for a review finding: `change_tenant_status`'s +/// `disableTenant`/`freezeTenant` mutations bulk-revoke the tenant's +/// members' sessions in the same transaction as the status flip, but the +/// resolver only ever established a cache barrier on `tenant_status` — never +/// on the sessions it was about to revoke. +/// +/// As with `tenant_restore_clears_reactivated_credential_cache_entries` +/// below, an end-to-end auth-behavior test can't actually isolate this: the +/// `tenant_status` barrier's own `end` always clears that entry's payload +/// too, so the very next authentication after `disableTenant` (or after a +/// subsequent `enableTenant`) is forced through a fresh-Postgres-reload path +/// regardless of whether the session was ever separately invalidated — that +/// fresh reload sees the correctly-revoked row and denies either way, and +/// self-heals the session cache entry in the process, masking the bug on +/// every subsequent check too. Confirmed by trying exactly that shape of +/// test and finding it passed even with the session invalidation reverted. +/// So this checks the one thing that actually isolates it: that +/// `disableTenant` clears the session's own cache entry directly, verified +/// against Redis, independent of what any later auth attempt would do. +#[tokio::test] +#[ignore] +async fn disabling_a_tenant_invalidates_its_members_session_cache_entries() { + let p = pool().await; + let (state, cache) = state_with_cache(p.clone()).await; + let tenant_id = tenant(&p).await; + let entity_id = active_entity_in_tenant(&p, tenant_id, "service").await; + let session_id = Uuid::new_v4(); + let expires_at = chrono::Utc::now() + chrono::Duration::hours(1); + sqlx::query("INSERT INTO sessions (id, entity_id, expires_at) VALUES ($1, $2, $3)") + .bind(session_id) + .bind(entity_id) + .bind(expires_at) + .execute(&p) + .await + .expect("insert session"); + + let primary = state.keys.read().await.primary.clone(); + let token = auth::encode_jwt( + entity_id, + session_id, + Some(tenant_id), + &primary, + state.config.jwt_expiry_secs, + &state.config.jwt_issuer, + &state.config.jwt_audience, + ) + .expect("encode jwt"); + + // Warm the session cache as a valid, stale-able hit. + auth::authenticate_token(&state, &token) + .await + .expect("initial authentication should succeed"); + let session_key = atom::cache::keys::session(session_id); + let (_, _, payload_before) = hmget_raw(&session_key).await; + assert!( + payload_before.is_some(), + "session cache should be warm before disabling the tenant" + ); + + let schema = build_schema(state.clone()); + let resp = schema + .execute(authed( + common::admin_id(), + cache.clone(), + format!(r#"mutation {{ disableTenant(id: "{tenant_id}") {{ id }} }}"#), + )) + .await; + assert!(resp.errors.is_empty(), "disable failed: {:?}", resp.errors); + + // `end`'s barrier always clears the payload of whatever it covers (see + // `src/cache/mod.rs`), so a lingering payload here means this session's + // key was never part of the barrier at all -- the exact gap the review + // found. + let (_, _, payload_after) = hmget_raw(&session_key).await; + assert!( + payload_after.is_none(), + "disabling a tenant must invalidate its members' session cache entries, not just \ + tenant_status -- a lingering \"still valid\" session cache entry here would keep \ + serving hits (masked only by tenant_status denying auth while the tenant stays \ + disabled) and would resurface the instant the tenant is re-enabled" + ); +} + /// Regression test for a review finding: `restore_tenant` reactivates /// credentials but the resolver originally only invalidated `tenant_status`, /// not the credential entries it reactivates.