diff --git a/.github/workflows/deploy-native.yml b/.github/workflows/deploy-native.yml index 752e546..2410fd4 100644 --- a/.github/workflows/deploy-native.yml +++ b/.github/workflows/deploy-native.yml @@ -245,6 +245,11 @@ jobs: # Unset -> endpoint 404s. An optional secret that neither caller # passes arrives here as the empty string. OPENAI_APPS_CHALLENGE_TOKEN: ${{ secrets.OPENAI_APPS_CHALLENGE_TOKEN }} + # Client ID Metadata Documents: advertised only where this GitHub + # Environment defines the OAUTH_CIMD_ENABLED variable as `1` (this job + # runs in that environment, so its variables resolve here). Unset + # arrives as the empty string, which the server reads as off. + OAUTH_CIMD_ENABLED: ${{ vars.OAUTH_CIMD_ENABLED }} run: deploy/native/deploy.sh # Confirm the host is actually running the commit we just shipped, rather diff --git a/Cargo.lock b/Cargo.lock index a0d6164..abc6e19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,6 +1830,8 @@ dependencies = [ "crc32fast", "getrandom 0.3.4", "hex", + "http", + "httpdate", "ic-agent", "pocket-ic", "regex", diff --git a/Cargo.toml b/Cargo.toml index 9e1b282..a834dc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ dist = false # codec, and candid lines can never drift between imcp2-core, imcp2, and # imcp2-local. [workspace.dependencies] +httpdate = "1.0.3" imcp2-core = { path = "crates/imcp2-core", version = "0.4.0" } rmcp = "1.7" axum = "0.8" diff --git a/README.md b/README.md index 7eedca1..c350143 100644 --- a/README.md +++ b/README.md @@ -692,6 +692,43 @@ its AS issuer is `/mcp` and everything OAuth lives under it: honoured (intersected with `authorization_code`). A **hosted** `redirect_uri` is rejected unless its host is on the allow-list (see the Companion-control note below); loopback redirects are always accepted. +- **Client ID Metadata Documents** — the MCP authorization spec's preferred + registration, advertised as `client_id_metadata_document_supported: true`. A + client may skip `/register` and use the https URL of its metadata document as + its `client_id`; `/mcp/oauth/authorize` fetches that document under the same + SSRF guard as app discovery (https only, public addresses only, pinned, no + redirects, 8 KiB cap, 5 s including DNS), requires its `client_id` to equal the URL, and checks the requested + redirect against the document's `redirect_uris` exactly as it would a DCR + registration's — hosted-redirect allow-list included, and checked before any + fetch, so a document can neither admit a redirect a DCR client couldn't + register nor make the server fetch a URL for a redirect it would refuse. A + hosted redirect must also be same-origin with the document URL (loopback + excepted), so a self-asserted document cannot point the code at another + party. Only clients that can authenticate as PUBLIC clients are accepted: a + `token_endpoint_auth_method` of `none` (or none given), or `none` among the + document's `token_endpoint_auth_methods_supported` — ChatGPT's case, since it + prefers `private_key_jwt` but lists `none`, which is what it uses here. + Documents must be served as `application/json`, and are cached (bounded; the + origin's remaining freshness — `max-age` less `Age` — honoured up to 24 h, + 10 min when it sends none, not at all on `no-store`), so a directory client + connecting thousands of times mints no registrations. Concurrent requests for + one document share a single fetch, and at most eight fetches are in flight at + once per process (however many instances the binary mounts), two per host, so + one slow host cannot hold up the rest — and no more than sixty a minute per + process, thirty per vendor domain, so an origin that answers at once cannot be + made to answer without end. The fetch connects directly, never + through a proxy from the environment, so the address pin always binds. Claude and ChatGPT both select CIMD over DCR when it is + advertised — which it is only where `OAUTH_CIMD_ENABLED=1` is set (the deploy + template takes it from the GitHub Environment's variable of that name, so a + deploy never enables it by itself; to roll back, unset it and redeploy — the + value is read once at start-up, so the variable alone changes nothing — and + clients re-read the metadata within minutes and fall back to DCR). Only a + document on a vetted vendor origin is fetched at all — a host on or under an + allow-listed domain, default port (the trust policy of PR #143); any other URL + `client_id` is refused before any request and pointed at the allow-listing + contact. A document-intrinsic failure (no document there, not JSON, about + another URL) is remembered for a minute so a repeat is cheap; a transient one + is not. - `GET /mcp/oauth/authorize` — validates the client + redirect, requires PKCE, sets the binding cookie, then redirects to II's handshake (with `registration_key`) diff --git a/crates/imcp2-core/Cargo.toml b/crates/imcp2-core/Cargo.toml index 79b86e5..b0c80f3 100644 --- a/crates/imcp2-core/Cargo.toml +++ b/crates/imcp2-core/Cargo.toml @@ -18,7 +18,7 @@ rmcp = { workspace = true, features = ["server", "macros"] } ic-agent = { workspace = true } candid = { workspace = true } candid_parser = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "net"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "net", "time"] } serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } @@ -31,6 +31,7 @@ reqwest = { workspace = true } getrandom = { workspace = true } urlencoding = { workspace = true } url = { workspace = true } +httpdate = { workspace = true } # Only for the end-to-end canister-tool tests (src/e2e_canister_tools.rs), # behind the `e2e` feature so the default build compiles neither. `pocket-ic` # runs a real replica locally (needs the POCKET_IC_BIN server binary at @@ -48,3 +49,8 @@ wat = { version = "1", optional = true } # `Identities::seed_app_identity` — are gated on `cfg(test)` as well, so they # exist only in this crate's own test binary, never in a library build. e2e = ["dep:pocket-ic", "dep:wat", "rmcp/client"] + +[dev-dependencies] +# Synthetic responses for pinning `public_fetch`'s acceptance rules without a +# network (`reqwest::Response: From>`). +http = { workspace = true } diff --git a/crates/imcp2-core/src/discover.rs b/crates/imcp2-core/src/discover.rs index 0aee7eb..17c6350 100644 --- a/crates/imcp2-core/src/discover.rs +++ b/crates/imcp2-core/src/discover.rs @@ -1350,6 +1350,9 @@ fn ipv4_is_global(ip: &Ipv4Addr) -> bool { || (o[0] == 100 && (o[1] & 0xc0) == 64) // 100.64.0.0/10 CGNAT (shared) || (o[0] == 192 && o[1] == 0 && o[2] == 0) // 192.0.0.0/24 IETF protocol || (o[0] == 198 && (o[1] & 0xfe) == 18) // 198.18.0.0/15 benchmarking + // 192.88.99.0/24, the deprecated 6to4 relay anycast block (RFC 7526), is not + // globally reachable — bar 192.88.99.2, the 6a44 relay anycast (RFC 6751). + || (o[0] == 192 && o[1] == 88 && o[2] == 99 && o[3] != 2) || o[0] >= 240) // 240.0.0.0/4 reserved } @@ -1362,30 +1365,88 @@ fn ipv6_is_global(ip: &Ipv6Addr) -> bool { return ipv4_is_global(&v4); } let seg = ip.segments(); - !(ip.is_unspecified() // :: - || ip.is_loopback() // ::1 - || ip.is_multicast() // ff00::/8 - || (seg[0] & 0xfe00) == 0xfc00 // fc00::/7 unique-local - || (seg[0] & 0xffc0) == 0xfe80 // fe80::/10 link-local unicast - || (seg[0] == 0x2001 && seg[1] == 0x0db8) // 2001:db8::/32 documentation - // Transition mechanisms embed an IPv4 address deeper in the v6 space than - // `to_ipv4` decodes, so a NAT64/6to4/Teredo host would otherwise translate - // one of these to loopback/link-local/RFC1918/metadata (ICPBB-377). imcp2 - // never needs to reach them, so refuse the prefixes outright. - || (seg[0] == 0x0064 && seg[1] == 0xff9b) // 64:ff9b::/32 NAT64 (RFC 6052 WKP + RFC 8215 local-use) - || seg[0] == 0x2002 // 2002::/16 6to4 - || (seg[0] == 0x2001 && seg[1] == 0x0000)) // 2001::/32 Teredo + // DEFAULT-DENY: IANA allocates global unicast only from 2000::/3 (RFC 4291 + // §2.5.4; RFC 3513 §2.5.6), so a native address outside it — `::`, `::1`, + // the discard-only 100::/64, the NAT64 well-known 64:ff9b::/32, the SRv6 + // 5f00::/16, unique-local fc00::/7, link-local fe80::/10, the deprecated + // site-local fec0::/10, multicast ff00::/8, and everything unallocated in + // between (4000::1 is nobody's) — is refused without being named, and so is + // whatever IANA allocates next, until it is audited here. + if (seg[0] & 0xe000) != 0x2000 { + return false; + } + // Within 2000::/3, 2001::/23 is IETF protocol assignments (RFC 2928): NOT + // globally reachable by default — Teredo (2001::/32), benchmarking + // (2001:2::/48), ORCHID and ORCHIDv2 (2001:10::/28, 2001:20::/28), and + // everything unassigned — with the IANA registry's globally reachable + // exceptions admitted by name, so a new assignment is refused until audited + // rather than accepted until noticed. + if seg[0] == 0x2001 && (seg[1] & 0xfe00) == 0 { + return ietf_protocol_assignment_is_global(&seg); + } + // The rest of 2000::/3 is global unicast, less its special-purpose carve-outs. + !((seg[0] == 0x2001 && seg[1] == 0x0db8) // 2001:db8::/32 documentation + || (seg[0] == 0x3fff && (seg[1] & 0xf000) == 0) // 3fff::/20 documentation (RFC 9637) + // 6to4 embeds an IPv4 address deeper in the v6 space than `to_ipv4` + // decodes, so a 6to4 host would otherwise translate one of these to + // loopback/link-local/RFC1918/metadata (ICPBB-377); NAT64 and Teredo are + // refused above. imcp2 never needs to reach them, so refuse the prefix. + || seg[0] == 0x2002) // 2002::/16 6to4 +} + +/// The globally reachable exceptions inside `2001::/23` (IETF protocol +/// assignments), per the IANA IPv6 Special-Purpose Address Registry. Everything +/// else in the block — assigned to a non-routable use or not assigned at all — +/// is refused. +fn ietf_protocol_assignment_is_global(seg: &[u16; 8]) -> bool { + let anycast = seg[1] == 0x0001 && seg[2..7] == [0; 5] && (1..=3).contains(&seg[7]); + anycast // 2001:1::1 PCP (RFC 7723), 2001:1::2 TURN (RFC 8155), 2001:1::3 DNS-SD SRP (RFC 9665) + || seg[1] == 0x0003 // 2001:3::/32 AMT (RFC 7450) + || (seg[1] == 0x0004 && seg[2] == 0x0112) // 2001:4:112::/48 AS112-v6 (RFC 7535) + || (seg[1] & 0xfff0) == 0x0030 // 2001:30::/28 Drone Remote ID (RFC 9374) +} + +/// Why [`resolve_public_url`] returned no addresses: the URL itself is refused — +/// it does not parse, is not https, names no host, or resolves to a non-public +/// address (the SSRF guard) — or its host could not be resolved RIGHT NOW, which +/// says nothing about the URL. A caller that remembers refusals (the OAuth +/// server's client-metadata cache) must not remember a resolver outage as one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ResolveError { + Refused(String), + Unresolved(String), +} + +impl std::fmt::Display for ResolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused(why) | Self::Unresolved(why) => f.write_str(why), + } + } +} + +/// The discovery crawl reports every failure as a message, and tells these apart +/// no further; `?` keeps working there. +impl From for String { + fn from(err: ResolveError) -> Self { + match err { + ResolveError::Refused(why) | ResolveError::Unresolved(why) => why, + } + } } /// Validate a user-supplied discovery URL against SSRF and return the parsed URL /// plus the socket addresses to PIN the client to. https only; every resolved /// address must be global. Async DNS (no blocking of the executor). -async fn resolve_public_url(raw: &str) -> Result<(url::Url, Vec), String> { - let url = url::Url::parse(raw).map_err(|e| format!("invalid discovery URL {raw}: {e}"))?; +pub(crate) async fn resolve_public_url( + raw: &str, +) -> Result<(url::Url, Vec), ResolveError> { + let url = url::Url::parse(raw) + .map_err(|e| ResolveError::Refused(format!("invalid discovery URL {raw}: {e}")))?; if url.scheme() != "https" { - return Err(format!( + return Err(ResolveError::Refused(format!( "refusing to fetch {raw}: only https:// discovery targets are allowed (SSRF guard)" - )); + ))); } let port = url.port_or_known_default().unwrap_or(443); let addrs: Vec = match url.host() { @@ -1393,19 +1454,19 @@ async fn resolve_public_url(raw: &str) -> Result<(url::Url, Vec), St Some(url::Host::Ipv6(v6)) => vec![SocketAddr::new(IpAddr::V6(v6), port)], Some(url::Host::Domain(host)) => tokio::net::lookup_host((host, port)) .await - .map_err(|e| format!("could not resolve {host}: {e}"))? + .map_err(|e| ResolveError::Unresolved(format!("could not resolve {host}: {e}")))? .collect(), - None => return Err(format!("refusing to fetch {raw}: no host")), + None => return Err(ResolveError::Refused(format!("refusing to fetch {raw}: no host"))), }; if addrs.is_empty() { - return Err(format!("refusing to fetch {raw}: host did not resolve")); + return Err(ResolveError::Unresolved(format!("could not resolve {raw}: no addresses"))); } if let Some(bad) = addrs.iter().find(|a| !ip_is_global(&a.ip())) { - return Err(format!( + return Err(ResolveError::Refused(format!( "refusing to fetch {raw}: it resolves to a non-public address ({}) — discovery is \ restricted to public hosts (SSRF guard)", bad.ip() - )); + ))); } Ok((url, addrs)) } @@ -1536,10 +1597,26 @@ enum Overflow { /// The shared read. `Err((partial, error))` carries what had arrived before the /// transfer failed, so the fail-soft caller can keep it and the strict one can /// report the failure. -async fn read_capped_inner( - mut resp: reqwest::Response, +pub(crate) async fn read_capped_inner( + resp: reqwest::Response, max: usize, ) -> Result { + // Lossy by design for the crawl: a stray byte must not cost a whole bundle. + // A caller that treats the body as a statement (the CIMD fetch) reads the + // bytes and decodes strictly instead. + match read_capped_bytes(resp, max).await { + Ok(buf) => Ok(String::from_utf8_lossy(&buf).into_owned()), + Err((buf, e)) => Err((String::from_utf8_lossy(&buf).into_owned(), e)), + } +} + +/// The shared read in bytes: up to `max` of the body, stopping — and so dropping +/// the response and its connection — at the cap. `Err((partial, error))` carries +/// what had arrived before the transfer failed. +pub(crate) async fn read_capped_bytes( + mut resp: reqwest::Response, + max: usize, +) -> Result, (Vec, String)> { let mut buf: Vec = Vec::new(); loop { if buf.len() >= max { @@ -1554,10 +1631,10 @@ async fn read_capped_inner( } } Ok(None) => break, - Err(e) => return Err((String::from_utf8_lossy(&buf).into_owned(), e.to_string())), + Err(e) => return Err((buf, e.to_string())), } } - Ok(String::from_utf8_lossy(&buf).into_owned()) + Ok(buf) } /// GET `url` and return up to `max` bytes of its body, distinguishing "this app @@ -2794,6 +2871,7 @@ mod tests { // Publicly-routable addresses. assert!(g("8.8.8.8")); assert!(g("1.1.1.1")); + assert!(g("192.88.99.2"), "the 6a44 relay anycast is the reachable exception in its /24"); assert!(g("2606:4700:4700::1111")); // Loopback / private / link-local / CGNAT / reserved / doc / bench, plus // IPv4 embedded in IPv6 as MAPPED (::ffff:…) and COMPATIBLE (::…) forms. @@ -2808,12 +2886,26 @@ mod tests { "255.255.255.255", "192.0.2.1", "198.18.0.1", + "192.88.99.1", // 6to4 relay anycast, deprecated "240.0.0.1", "::1", "::", "fc00::1", "fd12::1", "fe80::1", + "fec0::1", // site-local: deprecated, still routable on legacy networks + "100::1", // discard-only + "2001:2::1", // benchmarking + "2001:10::1", // ORCHID (deprecated) + "2001:20::1", // ORCHIDv2 (not routable) + "2001:1::4", // unassigned inside 2001::/23 (IETF protocol assignments) + "2001:5::1", // likewise + "2001:1ff::1", // the block's last /32, likewise + "3fff::1", // documentation (RFC 9637) + "5f00::1", // SRv6 SIDs (RFC 9602) + "4000::1", // outside 2000::/3: not allocated for global unicast + "8000::1", // likewise + "e000::1", // likewise "2001:db8::1", "::ffff:127.0.0.1", "::ffff:10.0.0.1", // IPv4-mapped private/loopback @@ -2833,6 +2925,13 @@ mod tests { } // A real public v6 that merely starts with 0x2001 (not db8/Teredo) stays global. assert!(g("2001:4860:4860::8888")); + assert!(g("2001:200::1"), "just past 2001::/23"); + // The globally reachable exceptions inside 2001::/23 stay global. + for good in + ["2001:1::1", "2001:1::2", "2001:1::3", "2001:3::1", "2001:4:112::1", "2001:30::1"] + { + assert!(g(good), "{good} is globally reachable per the IANA registry"); + } } // The exact vectors from the finding, plus https-to-internal, are refused diff --git a/crates/imcp2-core/src/lib.rs b/crates/imcp2-core/src/lib.rs index 782d9a0..e7e26d6 100644 --- a/crates/imcp2-core/src/lib.rs +++ b/crates/imcp2-core/src/lib.rs @@ -32,6 +32,7 @@ pub mod identities; pub mod iiconnect; +pub mod public_fetch; pub mod skills; pub mod tools; diff --git a/crates/imcp2-core/src/public_fetch.rs b/crates/imcp2-core/src/public_fetch.rs new file mode 100644 index 0000000..e34227e --- /dev/null +++ b/crates/imcp2-core/src/public_fetch.rs @@ -0,0 +1,674 @@ +//! One SSRF-guarded GET of a small public document, for callers outside the +//! discovery crawl that must fetch a URL a stranger handed them. Today that is +//! the hosted OAuth authorization server, fetching a client's *Client ID Metadata +//! Document* — the MCP authorization spec's preferred registration, where the +//! `client_id` an unauthenticated `/oauth/authorize` request carries IS an https +//! URL and the JSON at that URL is the client's registration. +//! +//! The guard is the discovery module's (CWE-918): https only; the host resolved +//! up front and refused if ANY address is loopback / private / link-local / +//! CGNAT / otherwise reserved; the validated addresses pinned into the client so +//! a re-resolution cannot rebind the connection (DNS rebinding); and the body +//! read under a hard byte cap (CWE-770). The connection is DIRECT: no proxy is +//! taken from the environment, since a proxy would resolve the host itself and +//! the pin would bind nothing. On top of that, this fetch is STRICT where the +//! crawl is opportunistic — the document is the URL's own statement about +//! itself, so: +//! +//! * redirects are not followed at all: a 3xx is not `200 OK` (nor is a 206 +//! fragment or any other 2xx), so no +//! other URL's bytes — on another host, another port, or another path of the +//! same origin — can ever stand in for the document at this one; +//! * a body over the cap, one whose transfer failed part-way, or one that is +//! not valid UTF-8 is an error, never a shorter or a normalised document; +//! * the caller's timeout is ONE deadline over the whole operation, DNS +//! resolution included, so a slow resolver cannot hold the caller past it — +//! and it is the only deadline, so however far the fetch got when it ran out +//! of time, the caller sees the same "did not complete" error. +//! +//! Failures are typed ([`FetchError`]) so a caller can tell what is about the URL +//! (the guard refuses it; the origin answers 404 or a redirect; the body is too +//! large or not UTF-8) from what is about the moment (a resolver that did not +//! answer, a deadline, a connection that failed, a 5xx) — the first kind may be +//! remembered, the second may not. +//! +//! The origin's caching instruction is reported as the REMAINING freshness +//! lifetime, per HTTP: every `Cache-Control` field line is read (a `no-store` on +//! a second line counts), and the response's current age — the larger of its +//! `Age` and the time since its `Date` — is subtracted from `max-age`; where +//! `Cache-Control` grants no freshness, `Expires` relative to `Date` decides. + +use std::{ + fmt, + time::{Duration, SystemTime}, +}; + +use reqwest::header::{HeaderMap, AGE, CACHE_CONTROL, CONTENT_TYPE, DATE, EXPIRES, VARY}; + +use crate::discover::{read_capped_bytes, resolve_public_url, ResolveError}; + +/// A small public document fetched under the SSRF guard. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicDocument { + /// The complete body (it fit under the caller's cap), valid UTF-8. + pub body: String, + /// The `Content-Type` the origin sent, if any. + pub content_type: Option, + /// How much longer a SHARED cache may reuse this, per HTTP caching (RFC + /// 9111), as [`freshness`] computes it: the lifetime — `s-maxage`, else + /// `max-age`, from every `Cache-Control` line combined, else `Expires` less + /// `Date` — minus the response's current age (the larger of `Age` and the + /// time since `Date`). `Some(0)` when the origin forbids reuse (`no-store`, + /// `no-cache`, `private`, `Vary: *`), gives an invalid or already-spent + /// lifetime, or the freshness has run out; `None` when it sent no freshness + /// information at all. A hint for the caller's own cache, for the caller to + /// bound — never binding. + pub cache_max_age: Option, + /// How old the response already was on receipt, per HTTP (RFC 9111 §4.2.3): + /// the larger of its `Age` (every line, the greatest; an unparseable one the + /// greatest of all) and the time since its `Date`. Already subtracted from + /// `cache_max_age`; for a caller applying a lifetime of ITS OWN where the + /// origin sent none, the amount to subtract from that too, so an answer some + /// cache held for a day is not given a fresh default. + pub current_age: Duration, +} + +/// Why a document was not returned, split by what the failure is ABOUT. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FetchError { + /// The URL itself: it does not parse, is not https, names no host, or names + /// one with a non-public address. No request was made. + Refused(String), + /// The moment: the host could not be resolved, the deadline passed, or the + /// request could not be sent or its body not read. The same URL may work + /// next time. + Unreachable(String), + /// The origin answered, but not `200 OK` — a redirect (never followed) or any + /// other 2xx (a 206 fragment, a 203 transformed by a proxy, a 204) included. + /// `status` lets the caller tell a 404 (no document there) from a 503. + Answered { status: u16, detail: String }, + /// The body is larger than the caller's cap. + TooLarge(String), + /// The body is not valid UTF-8. + NotUtf8(String), +} + +impl fmt::Display for FetchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Refused(s) | Self::Unreachable(s) | Self::TooLarge(s) | Self::NotUtf8(s) => { + f.write_str(s) + } + Self::Answered { detail, .. } => f.write_str(detail), + } + } +} + +impl std::error::Error for FetchError {} + +/// GET `url` and return its body, or why not ([`FetchError`]): the URL is refused +/// by the SSRF guard; resolving, connecting, answering and delivering the body +/// did not all complete within `timeout`; the answer was anything but `200 OK` +/// (a redirect or a 206 fragment included); the body is larger than `max_bytes`, +/// was cut off, or is not UTF-8. +pub async fn fetch_public_document( + url: &str, + max_bytes: usize, + timeout: Duration, +) -> Result { + // One deadline over everything, resolution included: `resolve_public_url` + // does the DNS lookup, and a resolver that never answers must not hold the + // caller (and whatever it is holding, such as an in-flight permit) forever. + // Deliberately the ONLY deadline — the client below sets none of its own — + // so the error is the same wherever the time ran out, and dropping the + // future on expiry is what aborts the connection. + tokio::time::timeout(timeout, fetch(url, max_bytes)).await.map_err(|_| { + FetchError::Unreachable(format!("fetching {url} did not complete within {timeout:?}")) + })? +} + +async fn fetch(url: &str, max_bytes: usize) -> Result { + let (parsed, pinned) = resolve_public_url(url).await.map_err(|e| match e { + ResolveError::Refused(why) => FetchError::Refused(why), + ResolveError::Unresolved(why) => FetchError::Unreachable(why), + })?; + let host = parsed.host_str().unwrap_or_default().to_ascii_lowercase(); + let client = reqwest::Client::builder() + .user_agent(concat!("imcp2-core/", env!("CARGO_PKG_VERSION"))) + // Never follow a redirect: the document is this URL's statement about + // itself, and a 3xx is that URL declining to make it. Refusing here (rather + // than following under the crawl's redirect guard and comparing origins + // afterwards) also closes the same-origin case, where a redirect to another + // path would have put a different document behind this URL. + .redirect(reqwest::redirect::Policy::none()) + // Direct, whatever the environment says: the pin below binds only a + // connection this client opens itself, and a proxy (reqwest takes one from + // `HTTPS_PROXY` by default) would resolve the host on its own — past the + // guard. A deployment that must egress through a proxy has that proxy do + // the guard's job, as a deliberate choice, not one this fetch takes silently. + .no_proxy() + .resolve_to_addrs(&host, &pinned) + .build() + .map_err(|e| FetchError::Unreachable(format!("http client: {e}")))?; + let resp = client + .get(parsed.as_str()) + .header(reqwest::header::ACCEPT, "application/json") + .send() + .await + .map_err(|e| FetchError::Unreachable(format!("could not fetch {url}: {e}")))?; + accept(url, resp, max_bytes).await +} + +/// Turn the origin's answer into a [`PublicDocument`], or refuse it: anything +/// but `200 OK` (a redirect is named as such, since it is not followed), a body over +/// `max_bytes` or cut off mid-transfer, or one that is not valid UTF-8. Kept apart +/// from the sending so the acceptance rules are pinned by tests on synthetic +/// responses, with no network. +async fn accept( + url: &str, + resp: reqwest::Response, + max_bytes: usize, +) -> Result { + // Exactly 200: the document is the URL's complete statement about itself, + // and only `200 OK` says the body is that. A `206 Partial Content` is a + // fragment (one that may well parse as JSON), a `203` has been through a + // transforming proxy, a `204` has no body — none of them is the document. + let status = resp.status(); + if status != reqwest::StatusCode::OK { + let redirect = + if status.is_redirection() { ", a redirect, which is not followed" } else { "" }; + return Err(FetchError::Answered { + status: status.as_u16(), + detail: format!("{url} answered {status}{redirect}"), + }); + } + let content_type = + resp.headers().get(CONTENT_TYPE).and_then(|v| v.to_str().ok()).map(str::to_owned); + let now = SystemTime::now(); + let cache_max_age = freshness(resp.headers(), now); + let current_age = current_age(resp.headers(), now); + // Read ONE byte past the cap so overflow is detectable: a truncated body is + // not a shorter document. Saturating, so a caller passing `usize::MAX` (no + // cap) reads everything rather than wrapping to a zero-byte read. + let bytes = match read_capped_bytes(resp, max_bytes.saturating_add(1)).await { + Ok(bytes) if bytes.len() > max_bytes => { + return Err(FetchError::TooLarge(format!( + "{url} is larger than the {max_bytes}-byte cap" + ))) + } + Ok(bytes) => bytes, + Err((_, e)) => { + return Err(FetchError::Unreachable(format!("reading {url} failed part-way: {e}"))) + } + }; + // Strict, not lossy: a byte that is not UTF-8 is refused rather than replaced, + // so the document parsed is exactly the one served. + let body = String::from_utf8(bytes) + .map_err(|e| FetchError::NotUtf8(format!("{url} is not valid UTF-8: {e}")))?; + Ok(PublicDocument { body, content_type, cache_max_age, current_age }) +} + +/// The remaining freshness lifetime the response's headers grant, per HTTP +/// caching (RFC 9111 §4.2). The lifetime is `max-age` (or `s-maxage`) from the +/// COMBINED `Cache-Control` fields (a header may be sent as several lines, and a +/// `no-store` on any of them wins); where those grant no freshness, it is +/// `Expires` less `Date` (less `now`, when there is no `Date`), an `Expires` +/// that is invalid — `0` is the classic — or already past meaning stale (§5.3). +/// From it the response's CURRENT AGE is subtracted — the larger of its `Age` +/// and its apparent age, `now` less its `Date`, so an answer some cache held for +/// an hour without saying so in `Age` is not given a new lifetime here. A `Date` +/// in the future (clock skew) is an apparent age of zero. A response that varies +/// on everything (`Vary: *`) has no freshness at all for a shared cache, whatever +/// else it says (§4.1: it can never match a later request). `None` when neither +/// a freshness directive nor `Expires` was sent. +fn freshness(headers: &HeaderMap, now: SystemTime) -> Option { + // A `Vary` or `Cache-Control` line this cannot decode (a quoted argument may + // carry obs-text) is read as forbidding reuse — it might have said so — rather + // than skipped, or a `foo="…", max-age=0` in obs-text would be dropped and + // the decodable `max-age=86400` beside it honoured. + let decode_all = |name| -> Option> { + headers.get_all(name).iter().map(|v| v.to_str().ok()).collect() + }; + let Some(vary) = decode_all(VARY) else { + return Some(Duration::ZERO); + }; + if vary.iter().flat_map(|v| v.split(',')).any(|field| field.trim() == "*") { + return Some(Duration::ZERO); + } + let header = |name| headers.get(name).and_then(|v| v.to_str().ok()).map(str::trim); + let date = header(DATE).and_then(|v| httpdate::parse_http_date(v).ok()); + let Some(cache_control) = decode_all(CACHE_CONTROL) else { + return Some(Duration::ZERO); + }; + let from_cache_control = + (!cache_control.is_empty()).then(|| cache_max_age(&cache_control.join(", "))).flatten(); + let lifetime = match from_cache_control { + Some(lifetime) => lifetime, + None => { + let expires = header(EXPIRES)?; + httpdate::parse_http_date(expires) + .ok() + .and_then(|expires| expires.duration_since(date.unwrap_or(now)).ok()) + .unwrap_or(Duration::ZERO) + } + }; + Some(lifetime.saturating_sub(current_age(headers, now))) +} + +/// How old the response already is (RFC 9111 §4.2.3): the larger of its `Age` +/// and its apparent age, `now` less its `Date` (a `Date` in the future — clock +/// skew — is an apparent age of zero). Every `Age` line counts and the greatest +/// wins; one that is sent but does not parse — overflowing, not a number, not +/// even ASCII — is taken as the largest age, not as none (RFC 9111 §1.2.2 has +/// oversized delta-seconds treated as the greatest value): a response of +/// unknowable age is not given a whole lifetime. +fn current_age(headers: &HeaderMap, now: SystemTime) -> Duration { + let age = headers + .get_all(AGE) + .iter() + .map(|v| { + v.to_str() + .ok() + .and_then(|v| v.trim().parse::().ok()) + .map_or(Duration::MAX, Duration::from_secs) + }) + .max() + .unwrap_or(Duration::ZERO); + let apparent_age = headers + .get(DATE) + .and_then(|v| v.to_str().ok()) + .and_then(|v| httpdate::parse_http_date(v.trim()).ok()) + .and_then(|date| now.duration_since(date).ok()) + .unwrap_or(Duration::ZERO); + age.max(apparent_age) +} + +/// The caching lifetime a `Cache-Control` value grants a SHARED cache — which the +/// caller's process-wide cache is: `s-maxage` if present, else `max-age`; zero +/// when the origin forbids shared reuse (`no-store`, `no-cache`, `private`); +/// `None` when it says none of these. Directives are recognised by NAME, so +/// `no-cache="set-cookie"` still counts. One given more than once is honoured at +/// its MOST RESTRICTIVE value, and one given without a valid number is ZERO — +/// stale, never a default lifetime (RFC 9111 §4.2.1: invalid or conflicting +/// freshness information must not extend freshness) — so `max-age=300, +/// max-age=0` is zero, not five minutes, and so is `max-age=soon`. A value that +/// cannot be parsed at all (an unterminated quoted-string) is zero too. +fn cache_max_age(cache_control: &str) -> Option { + let Some(directives) = cache_directives(cache_control) else { + return Some(Duration::ZERO); + }; + let has = |wanted: &str| directives.iter().any(|(name, _)| name.eq_ignore_ascii_case(wanted)); + if has("no-store") || has("no-cache") || has("private") { + return Some(Duration::ZERO); + } + // A directive's lifetime: `None` when absent; else the most restrictive of + // its values, an unparseable one counting as zero. + let lifetime = |wanted: &str| { + directives + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case(wanted)) + .map(|(_, arg)| { + arg.as_deref() + .and_then(|a| a.parse::().ok()) + .map_or(Duration::ZERO, Duration::from_secs) + }) + .min() + }; + lifetime("s-maxage").or_else(|| lifetime("max-age")) +} + +/// The directives of a `Cache-Control` value, each as (name, argument), split at +/// the commas that are NOT inside a quoted-string: an argument may be quoted and +/// then contain commas and backslash-escaped characters (RFC 9110 §5.6.4), so +/// `foo="x,s-maxage=86400", max-age=60` is `foo` and `max-age`, not an +/// `s-maxage` of a day. Quotes and escapes are removed from the argument. `None` +/// when the value cannot be parsed (a quoted-string never closes). +fn cache_directives(value: &str) -> Option)>> { + let mut raw: Vec = Vec::new(); + let mut current = String::new(); + let (mut in_quotes, mut escaped) = (false, false); + for c in value.chars() { + if escaped { + current.push(c); + escaped = false; + continue; + } + match c { + '\\' if in_quotes => { + escaped = true; + current.push(c); + } + '"' => { + in_quotes = !in_quotes; + current.push(c); + } + ',' if !in_quotes => raw.push(std::mem::take(&mut current)), + _ => current.push(c), + } + } + if in_quotes || escaped { + return None; + } + raw.push(current); + let unquote = |arg: &str| -> String { + let Some(inner) = arg.strip_prefix('"').and_then(|a| a.strip_suffix('"')) else { + return arg.to_owned(); + }; + let mut out = String::with_capacity(inner.len()); + let mut chars = inner.chars(); + while let Some(c) = chars.next() { + out.push(if c == '\\' { chars.next().unwrap_or(c) } else { c }); + } + out + }; + Some( + raw.iter() + .map(|d| d.trim()) + .filter(|d| !d.is_empty()) + .map(|d| match d.split_once('=') { + Some((name, arg)) => (name.trim().to_owned(), Some(unquote(arg.trim()))), + None => (d.to_owned(), None), + }) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::{accept, cache_max_age, current_age, fetch_public_document, freshness, FetchError}; + + /// A response as the origin might send it, for pinning the acceptance rules + /// without a network: `status`, `headers` (repeatable), `body`. + fn synthetic(status: u16, headers: &[(&str, &str)], body: &[u8]) -> reqwest::Response { + let mut b = http::Response::builder().status(status); + for (name, value) in headers { + b = b.header(*name, *value); + } + reqwest::Response::from(b.body(body.to_vec()).expect("synthetic response")) + } + + const URL: &str = "https://client.example/client.json"; + + /// The SSRF guard decides before any request: these never touch the network + /// (IP-literal hosts need no DNS), and each is refused for the reason the + /// guard names — as `Refused`, the failure that is about the URL. + #[tokio::test] + async fn guard_refuses_before_fetching() { + let fetch = |url: &'static str| fetch_public_document(url, 1024, Duration::from_secs(1)); + let Err(FetchError::Refused(why)) = fetch("http://example.com/client.json").await else { + panic!("http must be refused by the guard"); + }; + assert!(why.contains("only https"), "{why}"); + for internal in [ + "https://127.0.0.1/client.json", + "https://10.0.0.1/client.json", + "https://192.168.1.1/client.json", + "https://192.88.99.1/client.json", + "https://169.254.169.254/latest/meta-data/", + "https://[::1]/client.json", + "https://[::ffff:127.0.0.1]/client.json", + "https://[fec0::1]/client.json", + "https://[fe80::1]/client.json", + "https://[fd00::1]/client.json", + "https://[100::1]/client.json", + "https://[3fff::1]/client.json", + "https://[5f00::1]/client.json", + "https://[4000::1]/client.json", + "https://[2001:1::4]/client.json", + ] { + let Err(FetchError::Refused(why)) = fetch(internal).await else { + panic!("{internal} must be refused by the guard"); + }; + assert!(why.contains("non-public address"), "{internal}: {why}"); + } + assert!(matches!(fetch("not a url").await, Err(FetchError::Refused(_)))); + // An uncapped read is a valid request, not an overflow. + let uncapped = fetch_public_document("https://[::1]/x", usize::MAX, Duration::from_secs(1)); + assert!(matches!(uncapped.await, Err(FetchError::Refused(_)))); + } + + /// One deadline over the whole fetch: with no time at all, the operation fails + /// with the deadline's error whether it ran out during DNS resolution or after + /// (on a fast resolver, during the connect) — never with a request error of its + /// own, since the client sets no separate timeout. + #[tokio::test] + async fn one_deadline_covers_the_whole_fetch() { + let err = fetch_public_document("https://example.com/client.json", 1024, Duration::ZERO) + .await + .unwrap_err(); + let FetchError::Unreachable(why) = &err else { panic!("{err:?}") }; + assert!(why.contains("did not complete within"), "{why}"); + } + + /// A redirect is refused as such — its target is never requested, since the + /// client follows none — and so is any other non-2xx answer, each carrying its + /// status so the caller can tell "no document there" from "not right now". + #[tokio::test] + async fn accept_refuses_redirects_and_errors() { + for status in [301u16, 302, 307, 308] { + let resp = synthetic(status, &[("location", "https://client.example/other.json")], b""); + let err = accept(URL, resp, 1024).await.unwrap_err(); + let FetchError::Answered { status: got, detail } = &err else { panic!("{err:?}") }; + assert_eq!(*got, status); + assert!(detail.contains("not followed"), "{detail}"); + } + // Any other 2xx too: a 206 is a fragment (here a valid-looking one), a 203 + // has been transformed, a 204 has no body. Only 200 is the document. + for status in [404u16, 500, 503, 206, 203, 204] { + let body = br#"{"client_id":"x"}"#; + let err = accept(URL, synthetic(status, &[], body), 1024).await.unwrap_err(); + assert!( + matches!(err, FetchError::Answered { status: got, .. } if got == status), + "{err:?}" + ); + } + } + + /// The body is taken only complete and only as valid UTF-8; the media type and + /// the remaining freshness ride along. + #[tokio::test] + async fn accept_takes_only_a_complete_valid_body() { + let headers = + [("content-type", "application/json; charset=utf-8"), ("cache-control", "max-age=300")]; + let doc = + accept(URL, synthetic(200, &headers, br#"{"client_id":"x"}"#), 1024).await.unwrap(); + assert_eq!(doc.body, r#"{"client_id":"x"}"#); + assert_eq!(doc.content_type.as_deref(), Some("application/json; charset=utf-8")); + assert_eq!(doc.cache_max_age, Some(Duration::from_secs(300))); + // Over the cap: an error, not a truncated document. The cap is exact. + let body = br#"{"client_id":"x"}"#; + assert!(accept(URL, synthetic(200, &[], body), body.len()).await.is_ok()); + let err = accept(URL, synthetic(200, &[], body), body.len() - 1).await.unwrap_err(); + assert!(matches!(err, FetchError::TooLarge(_)), "{err:?}"); + // A byte that is not UTF-8 is refused, not replaced. + let err = accept(URL, synthetic(200, &[], b"{\"client_name\":\"\xff\"}"), 1024) + .await + .unwrap_err(); + assert!(matches!(err, FetchError::NotUtf8(_)), "{err:?}"); + } + + /// Freshness follows HTTP: every `Cache-Control` line counts, and the current + /// age — `Age`, or the time since `Date`, whichever is larger — is subtracted, + /// so a CDN answer near the end of its life is not given a new one. + #[test] + fn freshness_honours_age_and_every_cache_control_line() { + use std::time::{SystemTime, UNIX_EPOCH}; + // A whole-second "now", since an HTTP date has no finer resolution. + let since_epoch = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + let now = UNIX_EPOCH + Duration::from_secs(since_epoch); + let headers = |pairs: &[(&str, &str)]| { + let mut h = reqwest::header::HeaderMap::new(); + for (name, value) in pairs { + h.append( + reqwest::header::HeaderName::from_bytes(name.as_bytes()).unwrap(), + value.parse().unwrap(), + ); + } + h + }; + assert_eq!(freshness(&headers(&[]), now), None); + assert_eq!(freshness(&headers(&[("age", "10")]), now), None); + assert_eq!( + freshness(&headers(&[("cache-control", "public, max-age=86400")]), now), + Some(Duration::from_secs(86400)) + ); + assert_eq!( + freshness(&headers(&[("cache-control", "max-age=86400"), ("age", "86399")]), now), + Some(Duration::from_secs(1)) + ); + // Freshness already spent: zero, not negative and not a fresh lifetime. + assert_eq!( + freshness(&headers(&[("cache-control", "max-age=300"), ("age", "301")]), now), + Some(Duration::ZERO) + ); + // An Age that overflows, or is no number, is the greatest age, not none. + let overflowing = [("cache-control", "max-age=86400"), ("age", "99999999999999999999")]; + assert_eq!(freshness(&headers(&overflowing), now), Some(Duration::ZERO)); + let nonsense = [("cache-control", "max-age=86400"), ("age", "soon")]; + assert_eq!(freshness(&headers(&nonsense), now), Some(Duration::ZERO)); + // Every Age line counts, the greatest winning; one that is not even ASCII + // is the greatest age too. + let two_lines = [("cache-control", "max-age=300"), ("age", "10"), ("age", "400")]; + assert_eq!(freshness(&headers(&two_lines), now), Some(Duration::ZERO)); + let mut opaque = headers(&[("cache-control", "max-age=86400")]); + opaque.append( + reqwest::header::AGE, + reqwest::header::HeaderValue::from_bytes(b"\xff").unwrap(), + ); + assert_eq!(freshness(&opaque, now), Some(Duration::ZERO)); + // A Cache-Control (or Vary) line that cannot be decoded is read as + // forbidding reuse, never skipped: it may be the restrictive one. + let mut undecodable = headers(&[("cache-control", "max-age=86400")]); + undecodable.append( + reqwest::header::CACHE_CONTROL, + reqwest::header::HeaderValue::from_bytes(b"foo=\"\xff\", max-age=0").unwrap(), + ); + assert_eq!(freshness(&undecodable, now), Some(Duration::ZERO)); + // The current age is reported on its own too, for a caller applying a + // lifetime of its own where the origin sent none: a day-old answer must + // not get a fresh default there either. + assert_eq!(current_age(&headers(&[]), now), Duration::ZERO); + assert_eq!(current_age(&headers(&[("age", "86400")]), now), Duration::from_secs(86400)); + let dated_100 = httpdate::fmt_http_date(now - Duration::from_secs(100)); + let held_long = [("date", dated_100.as_str()), ("age", "50")]; + assert_eq!(current_age(&headers(&held_long), now), Duration::from_secs(100)); + assert_eq!(current_age(&headers(&[("age", "soon")]), now), Duration::MAX); + let mut odd_vary = headers(&[("cache-control", "max-age=86400")]); + odd_vary.append( + reqwest::header::VARY, + reqwest::header::HeaderValue::from_bytes(b"\xff").unwrap(), + ); + assert_eq!(freshness(&odd_vary, now), Some(Duration::ZERO)); + // Two Cache-Control lines: the no-store on the second is not missed. + assert_eq!( + freshness( + &headers(&[("cache-control", "max-age=300"), ("cache-control", "no-store")]), + now + ), + Some(Duration::ZERO) + ); + // The apparent age counts too: a Date an hour old with no Age means the + // answer has been held for an hour, and five fresh minutes are long gone. + let dated = |secs_ago: u64| httpdate::fmt_http_date(now - Duration::from_secs(secs_ago)); + let stale = [("cache-control", "max-age=300"), ("date", &dated(3600))]; + assert_eq!(freshness(&headers(&stale), now), Some(Duration::ZERO)); + // The current age is the LARGER of Age and apparent age, either way round. + let aged = [("cache-control", "max-age=300"), ("date", &dated(50)), ("age", "100")]; + assert_eq!(freshness(&headers(&aged), now), Some(Duration::from_secs(200))); + let held = [("cache-control", "max-age=300"), ("date", &dated(100)), ("age", "50")]; + assert_eq!(freshness(&headers(&held), now), Some(Duration::from_secs(200))); + // A Date in the future (clock skew) is an apparent age of zero, not a + // negative one. + let future = httpdate::fmt_http_date(now + Duration::from_secs(3600)); + let skewed = [("cache-control", "max-age=300"), ("date", &future)]; + assert_eq!(freshness(&headers(&skewed), now), Some(Duration::from_secs(300))); + // Where Cache-Control grants no freshness, Expires decides — relative to + // Date, or to now without one — so it comes to "Expires less now"; an + // Expires already past, or invalid ("0" is the classic), is stale, never + // the default lifetime. + let ahead = |secs: u64| httpdate::fmt_http_date(now + Duration::from_secs(secs)); + assert_eq!( + freshness(&headers(&[("expires", &ahead(120))]), now), + Some(Duration::from_secs(120)) + ); + let (date, expires) = (dated(100), ahead(200)); + let with_date = [("date", date.as_str()), ("expires", expires.as_str())]; + assert_eq!(freshness(&headers(&with_date), now), Some(Duration::from_secs(200))); + assert_eq!(freshness(&headers(&[("expires", &dated(3600))]), now), Some(Duration::ZERO)); + assert_eq!(freshness(&headers(&[("expires", "0")]), now), Some(Duration::ZERO)); + let public = [("cache-control", "public"), ("expires", &ahead(90))]; + assert_eq!(freshness(&headers(&public), now), Some(Duration::from_secs(90))); + // A Cache-Control that does speak to freshness wins over Expires, either way. + let both = [("cache-control", "max-age=60"), ("expires", &ahead(86400))]; + assert_eq!(freshness(&headers(&both), now), Some(Duration::from_secs(60))); + let forbidden = [("cache-control", "no-store"), ("expires", &ahead(86400))]; + assert_eq!(freshness(&headers(&forbidden), now), Some(Duration::ZERO)); + // `Vary: *` can never match a later request: no freshness for a shared + // cache, whatever the lifetime says — on its own or among other fields. + let varies = [("cache-control", "max-age=86400"), ("vary", "*")]; + assert_eq!(freshness(&headers(&varies), now), Some(Duration::ZERO)); + let among = [("cache-control", "max-age=60"), ("vary", "Accept-Encoding, *")]; + assert_eq!(freshness(&headers(&among), now), Some(Duration::ZERO)); + let ordinary = [("cache-control", "max-age=60"), ("vary", "accept-encoding")]; + assert_eq!(freshness(&headers(&ordinary), now), Some(Duration::from_secs(60))); + } + + #[test] + fn cache_control_lifetime() { + assert_eq!(cache_max_age("max-age=300"), Some(Duration::from_secs(300))); + assert_eq!( + cache_max_age("public, max-age=86400, immutable"), + Some(Duration::from_secs(86400)) + ); + assert_eq!(cache_max_age("Max-Age=\"60\""), Some(Duration::from_secs(60))); + assert_eq!(cache_max_age("no-store"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=300, no-cache"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("public"), None); + // Given but not a number: stale, never the default lifetime. + assert_eq!(cache_max_age("max-age=soon"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=300, max-age=soon"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("s-maxage=soon, max-age=300"), Some(Duration::ZERO)); + // A duplicated max-age is honoured at its most restrictive value, never + // the one that happens to come first. + assert_eq!(cache_max_age("max-age=300, max-age=0"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=0, max-age=300"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=300, public, max-age=60"), Some(Duration::from_secs(60))); + // The caller is a SHARED cache: `private` forbids it reuse, and `s-maxage` + // is its lifetime whenever present, over `max-age`. + assert_eq!(cache_max_age("private, max-age=86400"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("s-maxage=0, max-age=86400"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=600, s-maxage=60"), Some(Duration::from_secs(60))); + assert_eq!(cache_max_age("s-maxage=600, max-age=60"), Some(Duration::from_secs(600))); + // Directives are matched by name, however they are argued. + assert_eq!(cache_max_age("max-age=300, no-cache=\"set-cookie\""), Some(Duration::ZERO)); + assert_eq!(cache_max_age("No-Store"), Some(Duration::ZERO)); + // A comma inside a quoted argument does not start a directive, so an + // extension's argument cannot smuggle in a day of freshness; escapes are + // honoured; a quoted-string that never closes is not reused at all. + assert_eq!( + cache_max_age("foo=\"x,s-maxage=86400\", max-age=60"), + Some(Duration::from_secs(60)) + ); + assert_eq!(cache_max_age("ext=\"a\\\"b,c\", max-age=30"), Some(Duration::from_secs(30))); + assert_eq!(cache_max_age("foo=\"x, max-age=60"), Some(Duration::ZERO)); + assert_eq!(cache_max_age("max-age=\"45\""), Some(Duration::from_secs(45))); + } + + /// A host that cannot be resolved is a failure of the MOMENT, not of the URL: + /// `Unreachable`, never `Refused`, or a caller that remembers refusals would + /// remember a resolver outage as "no document there". + #[tokio::test] + async fn unresolvable_host_is_unreachable_not_refused() { + let err = fetch_public_document( + "https://does-not-exist.invalid/client.json", + 1024, + Duration::from_secs(5), + ) + .await + .unwrap_err(); + assert!(matches!(err, FetchError::Unreachable(_)), "{err:?}"); + } +} diff --git a/deploy/native/README.md b/deploy/native/README.md index 4c0d592..7f79c0e 100644 --- a/deploy/native/README.md +++ b/deploy/native/README.md @@ -300,6 +300,14 @@ addresses, since the ship job runs inside the VPN. | `DEPLOY_KNOWN_HOSTS` | `PROD_DEPLOY_KNOWN_HOSTS` | *(optional)* output of `ssh-keyscan `; pin it to avoid trust-on-first-use | | `OPENAI_APPS_CHALLENGE_TOKEN` | *(same name)* | *(optional)* OpenAI Apps domain-verification token, served at `/.well-known/openai-apps-challenge` (404 while unset). Submission-specific rather than host-specific, so one repository-level secret feeds both environments | +One setting is a GitHub Environment **variable** rather than a secret (**Settings → +Environments → *staging* / *production* → Variables**); `deploy.sh` renders it into +the unit like the secrets above: + +| Variable | Value | +|---|---| +| `OAUTH_CIMD_ENABLED` | `1` to advertise Client ID Metadata Documents — the registration mode Claude and ChatGPT prefer over DCR — on that environment; unset or empty is off. Off by default so a routine deploy never switches the directory clients over by itself: enable it on staging first, then production. The value is rendered into the unit at deploy time and read once at start-up, so to roll back, unset (or clear) the variable **and redeploy** — `workflow_dispatch` with the same ref is enough, no rebuild; changing the variable alone changes nothing on the host | + > **Set these as repository-level secrets** (**Settings → Secrets and variables → > Actions**). The callers pass them into the reusable workflow, and a job that calls > one with `uses:` cannot itself declare an `environment:` — so `${{ secrets.* }}` in diff --git a/deploy/native/deploy.sh b/deploy/native/deploy.sh index 47daf3d..d3150b2 100755 --- a/deploy/native/deploy.sh +++ b/deploy/native/deploy.sh @@ -80,7 +80,10 @@ tar -C "$repo_root" -cf - monitoring | $SSH "tar -C $REMOTE_DIR -xf -" echo ">> rendering + installing units and Caddyfile, then (re)starting services" # MCP_SERVE_BETA is set (to "1") only for the staging deployment, so /mcp-beta # is exposed there and not in production; it defaults to empty (off) otherwise. -unit_mcp="$(sed -e "s#__PUBLIC_URL__#https://$DOMAIN#g" -e "s#__MCP_SERVE_BETA__#${MCP_SERVE_BETA:-}#g" -e "s#__OPENAI_APPS_CHALLENGE_TOKEN__#${OPENAI_APPS_CHALLENGE_TOKEN:-}#g" "$here/imcp2.service")" +# OAUTH_CIMD_ENABLED ("1" to advertise Client ID Metadata Documents) comes from +# the GitHub Environment's variable of that name and defaults to empty (off), +# so enabling CIMD is a per-environment decision, never a side effect of a deploy. +unit_mcp="$(sed -e "s#__PUBLIC_URL__#https://$DOMAIN#g" -e "s#__MCP_SERVE_BETA__#${MCP_SERVE_BETA:-}#g" -e "s#__OPENAI_APPS_CHALLENGE_TOKEN__#${OPENAI_APPS_CHALLENGE_TOKEN:-}#g" -e "s#__OAUTH_CIMD_ENABLED__#${OAUTH_CIMD_ENABLED:-}#g" "$here/imcp2.service")" caddyfile="$(sed -e "s#__DOMAIN__#$DOMAIN#g" -e "s#__ACME_EMAIL__#$ACME_EMAIL#g" "$here/Caddyfile")" caddy_unit="$(cat "$here/caddy.service")" # The dashboard shows the monitored instances side by side. Every host renders diff --git a/deploy/native/imcp2.service b/deploy/native/imcp2.service index 6f64889..9c838b9 100644 --- a/deploy/native/imcp2.service +++ b/deploy/native/imcp2.service @@ -31,6 +31,15 @@ Environment=MCP_SERVE_METRICS=1 # endpoint by design; handled as a secret for consistency with the other # deploy inputs). Environment=OPENAI_APPS_CHALLENGE_TOKEN=__OPENAI_APPS_CHALLENGE_TOKEN__ +# Client ID Metadata Documents (CIMD), the registration mode Claude and ChatGPT +# prefer over DCR: OFF unless this is `1`. deploy.sh substitutes +# __OAUTH_CIMD_ENABLED__ from the OAUTH_CIMD_ENABLED variable of the GitHub +# Environment being deployed (empty when unset), so a routine deploy never +# switches the directory clients over by itself: enable it on staging first, +# then production, each deliberately. Rendered at deploy time and read once at +# start-up, so to roll back, unset the variable AND redeploy (the same ref will +# do — no rebuild); changing the variable alone changes nothing on the host. +Environment=OAUTH_CIMD_ENABLED=__OAUTH_CIMD_ENABLED__ # imcp2 creates its operational files (today: the dynamic-client-registration # store, so OAuth clients that cached their client_id keep working across # redeploys) in IMCP2_STATE_DIR. Point it at StateDirectory (=/var/lib/imcp2), diff --git a/docs/anthropic-directory-submission.md b/docs/anthropic-directory-submission.md index 8324ced..0b2828b 100644 --- a/docs/anthropic-directory-submission.md +++ b/docs/anthropic-directory-submission.md @@ -59,6 +59,7 @@ submission — and match a live scan of a deployed instance of that build | HTTPS remote server, Streamable HTTP transport | ✅ `rmcp` streamable-HTTP, stateless, JSON responses ([`src/lib.rs`](../src/lib.rs)) | | OAuth 2.0, authorization-code + PKCE **S256**, advertised in metadata | ✅ `code_challenge_methods_supported: ["S256"]` in the live RFC 8414 document | | Dynamic Client Registration (RFC 7591) — the out-of-the-box `oauth_dcr` mode | ✅ live probe: `POST /mcp/oauth/register` with the claude.ai callback → `201` | +| Client ID Metadata Documents — the `oauth_cimd` mode Anthropic recommends over DCR for directory listings | ✅ implemented, trust-policy-gated per the scoping in PR #143; advertised as `client_id_metadata_document_supported: true` alongside `"none"` in `token_endpoint_auth_methods_supported` — the two flags Claude requires to select CIMD — only where the deployment sets `OAUTH_CIMD_ENABLED=1` (off by default; enable per environment). Claude Code's live document (`https://claude.ai/oauth/claude-code-client-metadata`) is a fixture of the parsing test ([`src/auth.rs`](../src/auth.rs), `cimd_client_id` / `parse_client_metadata`) | | Claude's hosted callback `https://claude.ai/api/mcp/auth_callback` accepted | ✅ seeded in the redirect allow-list ([`src/auth.rs`](../src/auth.rs), `DEFAULT_ALLOWED_REDIRECTS`) | | Claude Code loopback redirects (RFC 8252) | ✅ loopback redirects are exempt from the hosted allow-list | | Discovery documents (RFC 8414 + RFC 9728, path-scoped + root fallback) | ✅ all four live, `WWW-Authenticate` on the 401 points at the resource metadata | @@ -75,11 +76,13 @@ submission — and match a live scan of a deployed instance of that build Notes on auth mode: pure M2M `client_credentials` is unsupported by Claude (every connection needs a user in the loop) — IMCP2's user-consent flow via -Internet Identity is exactly the supported shape. Claude registers a new DCR -client on each fresh connection; the server's registration store is a bounded -LRU of 10,000, which tolerates that churn, but Anthropic recommends **CIMD** -(Client ID Metadata Documents) for high-traffic directory listings — worth -considering as a follow-up if usage grows. +Internet Identity is exactly the supported shape. Against a DCR-only server +Claude registers a new client on each fresh connection (the registration store +is a bounded LRU of 10,000, which tolerates that churn); Anthropic recommends +**CIMD** (Client ID Metadata Documents) for high-traffic directory listings, +and the server implements it (PR #143's trust-policy-gated design) and +advertises it where `OAUTH_CIMD_ENABLED=1` is set, so there Claude selects CIMD +and registers nothing. ## Blockers to resolve before submitting diff --git a/docs/openai-directory-submission.md b/docs/openai-directory-submission.md index 746e258..e22314e 100644 --- a/docs/openai-directory-submission.md +++ b/docs/openai-directory-submission.md @@ -60,7 +60,7 @@ add details not published in the docs. | Requirement | Status | |---|---| | OAuth 2.1 authorization-code + PKCE **S256**, per the MCP authorization spec | ✅ live; `code_challenge_methods_supported: ["S256"]` | -| Client registration: DCR (`registration_endpoint`) — CIMD and predefined clients also accepted | ✅ RFC 7591 DCR live and verified | +| Client registration: CIMD preferred; DCR (`registration_endpoint`) and predefined clients also accepted | ✅ both. CIMD implemented (trust-policy-gated per PR #143), advertised as `client_id_metadata_document_supported: true` where the deployment sets `OAUTH_CIMD_ENABLED=1` (off by default) — ChatGPT's live document (`https://chatgpt.com/oauth/client.json`) is a fixture of the parsing test ([`src/auth.rs`](../src/auth.rs)); it prefers `private_key_jwt` but lists `none`, which is what it uses against this AS — and RFC 7591 DCR live and verified | | Discovery documents (RFC 8414 AS metadata + RFC 9728 protected-resource) | ✅ all live, path-scoped + root fallback | | Both of ChatGPT's callbacks accepted — `https://chatgpt.com/connector_platform_oauth_redirect` (the form it sends us) and `https://chatgpt.com/connector/oauth/{callback_id}` | ✅ the redirect allow-list pins both paths for `chatgpt.com` ([`src/auth.rs`](../src/auth.rs), `DEFAULT_ALLOWED_REDIRECTS`) | | No machine-to-machine grants (client credentials etc. unsupported by ChatGPT) | ✅ user-consent authorization-code flow only | diff --git a/monitoring/mcp-status/checks.js b/monitoring/mcp-status/checks.js index bb0070b..bf39b28 100644 --- a/monitoring/mcp-status/checks.js +++ b/monitoring/mcp-status/checks.js @@ -415,7 +415,7 @@ export const checkMcpEndpoints = async ( id: "as-metadata", label: "OAuth Authorization Server Metadata", description: - "Verifies the RFC 8414 metadata advertising the authorize/token/registration endpoints and PKCE support that clients need to log in.", + "Verifies the RFC 8414 metadata advertising the authorize/token/registration endpoints and PKCE support that clients need to log in, and reports whether Client ID Metadata Documents are advertised (the registration mode Claude and ChatGPT prefer over DCR; on only where the server runs with OAUTH_CIMD_ENABLED=1).", target: `GET ${url}`, expected: "200 JSON with issuer + authorize/token/register endpoints", status: pass ? "pass" : "fail", @@ -425,7 +425,7 @@ export const checkMcpEndpoints = async ( ? r.error ? `request failed: ${r.error.message}` : `${r.status}${redirectNote(r)}, missing fields: ${missing.join(", ") || "n/a"}` - : `issuer=${asMeta.issuer}, PKCE=${(asMeta.code_challenge_methods_supported || []).join(",") || "none"}`, + : `issuer=${asMeta.issuer}, PKCE=${(asMeta.code_challenge_methods_supported || []).join(",") || "none"}, CIMD=${asMeta.client_id_metadata_document_supported === true ? "on" : "off"}`, }); } diff --git a/monitoring/mcp-status/checks.test.js b/monitoring/mcp-status/checks.test.js index 6108cf5..b4c14eb 100644 --- a/monitoring/mcp-status/checks.test.js +++ b/monitoring/mcp-status/checks.test.js @@ -110,6 +110,7 @@ const healthyRoutes = (origin) => ({ token_endpoint: `${origin}/mcp/oauth/token`, registration_endpoint: `${origin}/mcp/oauth/register`, code_challenge_methods_supported: ["S256"], + client_id_metadata_document_supported: true, }), }), [`POST ${origin}/mcp`]: resp(401, { @@ -422,6 +423,9 @@ test("checkMcpEndpoints passes for a well-behaved server", async () => { assert.equal(byId(section, "root").status, "pass"); assert.equal(byId(section, "protected-resource").status, "pass"); assert.equal(byId(section, "as-metadata").status, "pass"); + // The detail line reports whether CIMD is advertised, so the kill switch's + // effect (or a regression that drops the flag) is visible on the dashboard. + assert.match(byId(section, "as-metadata").detail, /, CIMD=on$/); assert.equal(byId(section, "metadata-consistency").status, "pass"); assert.equal(byId(section, "mcp-challenge").status, "pass"); assert.equal(byId(section, "oauth-register").status, "pass"); diff --git a/src/auth.rs b/src/auth.rs index 6483236..0b2ece4 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -108,7 +108,7 @@ use base64::Engine; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, Semaphore}; use uuid::Uuid; use imcp2_core::identities::Identities; @@ -743,6 +743,746 @@ fn loopback_match(registered: &str, requested: &str) -> bool { a.host_str() == b.host_str() && a.path() == b.path() && a.query() == b.query() } +// ---- Client ID Metadata Documents (CIMD) ------------------------------------ +// +// The MCP authorization spec's preferred registration (draft-ietf-oauth-client- +// id-metadata-document): the client's `client_id` IS an https URL, and the +// RFC 7591-shaped JSON at that URL is its registration — `redirect_uris`, +// `client_name`, how it authenticates. Nothing is stored per client, so a +// directory client that connects thousands of times (Claude, ChatGPT) no longer +// mints a DCR registration each time. +// +// Adopted the way the scoping in PR #143 lays it out — TRUST-POLICY-GATED and +// additive. Only a `client_id` on a vetted vendor origin is fetched at all: its +// host must be on (or under) a domain of the hosted-redirect allow-list, on the +// default port ([`cimd_origin_trusted`]), so the one source of truth for who is +// a vetted vendor also decides whose document this server will GET. Any other +// URL `client_id` is refused before any request goes out and pointed at the +// allow-listing contact, exactly like a hosted redirect off the list; DCR stays +// for everyone else. That collapses the new outbound-fetch surface — an +// UNAUTHENTICATED `/oauth/authorize` naming a URL — from "any URL" to a finite +// set of vetted hosts. Even those are fetched under the SSRF guard +// ([`imcp2_core::public_fetch`]: every resolved address public and pinned, no +// redirect followed, a strict byte cap), because a vetted vendor's DNS is not +// this server's to trust. +// +// A fetched document is validated as the draft requires (its own `client_id` +// must equal the URL, byte for byte) and then given EXACTLY the checks a DCR +// registration gets: the requested `redirect_uri` must be one the document +// lists (loopback port-agnostically) AND pass the hosted-redirect allow-list +// ([`redirect_uri_permitted`]). A document cannot talk its way past the +// allow-list, so accepting one admits no redirect a DCR client could not already +// register. On top of that, a hosted redirect the document lists must be +// SAME-ORIGIN with the document URL (loopback excepted, for native clients): the +// document is self-asserted, so this is what ties the code's destination to the +// party that published the document, as Anthropic's reference authorization +// server also requires. Only public clients (`none`) are supported, as +// `token_endpoint_auth_methods_supported` says. +// +// Nothing in a document is trusted for DISPLAY. This server has no consent +// screen of its own (`/oauth/authorize` hands the browser to Internet Identity), +// so the relying party is not shown for CIMD clients any more than for DCR +// ones; were one added, the only attested fact is the HOST of the `client_id` +// URL — never the self-asserted `client_name` or `logo_uri`. +// +// OPT-IN per deployment: `OAUTH_CIMD_ENABLED=1` advertises the mechanism and +// accepts URL `client_id`s; unset, the metadata does not advertise it and a URL +// `client_id` is an unknown client. Claude and ChatGPT both switch to CIMD the +// moment an AS advertises it, so a routine deploy must never switch them over +// by itself (the deploy template takes the variable from the GitHub +// Environment). The rollback, should a vendor's document turn out to be shaped +// in a way this implementation refuses, is to unset the variable AND redeploy: +// it is rendered into the unit at deploy time and read here once at start-up +// ([`cimd_enabled_by_env`]), so changing it alone changes nothing on the host. +// Once the process restarts without it, the clients re-read the metadata within +// minutes and fall back to DCR. + +/// Byte cap on a `client_id` URL before it is treated as CIMD at all: the URL +/// becomes a key of the process-wide cache and single-flight map (and part of a +/// negative entry's reason), so an unauthenticated caller must not get to size +/// those entries at will — [`CIMD_CACHE_MAX`] entries of at most this many bytes +/// of key is the memory bound. The same cap a redirect URI gets; the real +/// identifiers are under 100 bytes. A longer value is an ordinary, unknown +/// client id: refused, not fetched, not remembered. +const CIMD_MAX_CLIENT_ID_LEN: usize = MAX_REDIRECT_URI_LEN; +/// Byte cap on a metadata document. The draft recommends documents stay under +/// 5 KB; the real ones are well under 1 KB, so this is generous yet bounded. +const CIMD_MAX_BYTES: usize = 8 * 1024; +/// Fetch timeout. Claude waits at most 10 s for OUR authorize endpoint, so the +/// fetch it triggers must finish well inside that. +const CIMD_FETCH_TIMEOUT: Duration = Duration::from_secs(5); +/// Fetches allowed in flight at once, across all documents and every mounted +/// instance — the bounds live in the process-wide [`CimdState`]. An excess +/// request is refused (told to retry), never queued, so a flood of distinct +/// `client_id` URLs at the unauthenticated endpoint holds at most this many +/// outbound requests open. Concurrent requests for ONE document never compete for these: +/// they wait for the single fetch in flight for it and read the cache after +/// ([`AuthStore::client_metadata_for`]). +const CIMD_MAX_INFLIGHT: usize = 8; +/// Fetches allowed in flight per `client_id` HOST, so one slow (or hostile) host +/// serving many distinct URLs cannot occupy every permit above: it gets this +/// many, and every other host keeps the rest. +const CIMD_MAX_INFLIGHT_PER_HOST: usize = 2; +/// Fetches allowed per minute, process-wide, however fast they complete. The +/// in-flight bounds cap how many run at once, not how many run in a minute: an +/// origin answering 404 at once gives its permit straight back, and distinct +/// paths on a vetted host defeat the negative cache, so this is what bounds the +/// outbound request rate an unauthenticated caller can drive at a vendor (the +/// rate cap PR #143 §5 requires alongside the concurrency cap). A token bucket: +/// this many tokens to start, and this many a minute of refill. Legitimate +/// traffic is a handful of documents, each cached for minutes to a day. +const CIMD_RATE_PER_MINUTE: u32 = 60; +/// Fetches allowed per minute for one vetted domain — the allow-list entry the +/// `client_id` host is on or under — so a flood at one vendor leaves the others +/// their share. Keyed by the DOMAIN, a finite vetted set, not the host: a +/// vendor's subdomains are not finite. +const CIMD_RATE_PER_DOMAIN_PER_MINUTE: u32 = 30; +/// Distinct `client_id` URLs cached. A handful of directory clients is the +/// expected population; the bound is against abuse, not for capacity. +const CIMD_CACHE_MAX: usize = 512; +/// How long a document is reused when its origin sends no `max-age`. +const CIMD_CACHE_DEFAULT_TTL: Duration = Duration::from_secs(10 * 60); +/// Ceiling on the origin's `max-age`: bounds how long a since-changed document is +/// still honoured. There is deliberately no floor — an origin's `no-store`, +/// `no-cache` or `max-age=0` means the document is not reused at all, so a +/// redirect the client withdraws is gone with the next request. The cost is a +/// fetch per request for a VALID document whose origin forbids reuse, which is +/// the origin's own choice and is contained like every other fetch: by the +/// in-flight and rate bounds. (An invalid document is a different case — it is +/// remembered for [`CIMD_NEGATIVE_TTL`], so a repeat costs nothing.) +const CIMD_CACHE_MAX_TTL: Duration = Duration::from_secs(24 * 60 * 60); +/// How long a URL whose document failed a DOCUMENT-INTRINSIC check — nothing +/// there (404), not JSON, about another URL, too large — is remembered as +/// invalid, so a repeat of the same bogus path costs no fetch (PR #143 §3.5). +/// Short, so a vendor fixing its document is not locked out for long. Only what +/// is about the URL itself is remembered: a per-request failure (a redirect the +/// document does not list) never is, or a probe with a bad redirect could lock +/// out a real client; nor is a transient (a deadline, a 5xx), which is retried. +const CIMD_NEGATIVE_TTL: Duration = Duration::from_secs(60); + +/// Whether this process is deployed with CIMD on: `OAUTH_CIMD_ENABLED` set to +/// an on-value ([`cimd_enabled_by`]). Read once (the env is process-static), +/// like the allow-list's `OAUTH_ALLOWED_REDIRECT_PREFIXES`; each [`AuthStore`] +/// takes its own copy at construction, which tests set directly. +fn cimd_enabled_by_env() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + let enabled = cimd_enabled_by(std::env::var("OAUTH_CIMD_ENABLED").ok().as_deref()); + if enabled { + tracing::info!("OAUTH_CIMD_ENABLED is set: Client ID Metadata Documents are on"); + } + enabled + }) +} + +/// The opt-in's reading of `OAUTH_CIMD_ENABLED`: `1`, `true`, `yes` and `on` +/// (any case) switch CIMD on; unset, empty, or anything else leaves it off. +fn cimd_enabled_by(value: Option<&str>) -> bool { + value.is_some_and(|v| { + matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on") + }) +} + +/// A validated Client ID Metadata Document: what this server needs from it. +#[derive(Clone, Debug, PartialEq, Eq)] +struct ClientMetadata { + /// The document URL, which is also the `client_id` (verified equal). + client_id: String, + client_name: Option, + redirect_uris: Vec, +} + +/// A cache entry: the validated document, or — a negative entry, held for +/// [`CIMD_NEGATIVE_TTL`] — why the URL yields none; either until `expires`. +#[derive(Clone, Debug)] +struct CachedClientMetadata { + outcome: Result, String>, + expires: Instant, +} + +/// Why a CIMD client's document did not yield a [`ClientMetadata`]. +#[derive(Clone, Debug, PartialEq, Eq)] +enum CimdError { + /// A failure of the MOMENT ([`classify_fetch_error`]): the host did not + /// resolve, the deadline passed, the connection failed, the origin answered + /// 5xx or one of the 4xx the client may retry (408, 421, 425, 429), or this + /// server's own fetch budget or in-flight bounds were spent. The same request may succeed next time, so the user is told + /// to retry and nothing is remembered. + Unavailable(String), + /// A failure of the URL or its document: the SSRF guard refuses the URL, the + /// origin has no document there (404, a redirect, any other 4xx), the body + /// is over the cap or not UTF-8, or it was fetched but is not a valid + /// document for that URL. The client is misconfigured or hostile: an unknown + /// client, not a retry, and remembered for [`CIMD_NEGATIVE_TTL`]. + Invalid(String), +} + +/// The verdict of [`AuthStore::validate_client`]. +#[derive(Debug, PartialEq, Eq)] +enum ClientCheck { + /// Known client, and `redirect_uri` is one it registered. + Allowed, + /// Unknown client, or a redirect it did not register or that is not permitted. + Refused, + /// A CIMD client whose document could not be fetched right now — retryable, + /// so the user is not told to re-add the connector. The reason is logged and + /// carried for the caller's own logging; it is not shown to the browser. + MetadataUnavailable(String), + /// A URL `client_id` whose origin is not on the vendor trust policy + /// ([`cimd_origin_trusted`]): refused before any fetch and — like a hosted + /// redirect off the allow-list — told where to request access. + UntrustedClientOrigin, +} + +/// One in-flight fetch of a document, shared by every request that missed the +/// cache while it ran: the slot holds the outcome once the fetch is done, and +/// whoever finds it empty on locking is the one to fetch. +type Flight = Arc, CimdError>>>>; + +/// Whether `client_id` is a Client ID Metadata Document URL — returned parsed — +/// or `None` for an ordinary (DCR) identifier. A CIMD `client_id` must be https, +/// name a host, carry a path beyond `/`, and have no fragment or userinfo (the +/// draft's MUSTs; a query is only discouraged there, so one is tolerated), and +/// must fit [`CIMD_MAX_CLIENT_ID_LEN`], since it is about to become a cache key. +/// The draft asks for no particular spelling beyond that: the identifier is +/// taken AS GIVEN — it is what the document must repeat byte for byte, and what +/// the cache is keyed by — so `https://ChatGPT.com/…` or an explicit `:443` is +/// a client like any other, provided its document says the same. Only the host +/// is normalised, and only for the trust policy and the per-host quota +/// ([`host_key`]), so no spelling of a vetted host is a stranger or a second +/// quota. Taken as given also means the raw string must BE the URL parsed: one +/// the parser would silently alter (stripping tab/newline/CR, or an empty +/// `@` userinfo) is refused. +fn cimd_client_id(client_id: &str) -> Option { + if client_id.len() > CIMD_MAX_CLIENT_ID_LEN { + return None; + } + // The WHATWG parser silently strips ASCII tab/newline/CR from anywhere in its + // input, trims leading and trailing C0 controls and spaces, reads a backslash + // as a slash (`https:\\host\path` parses as `https://host/path`, and one in + // the authority would end it before an `@` a raw scan expects there), and + // erases an EMPTY userinfo (`https://@host` parses as `https://host`). All of + // it is refused on the RAW string — as `resource_matches_issuer` does — or the + // identifier taken as given would not be the URL that was parsed and fetched. + let trimmed_by_parser = |c: char| c <= ' '; + if client_id.contains(['\t', '\n', '\r', '\\']) + || client_id.starts_with(trimmed_by_parser) + || client_id.ends_with(trimmed_by_parser) + || raw_authority_has_userinfo(client_id) + { + return None; + } + // Parsed, not prefix-matched: a scheme is case-insensitive (`HTTPS://` is + // https), and a DCR id (`client-…`) is no URL at all, so it parses to nothing. + let url = url::Url::parse(client_id).ok()?; + let well_formed = url.scheme() == "https" + && url.host_str().is_some_and(|h| !h.is_empty()) + && url.path().len() > 1 + && url.fragment().is_none() + && url.username().is_empty() + && url.password().is_none(); + well_formed.then_some(url) +} + +/// The trust policy of PR #143: whether a CIMD `client_id` URL (already shaped by +/// [`cimd_client_id`]) is on an origin this server will fetch from — its host on, +/// or a dot-boundary subdomain of, a domain of the hosted-redirect allow-list +/// ([`allowed_redirects`], the one source of truth for who is a vetted vendor), +/// on the default https port. The port matters: the SSRF guard connects to the +/// port the URL names, so a host-only check would let `https://claude.ai:8443/…` +/// past a `claude.ai` gate and on to a service nobody vetted. +fn cimd_origin_trusted(client_id: &url::Url) -> bool { + client_id.scheme() == "https" + && client_id.port().is_none() + && client_id.host_str().is_some_and(allow_listed_domain) +} + +/// Whether `host` equals, or is a dot-boundary subdomain of, an allow-listed +/// registrable domain — the host rule of [`redirect_uri_permitted`], on its own. +fn allow_listed_domain(host: &str) -> bool { + vetted_domain(host).is_some() +} + +/// The allow-listed registrable domain `host` equals or is a dot-boundary +/// subdomain of, if any: the trust policy's match, and the key of the per-vendor +/// fetch rate ([`CIMD_RATE_PER_DOMAIN_PER_MINUTE`]). +fn vetted_domain(host: &str) -> Option<&'static str> { + let host = host_key(host); + allowed_redirects().iter().map(|(domain, _, _)| domain.as_str()).find(|domain| { + host == *domain || host.strip_suffix(*domain).is_some_and(|p| p.ends_with('.')) + }) +} + +/// One spelling per host — lower-case, no trailing dot — so that whatever is +/// keyed by host (the trust policy's match, the per-host in-flight slots) treats +/// `claude.ai`, `Claude.AI` and `claude.ai.` as the one host they resolve to. +fn host_key(host: &str) -> String { + host.trim_end_matches('.').to_ascii_lowercase() +} + +/// Parse and validate the document fetched from `client_id` (RFC 7591 client +/// metadata, per the CIMD draft). Accepted only if it is a JSON object whose +/// `client_id` equals the URL exactly; whose `redirect_uris` is a non-empty +/// array of strings within what a DCR registration may send ([`MAX_REDIRECT_URIS`] +/// of at most [`MAX_REDIRECT_URI_LEN`] bytes each); that carries no client secret; +/// and that can authenticate +/// as a PUBLIC client — its `token_endpoint_auth_method` is `none` (or absent: +/// a document may not use a secret-based method, so absence cannot mean +/// RFC 7591's `client_secret_basic` default) OR its +/// `token_endpoint_auth_methods_supported` lists `none`. ChatGPT's document is +/// the case for the latter: it prefers `private_key_jwt` but lists `none`, which +/// is what it uses against an AS that, like this one, offers only `none`. And it +/// must be able to run the ONE flow this server offers: `grant_types`, if given, +/// must list `authorization_code` and `response_types`, if given, `code` (RFC +/// 7591's defaults when absent) — as DCR refuses a registration whose grant +/// types lose `authorization_code` ([`granted_grant_types`]). +/// +/// Of the `redirect_uris`, only those this server could ever honour are kept: +/// each must be one a DCR registration could have registered +/// ([`redirect_uri_permitted`]: loopback, or https on an allow-listed host and +/// pinned path, and in either case without query or fragment — a fragment would +/// otherwise be ignored by the loopback match and admit a redirect DCR refuses), +/// and a hosted one must be on the SAME ORIGIN as the document URL — a +/// self-asserted document may not point the code at another party. A document +/// left with none is refused. +fn parse_client_metadata(client_id: &str, body: &str) -> Result { + let doc: Value = serde_json::from_str(body).map_err(|e| format!("not valid JSON: {e}"))?; + let Some(obj) = doc.as_object() else { + return Err("not a JSON object".into()); + }; + match obj.get("client_id").and_then(Value::as_str) { + Some(id) if id == client_id => {} + Some(id) => return Err(format!("its client_id is {id:?}, not the document URL")), + None => return Err("no client_id".into()), + } + if obj.contains_key("client_secret") || obj.contains_key("client_secret_expires_at") { + return Err("carries a client secret, which a metadata document must not".into()); + } + // Absent means `none` (see above); present, it must be a string — a wrong type + // is a malformed document, not an omission to read charitably. + let method = match obj.get("token_endpoint_auth_method") { + None => "none", + Some(Value::String(method)) => method.as_str(), + Some(other) => { + return Err(format!("token_endpoint_auth_method must be a string, not {other}")) + } + }; + let lists_none = obj + .get("token_endpoint_auth_methods_supported") + .and_then(Value::as_array) + .is_some_and(|methods| methods.iter().any(|m| m.as_str() == Some("none"))); + if method != "none" && !lists_none { + return Err(format!( + "authenticates only as {method:?}; this server supports public clients (none) only" + )); + } + // The flow: absent means RFC 7591's defaults (`authorization_code` / `code`); + // present, each must be a string array naming this server's one flow, or the + // client could never complete an authorization here (DCR refuses the same). + for (field, needed) in [("grant_types", "authorization_code"), ("response_types", "code")] { + match obj.get(field) { + None => {} + Some(Value::Array(list)) if list.iter().all(Value::is_string) => { + if !list.iter().any(|v| v.as_str() == Some(needed)) { + return Err(format!("{field} does not include {needed:?}, the only flow here")); + } + } + Some(_) => return Err(format!("{field} must be an array of strings")), + } + } + let listed: Vec<&str> = match obj.get("redirect_uris").and_then(Value::as_array) { + Some(list) if list.len() > MAX_REDIRECT_URIS => { + return Err(format!("too many redirect_uris ({}, max {MAX_REDIRECT_URIS})", list.len())) + } + Some(list) + if list.iter().any(|u| u.as_str().is_some_and(|u| u.len() > MAX_REDIRECT_URI_LEN)) => + { + return Err(format!("a redirect_uri is too long (max {MAX_REDIRECT_URI_LEN} bytes)")) + } + Some(list) if !list.is_empty() && list.iter().all(Value::is_string) => { + list.iter().filter_map(Value::as_str).collect() + } + _ => return Err("redirect_uris must be a non-empty array of strings".into()), + }; + let own_origin = url::Url::parse(client_id).map_err(|e| format!("client_id: {e}"))?.origin(); + let redirect_uris: Vec = listed + .into_iter() + .filter(|u| { + redirect_uri_permitted(u) + && (is_loopback_redirect(u) + || url::Url::parse(u).is_ok_and(|r| r.origin() == own_origin)) + }) + .map(str::to_owned) + .collect(); + if redirect_uris.is_empty() { + return Err("lists no redirect_uri this server could honour: one on its own origin that \ + the hosted-redirect allow-list admits, or a loopback one, neither with a \ + query or fragment" + .into()); + } + let client_name = obj.get("client_name").and_then(Value::as_str).map(str::to_owned); + Ok(ClientMetadata { client_id: client_id.to_owned(), client_name, redirect_uris }) +} + +/// Whether a `Content-Type` is `application/json` — the media type a Client ID +/// Metadata Document must be served as — by its essence, so parameters such as +/// `charset=utf-8` are fine and case does not matter. A document served as +/// anything else (or as nothing) is refused before it is parsed: bytes that +/// happen to parse as JSON on a page the origin did not mean as its OAuth +/// statement are not that statement. +fn is_json_media_type(content_type: Option<&str>) -> bool { + content_type + .map(|ct| ct.split(';').next().unwrap_or("").trim()) + .is_some_and(|essence| essence.eq_ignore_ascii_case("application/json")) +} + +/// What this process knows about the web of metadata documents, and how much of +/// it is being asked at once: the cache, the single-flight map, and the in-flight +/// bounds. ONE per process, shared by every [`AuthStore`] — the bundled binary +/// mounts a store per II instance (`/mcp`, `/mcp-beta`) — so the bounds hold per +/// process, as their docs say, rather than multiplying with the mounts, and a +/// document fetched for one mount serves the other. +struct CimdState { + /// Documents fetched and validated — or found invalid — keyed by the + /// `client_id` URL, each with the instant it goes stale. Bounded at + /// [`CIMD_CACHE_MAX`]; see [`AuthStore::remember_client_metadata`]. + cache: RwLock>, + /// Bounds concurrent metadata-document fetches at [`CIMD_MAX_INFLIGHT`]: each + /// is an outbound request an UNAUTHENTICATED `/oauth/authorize` can trigger. + inflight: Semaphore, + /// Single-flight: the fetch in flight for one `client_id`, whose outcome — + /// document, invalid, or unavailable — every request that missed while it + /// ran shares, instead of each fetching and each spending a permit. An entry + /// lives only while a fetch is in flight ([`Flight`]). + fetching: std::sync::Mutex>, + /// Fetches in flight per `client_id` host ([`HostSlot`]). + hosts: std::sync::Mutex>, + /// Fetches per minute: the process's bucket, and one per vetted domain. + rates: std::sync::Mutex, +} + +impl CimdState { + fn new() -> Self { + Self { + cache: RwLock::default(), + inflight: Semaphore::new(CIMD_MAX_INFLIGHT), + fetching: std::sync::Mutex::default(), + hosts: std::sync::Mutex::default(), + rates: std::sync::Mutex::new(Rates { + all: TokenBucket::per_minute(CIMD_RATE_PER_MINUTE), + per_domain: HashMap::new(), + }), + } + } + + /// The one instance every store in the process shares. + fn shared() -> Arc { + static SHARED: std::sync::OnceLock> = std::sync::OnceLock::new(); + Arc::clone(SHARED.get_or_init(|| Arc::new(Self::new()))) + } + + /// Take `flight`'s entry for `key` out of the single-flight map — if it is + /// still the one there; a newer flight for the same key is left alone. + fn retire_flight(&self, key: &str, flight: &Flight) { + let mut fetching = self.fetching.lock().expect("cimd fetch locks"); + if fetching.get(key).is_some_and(|current| Arc::ptr_eq(current, flight)) { + fetching.remove(key); + } + } +} + +/// The fetch-rate buckets ([`CIMD_RATE_PER_MINUTE`], [`CIMD_RATE_PER_DOMAIN_PER_MINUTE`]). +struct Rates { + all: TokenBucket, + per_domain: HashMap<&'static str, TokenBucket>, +} + +/// A token bucket: `capacity` tokens to start, refilled continuously at +/// `capacity` a minute, one taken per fetch. +struct TokenBucket { + capacity: f64, + tokens: f64, + refilled_at: Instant, +} + +impl TokenBucket { + fn per_minute(capacity: u32) -> Self { + Self { + capacity: f64::from(capacity), + tokens: f64::from(capacity), + refilled_at: Instant::now(), + } + } + + /// Refill for the time passed, then say whether a token is there to take. + fn has_token(&mut self) -> bool { + let now = Instant::now(); + let refill = now.duration_since(self.refilled_at).as_secs_f64() * self.capacity / 60.0; + self.tokens = (self.tokens + refill).min(self.capacity); + self.refilled_at = now; + self.tokens >= 1.0 + } + + /// Take the token [`Self::has_token`] just said was there. + fn take(&mut self) { + self.tokens -= 1.0; + } +} + +/// Fetches in flight per `client_id` host, held as a guard so a slot is given +/// back however the fetch ends ([`CIMD_MAX_INFLIGHT_PER_HOST`]). +struct HostSlot { + state: Arc, + host: String, +} + +impl HostSlot { + /// Take a slot for `host`, or `None` when it already holds the maximum. + fn take(state: &Arc, host: &str) -> Option { + let mut map = state.hosts.lock().expect("cimd host slots"); + let held = map.get(host).copied().unwrap_or(0); + if held >= CIMD_MAX_INFLIGHT_PER_HOST { + return None; + } + map.insert(host.to_owned(), held + 1); + Some(Self { state: Arc::clone(state), host: host.to_owned() }) + } +} + +impl Drop for HostSlot { + fn drop(&mut self) { + let mut map = self.state.hosts.lock().expect("cimd host slots"); + match map.get_mut(&self.host) { + Some(held) if *held > 1 => *held -= 1, + _ => { + map.remove(&self.host); + } + } + } +} + +/// A request's hold on a flight, from joining it to leaving — by returning, or +/// by being DROPPED, since an authorize future can be dropped at any await (the +/// client resets the stream). A flight that publishes its outcome is retired by +/// its fetcher at once ([`AuthStore::client_metadata_for`]); this guard is for +/// the flight that never gets that far because every request in it was +/// cancelled: the LAST holder out retires it, so no entry is left behind (on a +/// vetted host with unique URLs, that would be unbounded growth). While another +/// request still holds such a flight it stays where newcomers find it, so a +/// fetcher cancelled mid-way hands over to a waiter — which, finding no outcome +/// once the lock is its, fetches — instead of leaving the waiters on one flight +/// and newcomers on a second, fetching the same document twice. Only this +/// flight's own entry is ever removed, never a newer one. +struct FlightGuard<'a> { + state: &'a CimdState, + key: &'a str, + flight: &'a Flight, +} + +impl Drop for FlightGuard<'_> { + fn drop(&mut self) { + // Two handles are the map's and this request's; more means other requests + // still hold the flight, and the last of them retires it. The count is + // read under the map lock, where a newcomer would take its handle, so the + // two cannot cross. + let mut fetching = self.state.fetching.lock().expect("cimd fetch locks"); + let Some(current) = fetching.get(self.key) else { return }; + if !Arc::ptr_eq(current, self.flight) || Arc::strong_count(self.flight) > 2 { + return; + } + fetching.remove(self.key); + } +} + +/// How long to reuse a document: the remaining freshness the origin granted +/// (already less the response's age) capped at [`CIMD_CACHE_MAX_TTL`]; ZERO — do +/// not cache — when it forbade reuse or that freshness is spent; and, when it +/// sent no freshness information at all, the default LESS the age the response +/// already has, so an answer some cache along the way held for a day is not +/// given ten fresh minutes here. +fn cimd_ttl(remaining: Option, current_age: Duration) -> Duration { + remaining + .unwrap_or_else(|| CIMD_CACHE_DEFAULT_TTL.saturating_sub(current_age)) + .min(CIMD_CACHE_MAX_TTL) +} + +/// GET a metadata document: the process-global test fixture when one is +/// registered for `url`, else the real SSRF-guarded fetch. +async fn fetch_client_metadata_document( + url: &str, +) -> Result { + #[cfg(test)] + if let Some(served) = cimd_fixture::get(url) { + return match served { + cimd_fixture::Served::Now(answer) => { + // A real fetch suspends here; so does the stand-in, so tests see + // what concurrent requests do while one is in flight. + tokio::task::yield_now().await; + answer + } + cimd_fixture::Served::Never => std::future::pending().await, + }; + } + imcp2_core::public_fetch::fetch_public_document(url, CIMD_MAX_BYTES, CIMD_FETCH_TIMEOUT).await +} + +/// GET the document at `key` and validate it: the document plus how long its +/// origin lets it be reused ([`cimd_ttl`]), or why it is not one. +async fn fetch_and_validate_client_metadata( + key: &str, +) -> Result<(Arc, Duration), CimdError> { + let doc = fetch_client_metadata_document(key).await.map_err(classify_fetch_error)?; + if !is_json_media_type(doc.content_type.as_deref()) { + return Err(CimdError::Invalid(format!( + "{key}: served as {}, not application/json", + doc.content_type.as_deref().unwrap_or("no media type") + ))); + } + let meta = parse_client_metadata(key, &doc.body) + .map(Arc::new) + .map_err(|why| CimdError::Invalid(format!("{key}: {why}")))?; + Ok((meta, cimd_ttl(doc.cache_max_age, doc.current_age))) +} + +/// Sort a fetch failure by what it is ABOUT (PR #143 §3.4). The URL itself — the +/// guard refuses it, the origin has no document there or answers with a redirect +/// or another 4xx, the body is over the cap or not UTF-8 — is `Invalid`, which is +/// remembered for [`CIMD_NEGATIVE_TTL`]. The moment — a resolver that did not +/// answer, the deadline, a connection that failed, a 5xx, or one of the 4xx +/// that are about the moment too, which their definitions say the client may +/// retry (408 Request Timeout, 421 Misdirected Request, 425 Too Early, 429 Too +/// Many Requests) — is `Unavailable`, which is never remembered. +fn classify_fetch_error(err: imcp2_core::public_fetch::FetchError) -> CimdError { + use imcp2_core::public_fetch::FetchError; + match err { + FetchError::Unreachable(why) => CimdError::Unavailable(why), + FetchError::Answered { status, detail } + if status >= 500 || matches!(status, 408 | 421 | 425 | 429) => + { + CimdError::Unavailable(detail) + } + FetchError::Answered { detail, .. } => CimdError::Invalid(detail), + FetchError::Refused(why) | FetchError::TooLarge(why) | FetchError::NotUtf8(why) => { + CimdError::Invalid(why) + } + } +} + +/// The tests' stand-in for the web: what each `client_id` URL serves. Process- +/// global, like the web it stands in for, so concurrent tests use distinct URLs. +#[cfg(test)] +mod cimd_fixture { + use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + }; + + use imcp2_core::public_fetch::{FetchError, PublicDocument}; + + /// What the fixture does for a fetch of a URL. + #[derive(Clone)] + pub(super) enum Served { + /// Answer this, after the one suspension a real fetch would take. + Now(Result), + /// Never answer: an origin that hangs. The request's only way out is to + /// be dropped, as a client resetting the stream would drop it. + Never, + } + + type Registry = Mutex>; + static DOCS: OnceLock = OnceLock::new(); + static HITS: OnceLock>> = OnceLock::new(); + + fn docs() -> &'static Registry { + DOCS.get_or_init(Default::default) + } + + /// Serve `body` (JSON) at `url`, with no cache hint. + pub(super) fn serve(url: &str, body: &str) { + serve_as(url, body, "application/json"); + } + + /// Serve `body` at `url` with the given `Content-Type`, no cache hint. + pub(super) fn serve_as(url: &str, body: &str, content_type: &str) { + let doc = PublicDocument { + body: body.into(), + content_type: Some(content_type.into()), + cache_max_age: None, + current_age: std::time::Duration::ZERO, + }; + docs().lock().expect("fixture registry").insert(url.into(), Served::Now(Ok(doc))); + } + + /// Serve `body` (JSON) at `url` with no cache hint, as an answer some cache + /// along the way has already held for `age`. + pub(super) fn serve_aged(url: &str, body: &str, age: std::time::Duration) { + let doc = PublicDocument { + body: body.into(), + content_type: Some("application/json".into()), + cache_max_age: None, + current_age: age, + }; + docs().lock().expect("fixture registry").insert(url.into(), Served::Now(Ok(doc))); + } + + /// Make a fetch of `url` hang until the request is dropped. + pub(super) fn hang(url: &str) { + docs().lock().expect("fixture registry").insert(url.into(), Served::Never); + } + + /// How many times `url` has been fetched. + pub(super) fn hits(url: &str) -> usize { + HITS.get_or_init(Default::default) + .lock() + .expect("fixture hits") + .get(url) + .copied() + .unwrap_or(0) + } + + /// Serve `body` at `url` with `Cache-Control: no-store` (a zero max-age). + pub(super) fn serve_uncacheable(url: &str, body: &str) { + let doc = PublicDocument { + body: body.into(), + content_type: Some("application/json".into()), + cache_max_age: Some(std::time::Duration::ZERO), + current_age: std::time::Duration::ZERO, + }; + docs().lock().expect("fixture registry").insert(url.into(), Served::Now(Ok(doc))); + } + + /// Make fetching `url` fail TRANSIENTLY — an origin that cannot be reached — + /// with `why`. + pub(super) fn fail(url: &str, why: &str) { + let err = FetchError::Unreachable(why.into()); + docs().lock().expect("fixture registry").insert(url.into(), Served::Now(Err(err))); + } + + /// Make `url` answer 404: no document there, a failure about the URL itself. + pub(super) fn not_found(url: &str) { + answer(url, 404); + } + + /// Make `url` answer with `status` (anything but the `200 OK` a document is). + pub(super) fn answer(url: &str, status: u16) { + let err = FetchError::Answered { status, detail: format!("{url} answered {status}") }; + docs().lock().expect("fixture registry").insert(url.into(), Served::Now(Err(err))); + } + + pub(super) fn get(url: &str) -> Option { + let served = docs().lock().expect("fixture registry").get(url).cloned(); + if served.is_some() { + *HITS + .get_or_init(Default::default) + .lock() + .expect("fixture hits") + .entry(url.into()) + .or_insert(0) += 1; + } + served + } +} + #[derive(Clone)] pub struct AuthStore { clients: Arc, @@ -775,6 +1515,13 @@ pub struct AuthStore { /// [`crate::McpConfig::require_resource`]); when clear, a missing `resource` /// is tolerated. require_resource: bool, + /// Client ID Metadata Documents: the cache, single-flight map and in-flight + /// bounds — the PROCESS's ([`CimdState::shared`]), so every mounted instance + /// draws on the same bounds; a test may give a store its own. + cimd: Arc, + /// Whether URL `client_id`s are accepted and CIMD advertised: the process's + /// `OAUTH_CIMD_ENABLED` ([`cimd_enabled_by_env`]), or what a test set. + cimd_enabled: bool, } /// An auth-code connect awaiting the user's II handshake. @@ -893,9 +1640,20 @@ impl AuthStore { public_url, mcp_path, require_resource, + cimd: CimdState::shared(), + cimd_enabled: cimd_enabled_by_env(), } } + /// This store with CIMD switched on or off, whatever the environment says, + /// and with CIMD state of its own, so tests do not see each other's flights. + #[cfg(test)] + fn with_cimd(mut self, enabled: bool) -> Self { + self.cimd_enabled = enabled; + self.cimd = Arc::new(CimdState::new()); + self + } + /// The II instance this store serves. fn instance(&self) -> &imcp2_core::identities::IiInstance { self.identities.instance() @@ -921,13 +1679,222 @@ impl AuthStore { format!("{}/.well-known/oauth-protected-resource{}", self.public_url, self.mcp_path) } - /// Whether `redirect_uri` is acceptable for `client_id`: the client must be - /// registered, and the redirect must match a registered URI (exactly, or - /// port-agnostically for loopback per RFC 8252 §7.3). A match also marks the + /// Whether `redirect_uri` is acceptable for `client_id`. A CIMD client (its + /// `client_id` is an https URL, [`cimd_client_id`], and CIMD is on) must be + /// on a vetted vendor origin ([`cimd_origin_trusted`]) and is then checked + /// against its fetched, validated document; any other client must hold a + /// registration in the DCR store. Either way the redirect must match one the + /// client registered (exactly, or port-agnostically for loopback per RFC 8252 + /// §7.3) AND pass the hosted-redirect allow-list. A DCR match also marks the /// registration as recently used (it is about to sign a user in), which is /// what keeps it ahead of the store's LRU eviction. - async fn validate_client(&self, client_id: &str, redirect_uri: &str) -> bool { - self.clients.redirect_allowed_for(client_id, redirect_uri).await + async fn validate_client(&self, client_id: &str, redirect_uri: &str) -> ClientCheck { + let Some(cimd_url) = cimd_client_id(client_id).filter(|_| self.cimd_enabled) else { + return if self.clients.redirect_allowed_for(client_id, redirect_uri).await { + ClientCheck::Allowed + } else { + ClientCheck::Refused + }; + }; + // The trust policy, BEFORE anything else: a document is fetched from a + // vetted vendor origin or not at all, so an unauthenticated request naming + // a stranger's URL costs this server nothing and admits nothing. + if !cimd_origin_trusted(&cimd_url) { + // Debug, not info: this runs for every unauthenticated request, before + // any rate limit, and carries a caller-chosen URL — a flood of distinct + // strangers must not be a flood of log lines. + tracing::debug!(client_id, "refusing a client_id URL off the vendor trust policy"); + return ClientCheck::UntrustedClientOrigin; + } + // Allow-list BEFORE any fetch too: a redirect this server would refuse + // anyway must not cost an outbound request, so even a vetted host is not + // asked for a document on behalf of a redirect that could never be used. + if !redirect_uri_permitted(redirect_uri) { + return ClientCheck::Refused; + } + match self.client_metadata_for(client_id, &cimd_url).await { + Ok(meta) => { + // The same check a DCR registration gets, over the document's redirects. + let reg = ClientReg::new(meta.redirect_uris.clone()); + if redirect_allowed(Some(®), redirect_uri) { + ClientCheck::Allowed + } else { + ClientCheck::Refused + } + } + // Both outcomes are logged at warn WHERE THE FETCH HAPPENS (bounded by + // the fetch rate); here, per request — a negative-cache hit or a refused + // permit costs no fetch — only at debug, or a flood of requests for one + // bad URL would be a flood of log lines carrying its caller-chosen text. + Err(CimdError::Invalid(why)) => { + tracing::debug!(client_id, %why, "client metadata document is invalid"); + ClientCheck::Refused + } + Err(CimdError::Unavailable(why)) => { + tracing::debug!(client_id, %why, "client metadata document unavailable"); + ClientCheck::MetadataUnavailable(why) + } + } + } + + /// The validated metadata document behind a CIMD `client_id`, from the cache + /// while fresh, else fetched — once, however many requests miss at the same + /// time, all of which share that one fetch's outcome — under the in-flight + /// bounds. What is then cached, and for how long, is + /// [`AuthStore::fetch_and_cache_client_metadata`]'s call. + async fn client_metadata_for( + &self, + key: &str, + client_id: &url::Url, + ) -> Result, CimdError> { + // `key` is the identifier AS GIVEN — the string the document must repeat, + // and what the cache and single-flight map are keyed by; `client_id` is + // it parsed, for the host. + if let Some(cached) = self.cached_client_metadata(key).await { + return cached; + } + // Single-flight: the first miss for a document fetches it; the others wait + // for that fetch and take its outcome — a failure or an uncacheable + // document included, so nobody re-fetches serially behind a slow origin. + // Without this a popular client's cold start (or a document's expiry) + // would have every concurrent authorize fetch the same bytes and spend a + // permit each. + let flight: Flight = Arc::clone( + self.cimd.fetching.lock().expect("cimd fetch locks").entry(key.to_owned()).or_default(), + ); + // Held for as long as this request is in the flight, however it leaves; + // the last holder out retires the flight ([`FlightGuard`]). + let _hold = FlightGuard { state: &self.cimd, key, flight: &flight }; + let mut slot = flight.lock().await; + if let Some(outcome) = slot.as_ref() { + return outcome.clone(); + } + // First through the lock: this request fetches — unless a flight that + // finished between the miss above and here has filled the cache meanwhile + // — and publishes, for the waiters to read from the handle they hold. + let outcome = match self.cached_client_metadata(key).await { + Some(cached) => cached, + None => self.fetch_and_cache_client_metadata(key, client_id).await, + }; + // Retire the flight BEFORE publishing, still under its lock, so no request + // can join it once the outcome is there: a request arriving from here on + // goes to the cache, or, for an outcome the cache does not hold (a + // `no-store` document, a transient failure), fetches afresh — never joins + // this flight to reuse an outcome the origin said not to reuse, or a + // failure that may be over. Everyone who joined before this reads the + // outcome from the handle they already hold. + self.cimd.retire_flight(key, &flight); + *slot = Some(outcome.clone()); + outcome + } + + /// The cached outcome for `key`, if one is held and still fresh: the + /// document, or — a negative entry — why the URL yields none. + async fn cached_client_metadata( + &self, + key: &str, + ) -> Option, CimdError>> { + let cache = self.cimd.cache.read().await; + let hit = cache.get(key).filter(|hit| hit.expires > Instant::now())?; + Some(hit.outcome.clone().map_err(CimdError::Invalid)) + } + + /// Fetch, validate and cache the document at `client_id`. Bounded before any + /// request goes out: in rate, per vendor and overall, and in flight, per host + /// (so one slow host cannot take every permit) and overall. None of these + /// queues — an excess request is told to retry. + /// A document is cached for as long as [`cimd_ttl`] says, which is not at all + /// when its origin forbids reuse; a failure about the URL itself is cached + /// negatively for [`CIMD_NEGATIVE_TTL`]; a transient failure is not cached. + async fn fetch_and_cache_client_metadata( + &self, + key: &str, + client_id: &url::Url, + ) -> Result, CimdError> { + let host = host_key(client_id.host_str().unwrap_or_default()); + let Some(_slot) = HostSlot::take(&self.cimd, &host) else { + return Err(CimdError::Unavailable(format!( + "too many client metadata fetches in flight for {host}; retry shortly" + ))); + }; + let Ok(_permit) = self.cimd.inflight.try_acquire() else { + return Err(CimdError::Unavailable( + "too many client metadata fetches in flight; retry shortly".into(), + )); + }; + // The RATE last, once a slot and a permit are held, so a token is spent + // only on a fetch that goes out: a request refused for congestion drains + // no budget, or a burst during congestion could spend the minute's budget + // without a single fetch and lock the real clients out once it clears. + // The in-flight bounds cap how many fetches run at once, not how many run + // in a minute; this does. The vendor's share, then the process's — both + // must have a token before either is taken. + { + let mut rates = self.cimd.rates.lock().expect("cimd rates"); + let rates = &mut *rates; + let domain = vetted_domain(&host).unwrap_or_default(); + let vendor = rates + .per_domain + .entry(domain) + .or_insert_with(|| TokenBucket::per_minute(CIMD_RATE_PER_DOMAIN_PER_MINUTE)); + if !vendor.has_token() { + return Err(CimdError::Unavailable(format!( + "client metadata fetch rate for {domain} exceeded; retry shortly" + ))); + } + if !rates.all.has_token() { + return Err(CimdError::Unavailable( + "client metadata fetch rate exceeded; retry shortly".into(), + )); + } + vendor.take(); + rates.all.take(); + } + let fetched = Instant::now(); + // The one place these are logged at warn: a fetch happened, and fetches + // are rate-limited, so the log is bounded however the requests flood. + let (outcome, ttl) = match fetch_and_validate_client_metadata(key).await { + Ok((meta, ttl)) => (Ok(meta), ttl), + Err(CimdError::Invalid(why)) => { + tracing::warn!( + client_id = key, %why, + "client metadata document is invalid; refusing its client for a minute" + ); + (Err(why), CIMD_NEGATIVE_TTL) + } + Err(CimdError::Unavailable(why)) => { + tracing::warn!(client_id = key, %why, "client metadata document unavailable"); + return Err(CimdError::Unavailable(why)); + } + }; + if !ttl.is_zero() { + self.remember_client_metadata(key, outcome.clone(), fetched + ttl).await; + } + outcome.map_err(CimdError::Invalid) + } + + /// Hold `outcome` for `key` until `expires`, making room under + /// [`CIMD_CACHE_MAX`] first. + async fn remember_client_metadata( + &self, + key: &str, + outcome: Result, String>, + expires: Instant, + ) { + let now = Instant::now(); + let mut cache = self.cimd.cache.write().await; + if cache.len() >= CIMD_CACHE_MAX && !cache.contains_key(key) { + // Make room: drop what has expired; if that frees nothing, the entry + // closest to expiry (an LRU stand-in that needs no write per hit). + cache.retain(|_, c| c.expires > now); + if cache.len() >= CIMD_CACHE_MAX { + let victim = cache.iter().min_by_key(|(_, c)| c.expires).map(|(k, _)| k.clone()); + if let Some(victim) = victim { + cache.remove(&victim); + } + } + } + cache.insert(key.to_owned(), CachedClientMetadata { outcome, expires }); } /// The verified principal + session id behind a bearer token, if valid. @@ -1145,7 +2112,41 @@ pub async fn authorize( ) } } - if !store.validate_client(&q.client_id, &q.redirect_uri).await { + let client_check = store.validate_client(&q.client_id, &q.redirect_uri).await; + if let ClientCheck::MetadataUnavailable(_) = &client_check { + // A CIMD client this server could not verify RIGHT NOW (the cause is + // logged by `validate_client`): neither a malformed request nor a client + // to re-add, so say retry. Nothing about the failure is reflected here — + // the cause quotes the caller-supplied `client_id` URL. + return signin_error( + &headers, + StatusCode::SERVICE_UNAVAILABLE, + "temporarily_unavailable", + "the client's metadata document could not be fetched", + SIGNIN_HEADLINE, + "We couldn't fetch your MCP client's identity document just now. Try again in a \ + moment.", + ); + } + if client_check == ClientCheck::UntrustedClientOrigin { + // A URL `client_id` on an origin that is not a vetted vendor (the trust + // policy of PR #143), refused before any fetch. Like a hosted redirect + // off the allow-list, this is an approval gap with a concrete next step, + // not a malformed request — so the same "not approved" page, or its JSON. + return if accepts_html(&headers) { + not_allowlisted_page() + } else { + oauth_err( + StatusCode::FORBIDDEN, + "invalid_client", + &format!( + "client_id URL is not on a vetted vendor origin; contact {CONTACT} to request \ + access" + ), + ) + }; + } + if client_check != ClientCheck::Allowed { if !redirect_uri_permitted(&q.redirect_uri) { // Two distinct failures reach here. A WELL-FORMED hosted `redirect_uri` // that simply isn't on the allow-list is an approval gap, not a @@ -2076,6 +3077,14 @@ pub async fn authorization_server_metadata(State(store): State) -> Re "grant_types_supported": ["authorization_code"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["none"], + // The MCP authorization spec's preferred registration: a client may + // identify itself with the https URL of its Client ID Metadata Document + // instead of registering (see `cimd_client_id`). Claude and ChatGPT both + // select CIMD over DCR when this is advertised alongside `none` above — + // which is why it is advertised only where the deployment opts in with + // `OAUTH_CIMD_ENABLED`, and a redeploy without that withdraws it (no + // rebuild; the value is read once at start-up). + "client_id_metadata_document_supported": store.cimd_enabled, // RFC 9207: we emit `iss` on every authorization response, so we MUST // advertise it here (a client that sees this flag rejects any response // missing `iss`). See `build_redirect`. @@ -2416,6 +3425,857 @@ mod tests { } } + /// A Client ID Metadata Document `client_id` is an https URL naming a host + /// and a path beyond `/`, with no fragment or userinfo, within the length cap + /// — taken as given, in whatever spelling its document repeats (a query is + /// tolerated, as the draft only discourages one). Anything else is an + /// ordinary (DCR) identifier. + #[test] + fn cimd_client_id_shape() { + use super::cimd_client_id; + // The two directory clients' real identifiers. + assert!(cimd_client_id("https://chatgpt.com/oauth/client.json").is_some()); + assert!(cimd_client_id("https://claude.ai/oauth/claude-code-client-metadata").is_some()); + // A DCR identifier, and other non-URLs, are not CIMD. + assert!(cimd_client_id("client-3f6a9b2c-1d4e-4f5a-8b6c-7d8e9f0a1b2c").is_none()); + assert!(cimd_client_id("").is_none()); + // The draft's MUSTs. + assert!(cimd_client_id("http://chatgpt.com/oauth/client.json").is_none()); + assert!(cimd_client_id("https://chatgpt.com").is_none()); + assert!(cimd_client_id("https://chatgpt.com/").is_none()); + assert!(cimd_client_id("https://chatgpt.com/oauth/client.json#x").is_none()); + // A query is only discouraged by the draft, so it is tolerated — canonically. + assert!(cimd_client_id("https://chatgpt.com/oauth/client.json?v=2").is_some()); + assert!(cimd_client_id("https://user@chatgpt.com/oauth/client.json").is_none()); + // …including what the parser would silently alter: an EMPTY userinfo it + // erases, and the tab/newline/CR it strips from anywhere. + assert!(cimd_client_id("https://@chatgpt.com/oauth/client.json").is_none()); + assert!(cimd_client_id("https://chat\tgpt.com/oauth/client.json").is_none()); + assert!(cimd_client_id("https://chatgpt.com/oauth/client.json\n").is_none()); + assert!(cimd_client_id("https:\r//chatgpt.com/oauth/client.json").is_none()); + // …the backslashes it reads as slashes, wherever they are… + assert!(cimd_client_id("https:\\\\chatgpt.com\\oauth\\client.json").is_none()); + assert!(cimd_client_id("https://chatgpt.com\\@evil.example/oauth/client.json").is_none()); + assert!(cimd_client_id("https://chatgpt.com/oauth\\client.json").is_none()); + // …and the leading/trailing C0 controls and spaces it trims. + assert!(cimd_client_id(" https://chatgpt.com/oauth/client.json").is_none()); + assert!(cimd_client_id("https://chatgpt.com/oauth/client.json ").is_none()); + assert!(cimd_client_id("\u{1}https://chatgpt.com/oauth/client.json").is_none()); + // The draft asks for an https URL the document repeats byte for byte, not + // for one spelling of it: these are accepted as given (they are the + // identity and the cache key), and only the host is normalised, for the + // trust policy and the per-host quota (`host_key`). + for spelling in [ + "HTTPS://chatgpt.com/oauth/client.json", + "https://ChatGPT.com/oauth/client.json", + "https://chatgpt.com:443/oauth/client.json", + "https://chatgpt.com./oauth/client.json", + "https://chatgpt.com/oauth/../oauth/client.json", + ] { + assert!(cimd_client_id(spelling).is_some(), "{spelling}"); + } + // Bounded, since it is about to become a cache key: the cap exactly, not + // one byte more. + let at_cap = + format!("https://chatgpt.com/{}", "x".repeat(super::CIMD_MAX_CLIENT_ID_LEN - 20)); + assert_eq!(at_cap.len(), super::CIMD_MAX_CLIENT_ID_LEN); + assert!(cimd_client_id(&at_cap).is_some()); + assert!(cimd_client_id(&format!("{at_cap}x")).is_none()); + } + + /// Whatever is keyed by host sees one spelling per host, so a trailing dot or + /// upper case cannot buy a second per-host quota. + #[test] + fn cimd_host_key_is_one_spelling_per_host() { + use super::host_key; + for spelling in ["claude.ai", "Claude.AI", "claude.ai.", "CLAUDE.AI."] { + assert_eq!(host_key(spelling), "claude.ai", "{spelling}"); + } + } + + /// A document is accepted only as the draft and this public-client-only AS + /// require. The bodies are the two directory clients' real documents (as + /// served on 2026-09-03), so a change in how either identifies itself lands + /// here first. + #[test] + fn client_metadata_parsing() { + use super::parse_client_metadata; + use serde_json::json; + const CHATGPT: &str = "https://chatgpt.com/oauth/client.json"; + const CHATGPT_REDIRECT: &str = "https://chatgpt.com/connector_platform_oauth_redirect"; + let chatgpt_doc = json!({ + "client_id": CHATGPT, + "client_uri": "https://chatgpt.com/", + "redirect_uris": [CHATGPT_REDIRECT], + "token_endpoint_auth_method": "private_key_jwt", + "token_endpoint_auth_methods_supported": ["none", "private_key_jwt"], + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "client_name": "ChatGPT", + "logo_uri": "https://persistent.oaistatic.com/sonic/misc/openai-logo.png", + "token_endpoint_auth_signing_alg": "RS256", + "jwks_uri": "https://chatgpt.com/oauth/jwks.json", + }) + .to_string(); + const CLAUDE_CODE: &str = "https://claude.ai/oauth/claude-code-client-metadata"; + let claude_code_doc = json!({ + "client_id": CLAUDE_CODE, + "client_name": "Claude Code", + "client_uri": "https://claude.ai", + "redirect_uris": ["http://localhost/callback", "http://127.0.0.1/callback"], + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "token_endpoint_auth_method": "none", + }) + .to_string(); + // ChatGPT PREFERS private_key_jwt but lists `none`, which is what it uses here. + let chatgpt = parse_client_metadata(CHATGPT, &chatgpt_doc).expect("ChatGPT's document"); + assert_eq!(chatgpt.client_name.as_deref(), Some("ChatGPT")); + assert_eq!(chatgpt.redirect_uris, [CHATGPT_REDIRECT]); + let claude = parse_client_metadata(CLAUDE_CODE, &claude_code_doc).expect("Claude Code's"); + assert_eq!( + claude.redirect_uris, + ["http://localhost/callback", "http://127.0.0.1/callback"] + ); + // The document must be about the URL it was fetched from. + let err = parse_client_metadata(CLAUDE_CODE, &chatgpt_doc).unwrap_err(); + assert!(err.contains("not the document URL"), "{err}"); + // Required shape. + assert!(parse_client_metadata(CHATGPT, "not json").is_err()); + assert!(parse_client_metadata(CHATGPT, "[]").is_err()); + let no_id = json!({ "redirect_uris": [CHATGPT_REDIRECT] }).to_string(); + assert!(parse_client_metadata(CHATGPT, &no_id).unwrap_err().contains("no client_id")); + let refused_for = + |doc: serde_json::Value| parse_client_metadata(CHATGPT, &doc.to_string()).unwrap_err(); + assert!(refused_for(json!({ "client_id": CHATGPT })).contains("redirect_uris")); + let no_uris = json!({ "client_id": CHATGPT, "redirect_uris": [] }); + assert!(refused_for(no_uris).contains("redirect_uris")); + let bad_uris = json!({ "client_id": CHATGPT, "redirect_uris": [1] }); + assert!(refused_for(bad_uris).contains("redirect_uris")); + // No secret in a public document, and no secret-only or JWT-only client. + let secret = json!({ + "client_id": CHATGPT, + "redirect_uris": [CHATGPT_REDIRECT], + "client_secret": "s", + }); + assert!(refused_for(secret).contains("secret")); + let jwt_only = json!({ + "client_id": CHATGPT, + "redirect_uris": [CHATGPT_REDIRECT], + "token_endpoint_auth_method": "private_key_jwt", + }); + assert!(refused_for(jwt_only).contains("public clients")); + // An absent method is `none`: a document may not use a secret-based one. + let no_method = json!({ "client_id": CHATGPT, "redirect_uris": [CHATGPT_REDIRECT] }); + assert!(parse_client_metadata(CHATGPT, &no_method.to_string()).is_ok()); + // A PRESENT method of the wrong type is malformed, not absent. + let odd_method = json!({ + "client_id": CHATGPT, + "redirect_uris": [CHATGPT_REDIRECT], + "token_endpoint_auth_method": 1, + }); + assert!(refused_for(odd_method).contains("must be a string")); + // The flow: absent is the RFC default (accepted above); present, it must + // include the one flow this server runs, and be a string array to say so. + let with = |field: &str, value: serde_json::Value| json!({ "client_id": CHATGPT, "redirect_uris": [CHATGPT_REDIRECT], field: value }); + let ok = |doc: serde_json::Value| parse_client_metadata(CHATGPT, &doc.to_string()).is_ok(); + assert!(ok(with("grant_types", json!(["authorization_code", "refresh_token"])))); + assert!(ok(with("response_types", json!(["code"])))); + let err = refused_for(with("grant_types", json!(["client_credentials"]))); + assert!(err.contains("grant_types") && err.contains("authorization_code"), "{err}"); + let err = refused_for(with("response_types", json!(["token"]))); + assert!(err.contains("response_types") && err.contains("\"code\""), "{err}"); + assert!(refused_for(with("grant_types", json!([]))).contains("authorization_code")); + let err = refused_for(with("grant_types", json!("authorization_code"))); + assert!(err.contains("array of strings"), "{err}"); + assert!(refused_for(with("response_types", json!([1]))).contains("array of strings")); + // Hosted redirects must be same-origin with the document; loopback is exempt. + const OTHER: &str = "https://cimd-other.claude.ai/client.json"; + const OWN: &str = "https://cimd-other.claude.ai/api/mcp/auth_callback"; + let borrowed = json!({ "client_id": OTHER, "redirect_uris": [CHATGPT_REDIRECT] }); + let err = parse_client_metadata(OTHER, &borrowed.to_string()).unwrap_err(); + assert!(err.contains("own origin"), "{err}"); + let mixed = json!({ + "client_id": OTHER, + "redirect_uris": [ + CHATGPT_REDIRECT, OWN, "http://127.0.0.1/cb" + ], + }); + let kept = parse_client_metadata(OTHER, &mixed.to_string()).expect("own + loopback kept"); + assert_eq!(kept.redirect_uris, [OWN, "http://127.0.0.1/cb"]); + // Same host, different port or scheme, is a different origin. + let off_origin = json!({ "client_id": OTHER, "redirect_uris": [format!("{OWN}:8443")] }); + assert!(parse_client_metadata(OTHER, &off_origin.to_string()).is_err()); + // Only what a DCR registration could have registered is kept: a loopback + // redirect with a fragment (which the port-agnostic match would ignore, + // admitting a redirect DCR refuses), or an own-origin path the allow-list + // does not pin, leaves the document with nothing. + let fragment = json!({ "client_id": OTHER, "redirect_uris": ["http://127.0.0.1/cb#x"] }); + assert!(parse_client_metadata(OTHER, &fragment.to_string()).is_err()); + let unpinned = + json!({ "client_id": OTHER, "redirect_uris": ["https://cimd-other.claude.ai/cb"] }); + assert!(parse_client_metadata(OTHER, &unpinned.to_string()).is_err()); + // No more redirect_uris than a DCR registration may send. + let many: Vec = (0..=super::MAX_REDIRECT_URIS) + .map(|i| format!("https://cimd-other.test/cb/{i}")) + .collect(); + let too_many = json!({ "client_id": OTHER, "redirect_uris": many }).to_string(); + let err = parse_client_metadata(OTHER, &too_many).unwrap_err(); + assert!(err.contains("too many redirect_uris"), "{err}"); + // …nor a longer one: a same-origin redirect DCR would refuse is refused here. + let long = format!("https://cimd-other.test/{}", "x".repeat(super::MAX_REDIRECT_URI_LEN)); + let too_long = json!({ "client_id": OTHER, "redirect_uris": [long] }).to_string(); + let err = parse_client_metadata(OTHER, &too_long).unwrap_err(); + assert!(err.contains("too long"), "{err}"); + } + + /// What a fetch failure is ABOUT decides whether it is remembered: the URL + /// (refused, nothing there, a redirect, another 4xx, too large, not UTF-8) is + /// invalid and cached; the moment (unreachable, 5xx, 408, 429) is unavailable + /// and retried. + #[test] + fn cimd_fetch_error_classification() { + use super::{classify_fetch_error, CimdError}; + use imcp2_core::public_fetch::FetchError; + let answered = |status: u16| FetchError::Answered { status, detail: format!("{status}") }; + let unavailable = + |e: FetchError| matches!(classify_fetch_error(e), CimdError::Unavailable(_)); + let invalid = |e: FetchError| matches!(classify_fetch_error(e), CimdError::Invalid(_)); + assert!(unavailable(FetchError::Unreachable("dns".into()))); + for status in [500u16, 502, 503, 504, 408, 421, 425, 429] { + assert!(unavailable(answered(status)), "{status} is about the moment"); + } + for status in [301u16, 302, 400, 401, 403, 404, 410, 451] { + assert!(invalid(answered(status)), "{status} is about the URL"); + } + assert!(invalid(FetchError::Refused("private address".into()))); + assert!(invalid(FetchError::TooLarge("cap".into()))); + assert!(invalid(FetchError::NotUtf8("bytes".into()))); + } + + /// The deploy-time opt-in's reading of its variable: off unless it says on. + #[test] + fn cimd_opt_in_values() { + use super::cimd_enabled_by; + let off = [None, Some(""), Some(" "), Some("0"), Some("false"), Some("no"), Some("off")]; + for value in off.into_iter().chain([Some("enabled"), Some("2")]) { + assert!(!cimd_enabled_by(value), "{value:?} should leave CIMD off"); + } + for value in [Some("1"), Some("true"), Some("Yes"), Some("ON"), Some(" 1 ")] { + assert!(cimd_enabled_by(value), "{value:?} should turn CIMD on"); + } + } + + /// PR #143's trust policy: a document is fetched only from a vetted vendor + /// origin — a host on or under an allow-listed domain, on the default port. + /// Any other URL `client_id` is refused before any fetch and told where to + /// request access; with CIMD off, it is simply an unknown client. + #[tokio::test] + async fn cimd_origin_trust_policy() { + use super::{cimd_client_id, cimd_fixture, cimd_origin_trusted, ClientCheck}; + let trusted = + |id: &str| cimd_origin_trusted(&cimd_client_id(id).expect("a CIMD client_id")); + // The directory clients' real identifiers, and subdomains of vetted domains. + assert!(trusted("https://chatgpt.com/oauth/client.json")); + assert!(trusted("https://claude.ai/oauth/claude-code-client-metadata")); + assert!(trusted("https://www.cursor.com/mcp/client.json")); + assert!(trusted("https://www.perplexity.ai/client.json")); + // Not vetted: a stranger; a look-alike that is no dot-boundary subdomain; a + // vetted name as a SUBDOMAIN of a stranger; a non-default port on a vetted + // host (the SSRF guard would connect to it, and nobody vetted that service). + assert!(!trusted("https://cimd-stranger.test/client.json")); + assert!(!trusted("https://evilclaude.ai/client.json")); + assert!(!trusted("https://claude.ai.evil.test/client.json")); + assert!(!trusted("https://claude.ai:8443/oauth/client.json")); + + // Refused BEFORE any fetch: were one made, these fixtures would turn the + // verdict into MetadataUnavailable, and the hit counts would say so. + let store = test_store(); + const STRANGER: &str = "https://cimd-stranger.test/client.json"; + const PORT: &str = "https://claude.ai:8443/oauth/client.json"; + for id in [STRANGER, PORT] { + cimd_fixture::fail(id, "must not be fetched"); + let verdict = store.validate_client(id, "http://127.0.0.1:1/cb").await; + assert_eq!(verdict, ClientCheck::UntrustedClientOrigin, "{id}"); + assert_eq!(cimd_fixture::hits(id), 0, "{id} must not be fetched"); + } + + // With CIMD off, a URL client_id — vetted or not — is an unknown client: + // nothing is fetched, and no page points at the contact for a mechanism + // this deployment does not offer. + let off = test_store().with_cimd(false); + const VETTED: &str = "https://cimd-off.claude.ai/client.json"; + cimd_fixture::fail(VETTED, "must not be fetched"); + assert_eq!( + off.validate_client(VETTED, "http://127.0.0.1:1/cb").await, + ClientCheck::Refused + ); + assert_eq!( + off.validate_client(STRANGER, "http://127.0.0.1:1/cb").await, + ClientCheck::Refused + ); + assert_eq!(cimd_fixture::hits(VETTED), 0); + } + + #[test] + fn cimd_cache_ttl_is_bounded() { + use super::{cimd_ttl, CIMD_CACHE_DEFAULT_TTL, CIMD_CACHE_MAX_TTL}; + let fresh = Duration::ZERO; + assert_eq!(cimd_ttl(None, fresh), CIMD_CACHE_DEFAULT_TTL); + // The origin's value is honoured as given, however small, up to the ceiling. + assert_eq!(cimd_ttl(Some(Duration::from_secs(300)), fresh), Duration::from_secs(300)); + assert_eq!(cimd_ttl(Some(Duration::from_secs(5)), fresh), Duration::from_secs(5)); + assert_eq!(cimd_ttl(Some(Duration::from_secs(10 * 24 * 3600)), fresh), CIMD_CACHE_MAX_TTL); + // `no-store` / `no-cache` / `max-age=0` is zero: not cached at all. + assert_eq!(cimd_ttl(Some(Duration::ZERO), fresh), Duration::ZERO); + // No freshness information: the default LESS the age the answer already + // has — an origin's own value is already net of it. + let aged = Duration::from_secs(4 * 60); + assert_eq!(cimd_ttl(None, aged), CIMD_CACHE_DEFAULT_TTL - aged); + assert_eq!(cimd_ttl(None, Duration::from_secs(86400)), Duration::ZERO); + assert_eq!( + cimd_ttl(Some(Duration::from_secs(300)), Duration::from_secs(86400)), + Duration::from_secs(300) + ); + } + + /// The media type a document must be served as, by essence: parameters and + /// case are fine, anything else — or nothing — is not. + #[test] + fn cimd_media_type() { + use super::is_json_media_type; + assert!(is_json_media_type(Some("application/json"))); + assert!(is_json_media_type(Some("application/json; charset=utf-8"))); + assert!(is_json_media_type(Some("Application/JSON"))); + assert!(!is_json_media_type(Some("text/html; charset=utf-8"))); + assert!(!is_json_media_type(Some("text/plain"))); + assert!(!is_json_media_type(Some("application/jose+json"))); + assert!(!is_json_media_type(None)); + } + + /// Concurrent requests for one cold document share a single fetch, and one + /// host cannot take every permit: it gets [`CIMD_MAX_INFLIGHT_PER_HOST`] and + /// the excess is told to retry — with every slot given back afterwards. + #[tokio::test] + async fn cimd_fetches_are_coalesced_and_bounded_per_host() { + use super::{cimd_fixture, ClientCheck}; + use serde_json::json; + let store = test_store(); + let check = |id: &'static str, redirect: &'static str| store.validate_client(id, redirect); + let native = |id: &str| { + json!({ "client_id": id, "redirect_uris": ["http://127.0.0.1/cb"] }).to_string() + }; + const REDIRECT: &str = "http://127.0.0.1:4242/cb"; + + // Three misses at once for one document: one fetch, three admissions. + const HERD: &str = "https://cimd-herd.claude.ai/client.json"; + cimd_fixture::serve(HERD, &native(HERD)); + let (a, b, c) = + tokio::join!(check(HERD, REDIRECT), check(HERD, REDIRECT), check(HERD, REDIRECT)); + assert_eq!((a, b, c), (ClientCheck::Allowed, ClientCheck::Allowed, ClientCheck::Allowed)); + assert_eq!(cimd_fixture::hits(HERD), 1, "concurrent misses must share one fetch"); + + // A failure is shared the same way: three misses for a document whose + // origin is down make ONE attempt, and all three are told to retry — none + // queues behind the others for a fetch of its own. + const DOWN: &str = "https://cimd-herd-down.claude.ai/client.json"; + cimd_fixture::fail(DOWN, "origin down"); + let (a, b, c) = + tokio::join!(check(DOWN, REDIRECT), check(DOWN, REDIRECT), check(DOWN, REDIRECT)); + for verdict in [a, b, c] { + assert!(matches!(verdict, ClientCheck::MetadataUnavailable(_)), "{verdict:?}"); + } + assert_eq!(cimd_fixture::hits(DOWN), 1, "concurrent misses must share one failure"); + + // Three distinct documents on ONE host at once: two fetch, the third is + // told to retry rather than taking a third permit for that host. + const ONE: &str = "https://cimd-busy.claude.ai/one.json"; + const TWO: &str = "https://cimd-busy.claude.ai/two.json"; + const THREE: &str = "https://cimd-busy.claude.ai/three.json"; + for id in [ONE, TWO, THREE] { + cimd_fixture::serve(id, &native(id)); + } + let (one, two, three) = + tokio::join!(check(ONE, REDIRECT), check(TWO, REDIRECT), check(THREE, REDIRECT)); + assert_eq!((one, two), (ClientCheck::Allowed, ClientCheck::Allowed)); + match three { + ClientCheck::MetadataUnavailable(why) => { + assert!(why.contains("cimd-busy.claude.ai"), "{why}") + } + other => panic!("the third fetch for one host must be refused, got {other:?}"), + } + // The refusal spent no rate budget: a token goes only with a fetch that + // goes out — herd, herd-down, one and two so far, four in all. + let tokens_left = store.cimd.rates.lock().unwrap().all.tokens.floor() as u32; + assert_eq!(tokens_left, super::CIMD_RATE_PER_MINUTE - 4, "a refused slot drains no budget"); + // Slots and single-flight entries are released, not leaked. + assert!(store.cimd.hosts.lock().unwrap().is_empty()); + assert!(store.cimd.fetching.lock().unwrap().is_empty()); + // The refused one succeeds on retry (its document was never fetched). + assert_eq!(check(THREE, REDIRECT).await, ClientCheck::Allowed); + } + + /// A token bucket starts full, gives one token per take, and refills only + /// with time. + #[test] + fn cimd_rate_bucket() { + use super::TokenBucket; + let mut bucket = TokenBucket::per_minute(2); + assert!(bucket.has_token()); + bucket.take(); + assert!(bucket.has_token()); + bucket.take(); + assert!(!bucket.has_token(), "two tokens a minute means two, not three, at once"); + } + + /// The fetch RATE is bounded, not only how many are in flight: an origin that + /// answers at once gives its permit straight back, so without this a caller + /// could drive one request after another at a vendor with distinct paths. + /// A vendor's share runs out first, leaving the others theirs; then the + /// process's budget does, for everyone. + #[tokio::test] + async fn cimd_fetch_rate_is_bounded() { + use super::{ + cimd_fixture, ClientCheck, CIMD_RATE_PER_DOMAIN_PER_MINUTE, CIMD_RATE_PER_MINUTE, + }; + let store = test_store(); + const REDIRECT: &str = "http://127.0.0.1:1/cb"; + fn rate_limited(verdict: &ClientCheck) -> bool { + matches!(verdict, ClientCheck::MetadataUnavailable(why) if why.contains("rate")) + } + // Distinct, never-seen paths on one vendor, each answered 404 at once (so + // nothing legitimate is cached): each is a fetch, until that vendor's + // share for the minute is spent. + for i in 0..CIMD_RATE_PER_DOMAIN_PER_MINUTE { + let id = format!("https://cimd-rate.claude.ai/{i}.json"); + cimd_fixture::not_found(&id); + assert_eq!(store.validate_client(&id, REDIRECT).await, ClientCheck::Refused, "{id}"); + } + const OVER: &str = "https://cimd-rate.claude.ai/over.json"; + cimd_fixture::not_found(OVER); + let verdict = store.validate_client(OVER, REDIRECT).await; + assert!(rate_limited(&verdict), "{verdict:?}"); + assert_eq!(cimd_fixture::hits(OVER), 0, "refused before any fetch"); + // Another vendor still has its share… + assert_eq!( + CIMD_RATE_PER_MINUTE, + 2 * CIMD_RATE_PER_DOMAIN_PER_MINUTE, + "this test spends exactly the process's budget over two vendors" + ); + for i in 0..CIMD_RATE_PER_DOMAIN_PER_MINUTE { + let id = format!("https://cimd-rate.chatgpt.com/{i}.json"); + cimd_fixture::not_found(&id); + assert_eq!(store.validate_client(&id, REDIRECT).await, ClientCheck::Refused, "{id}"); + } + // …until the process's budget is spent, which refuses a third vendor too. + const THIRD: &str = "https://cimd-rate.cursor.com/one.json"; + cimd_fixture::not_found(THIRD); + let verdict = store.validate_client(THIRD, REDIRECT).await; + assert!(rate_limited(&verdict), "{verdict:?}"); + assert_eq!(cimd_fixture::hits(THIRD), 0); + // A refusal holds nothing: no slot, no permit, no flight. + assert!(store.cimd.hosts.lock().unwrap().is_empty()); + assert!(store.cimd.fetching.lock().unwrap().is_empty()); + } + + /// A fetcher dropped mid-fetch while a waiter is in the flight hands over: + /// the flight stays where the waiter and any newcomer find it, the waiter + /// takes over the one fetch, and nobody fetches the document twice at once. + #[tokio::test] + async fn cimd_cancelled_fetcher_hands_over_to_a_waiter() { + use super::{cimd_fixture, ClientCheck}; + use serde_json::json; + let store = test_store(); + const HANDOVER: &str = "https://cimd-handover.claude.ai/client.json"; + let request = |store: super::AuthStore| async move { + store.validate_client(HANDOVER, "http://127.0.0.1:1/cb").await + }; + // The fetcher's fetch hangs (an origin that has not answered)… + cimd_fixture::hang(HANDOVER); + let fetcher = tokio::spawn(request(store.clone())); + tokio::task::yield_now().await; + // …a waiter joins the flight and waits for that fetch… + let waiter = tokio::spawn(request(store.clone())); + tokio::task::yield_now().await; + assert_eq!(cimd_fixture::hits(HANDOVER), 1, "the waiter must not fetch for itself"); + // …and the fetcher is dropped. The flight survives for its waiter, which + // takes over the fetch (the origin answers now); a newcomer arriving then + // finds the one flight, or the document it cached, and fetches nothing. + let doc = json!({ "client_id": HANDOVER, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve(HANDOVER, &doc.to_string()); + fetcher.abort(); + assert!(fetcher.await.unwrap_err().is_cancelled()); + assert!(store.cimd.fetching.lock().unwrap().contains_key(HANDOVER), "flight must survive"); + let newcomer = tokio::spawn(request(store.clone())); + assert_eq!(waiter.await.unwrap(), ClientCheck::Allowed); + assert_eq!(newcomer.await.unwrap(), ClientCheck::Allowed); + assert_eq!( + cimd_fixture::hits(HANDOVER), + 2, + "one attempt each by fetcher and waiter, none more" + ); + assert!( + store.cimd.fetching.lock().unwrap().is_empty(), + "the last one out retires the flight" + ); + assert!(store.cimd.hosts.lock().unwrap().is_empty()); + } + + /// A `client_id` spelt otherwise than canonically is the same vetted host to + /// the trust policy, and fetched — from the URL as given, which its document + /// must repeat. + #[tokio::test] + async fn cimd_client_id_is_taken_as_given() { + use super::{cimd_fixture, ClientCheck}; + use serde_json::json; + let store = test_store(); + const SPELT: &str = "https://Cimd-Spelling.claude.ai:443/client.json"; + let doc = json!({ "client_id": SPELT, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve(SPELT, &doc.to_string()); + assert_eq!( + store.validate_client(SPELT, "http://127.0.0.1:1/cb").await, + ClientCheck::Allowed + ); + assert_eq!(cimd_fixture::hits(SPELT), 1, "fetched from the URL as given"); + // The same document under a differently spelt id is a different client, + // and its document, saying otherwise, does not vouch for it. + const OTHERWISE: &str = "https://cimd-spelling.claude.ai/client.json"; + cimd_fixture::serve(OTHERWISE, &doc.to_string()); + assert_eq!( + store.validate_client(OTHERWISE, "http://127.0.0.1:1/cb").await, + ClientCheck::Refused + ); + } + + /// The two rules a flight lives by. Publishing retires it at once, however + /// many requests still hold it — they read the outcome from their own handle, + /// and a newcomer must not join a published flight to reuse an outcome the + /// origin said not to reuse (`no-store`) or a failure that may be over. An + /// UNPUBLISHED flight, its fetcher cancelled, stays for its holders and is + /// retired by the last of them, so a waiter can take over the fetch and no + /// entry is left behind. Neither ever touches a newer flight for the key. + #[test] + fn cimd_flight_retirement_rules() { + use super::{CimdState, Flight, FlightGuard}; + let state = CimdState::new(); + let key = "https://cimd-rules.claude.ai/client.json"; + let join = || -> Flight { + std::sync::Arc::clone(state.fetching.lock().unwrap().entry(key.to_owned()).or_default()) + }; + let held = || state.fetching.lock().unwrap().contains_key(key); + + // Published while a waiter still holds it: retired at once. + let (fetcher, waiter) = (join(), join()); + let (fetcher_guard, waiter_guard) = ( + FlightGuard { state: &state, key, flight: &fetcher }, + FlightGuard { state: &state, key, flight: &waiter }, + ); + state.retire_flight(key, &fetcher); + assert!(!held(), "a published flight is retired however many hold it"); + // A newer flight for the key is not touched by the old one's holders leaving. + let newer = join(); + drop(fetcher_guard); + drop(waiter_guard); + assert!(held(), "an old flight's holders must not retire a newer flight"); + state.retire_flight(key, &fetcher); + assert!(held(), "nor does retiring the old flight"); + state.retire_flight(key, &newer); + assert!(!held()); + + // Unpublished — the fetcher is cancelled — with a waiter: stays for the + // waiter; the last holder out retires it. + let (fetcher, waiter) = (join(), join()); + let waiter_guard = FlightGuard { state: &state, key, flight: &waiter }; + drop(FlightGuard { state: &state, key, flight: &fetcher }); + drop(fetcher); + assert!(held(), "a cancelled fetcher leaves the flight for its waiter"); + drop(waiter_guard); + assert!(!held(), "the last holder out retires an unpublished flight"); + } + + /// A request dropped mid-fetch — the client reset the stream, so the + /// authorize future was dropped — leaves nothing behind: not its single-flight + /// entry, not its host slot, not its permit. The next request for the same + /// document starts afresh and succeeds. + #[tokio::test] + async fn cimd_cancelled_fetch_leaves_nothing_behind() { + use super::{cimd_fixture, ClientCheck, CIMD_MAX_INFLIGHT}; + use serde_json::json; + let store = test_store(); + const GONE: &str = "https://cimd-gone.claude.ai/client.json"; + let doc = json!({ "client_id": GONE, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve(GONE, &doc.to_string()); + // The leader runs until its fetch suspends (the fixture yields there, as a + // real fetch would), then is aborted — as a dropped connection drops the + // authorize future. + let leader = tokio::spawn({ + let store = store.clone(); + async move { store.validate_client(GONE, "http://127.0.0.1:1/cb").await } + }); + tokio::task::yield_now().await; + assert!(store.cimd.fetching.lock().unwrap().contains_key(GONE), "leader must be mid-fetch"); + assert_eq!(cimd_fixture::hits(GONE), 1); + leader.abort(); + assert!(leader.await.unwrap_err().is_cancelled()); + assert!( + store.cimd.fetching.lock().unwrap().is_empty(), + "a dropped fetch retires its flight" + ); + assert!(store.cimd.hosts.lock().unwrap().is_empty(), "…and gives its host slot back"); + assert_eq!(store.cimd.inflight.available_permits(), CIMD_MAX_INFLIGHT, "…and its permit"); + // Nothing was cached (the fetch never completed), so the next request + // fetches afresh — and succeeds. + assert_eq!( + store.validate_client(GONE, "http://127.0.0.1:1/cb").await, + ClientCheck::Allowed + ); + assert_eq!(cimd_fixture::hits(GONE), 2); + } + + /// `/oauth/authorize` with a CIMD client: the document's redirects get the + /// checks a DCR registration gets — hosted-redirect allow-list included, and + /// checked BEFORE any fetch — the document is cached, an invalid document is + /// an unknown client (remembered as one, briefly), and an unfetchable one is + /// a retry, not an unknown client (and not remembered). + #[tokio::test] + async fn cimd_client_authorization() { + use super::{cimd_fixture, ClientCheck}; + use serde_json::json; + let store = test_store(); + const CHATGPT_REDIRECT: &str = "https://chatgpt.com/connector_platform_oauth_redirect"; + let check = |id: &'static str, redirect: &'static str| store.validate_client(id, redirect); + + // ChatGPT, exactly as it identifies itself: its real client_id and document, + // served by the fixture. Its redirect is same-origin AND allow-listed. + const HOSTED: &str = "https://chatgpt.com/oauth/client.json"; + let hosted_doc = json!({ + "client_id": HOSTED, + "client_name": "ChatGPT", + "redirect_uris": [CHATGPT_REDIRECT], + "token_endpoint_auth_method": "private_key_jwt", + "token_endpoint_auth_methods_supported": ["none", "private_key_jwt"], + }); + cimd_fixture::serve(HOSTED, &hosted_doc.to_string()); + assert_eq!(check(HOSTED, CHATGPT_REDIRECT).await, ClientCheck::Allowed); + // A redirect the document does NOT list is refused, allow-listed or not. + let other_vendor = "https://claude.ai/api/mcp/auth_callback"; + assert_eq!(check(HOSTED, other_vendor).await, ClientCheck::Refused); + // Cached: with its URL now failing, the client still passes. + cimd_fixture::fail(HOSTED, "origin down"); + assert_eq!(check(HOSTED, CHATGPT_REDIRECT).await, ClientCheck::Allowed); + + // Another origin's document borrowing ChatGPT's (allow-listed) redirect is + // refused: a self-asserted document may not point the code at another party. + const CROSS: &str = "https://cimd-cross.claude.ai/client.json"; + let cross_doc = json!({ "client_id": CROSS, "redirect_uris": [CHATGPT_REDIRECT] }); + cimd_fixture::serve(CROSS, &cross_doc.to_string()); + assert_eq!(check(CROSS, CHATGPT_REDIRECT).await, ClientCheck::Refused); + + // A document listing a redirect that is NOT allow-listed gains nothing: the + // allow-list is checked before the fetch, so the document is never asked + // for (were it, this fixture would turn the verdict into Unavailable). + const ROGUE: &str = "https://cimd-rogue.claude.ai/client.json"; + cimd_fixture::fail(ROGUE, "must not be fetched"); + let rogue_redirect = "https://cimd-rogue.claude.ai/callback"; + assert_eq!(check(ROGUE, rogue_redirect).await, ClientCheck::Refused); + assert_eq!(cimd_fixture::hits(ROGUE), 0); + + // Claude-Code-shaped: loopback redirects match port-agnostically. + const LOOPBACK: &str = "https://cimd-loopback.claude.ai/client-metadata"; + let loopback_doc = json!({ + "client_id": LOOPBACK, + "client_name": "Native", + "redirect_uris": ["http://localhost/callback", "http://127.0.0.1/callback"], + "token_endpoint_auth_method": "none", + }); + cimd_fixture::serve(LOOPBACK, &loopback_doc.to_string()); + assert_eq!(check(LOOPBACK, "http://localhost:3118/callback").await, ClientCheck::Allowed); + assert_eq!(check(LOOPBACK, "http://127.0.0.1:51234/callback").await, ClientCheck::Allowed); + assert_eq!(check(LOOPBACK, "http://127.0.0.1:51234/other").await, ClientCheck::Refused); + + // Unfetchable: a retry, not an unknown client — and not remembered, so + // once the origin is back the next request fetches the document. + const DOWN: &str = "https://cimd-down.claude.ai/client.json"; + cimd_fixture::fail(DOWN, "connection refused"); + let verdict = check(DOWN, "http://127.0.0.1:9/cb").await; + assert!(matches!(verdict, ClientCheck::MetadataUnavailable(_)), "{verdict:?}"); + let down_doc = json!({ "client_id": DOWN, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve(DOWN, &down_doc.to_string()); + assert_eq!(check(DOWN, "http://127.0.0.1:9/cb").await, ClientCheck::Allowed); + assert_eq!(cimd_fixture::hits(DOWN), 2, "a transient failure is retried"); + + // A document about ANOTHER URL is an unknown client (misconfigured or + // hostile) — remembered as one, so the repeat costs no fetch. + const LIAR: &str = "https://cimd-liar.claude.ai/client.json"; + let liar_doc = json!({ "client_id": HOSTED, "redirect_uris": [CHATGPT_REDIRECT] }); + cimd_fixture::serve(LIAR, &liar_doc.to_string()); + assert_eq!(check(LIAR, CHATGPT_REDIRECT).await, ClientCheck::Refused); + assert_eq!(check(LIAR, CHATGPT_REDIRECT).await, ClientCheck::Refused); + assert_eq!(cimd_fixture::hits(LIAR), 1, "an invalid document is remembered"); + + // Nothing at the URL (404) is likewise about the URL: remembered, so even a + // document appearing there is not seen until the negative entry lapses. + const MISSING: &str = "https://cimd-missing.claude.ai/client.json"; + cimd_fixture::not_found(MISSING); + assert_eq!(check(MISSING, "http://127.0.0.1:7/cb").await, ClientCheck::Refused); + let late_doc = json!({ "client_id": MISSING, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve(MISSING, &late_doc.to_string()); + assert_eq!(check(MISSING, "http://127.0.0.1:7/cb").await, ClientCheck::Refused); + assert_eq!(cimd_fixture::hits(MISSING), 1, "a missing document is remembered"); + + // A document with no cache hint that some cache along the way held for a + // day is not given ten fresh minutes here: the default lifetime is spent, + // so the next request fetches again. + const AGED: &str = "https://cimd-aged.claude.ai/client.json"; + let aged_doc = json!({ "client_id": AGED, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve_aged(AGED, &aged_doc.to_string(), Duration::from_secs(86400)); + assert_eq!(check(AGED, "http://127.0.0.1:3/cb").await, ClientCheck::Allowed); + assert_eq!(check(AGED, "http://127.0.0.1:3/cb").await, ClientCheck::Allowed); + assert_eq!(cimd_fixture::hits(AGED), 2, "a spent default lifetime is not cached"); + + // Whereas a PER-REQUEST failure never poisons a real client: HOSTED was + // probed above with a redirect its document does not list, and its + // legitimate redirect still passes from the (positive) cache. + assert_eq!(check(HOSTED, other_vendor).await, ClientCheck::Refused); + assert_eq!(check(HOSTED, CHATGPT_REDIRECT).await, ClientCheck::Allowed); + + // A document served `no-store` is honoured, then NOT reused: once its URL + // fails, so does the next authorize (contrast HOSTED above, which was cached). + const VOLATILE: &str = "https://claude.ai/oauth/claude-code-client-metadata"; + let volatile_doc = json!({ + "client_id": VOLATILE, + "client_name": "Claude Code", + "redirect_uris": ["http://localhost/callback", "http://127.0.0.1/callback"], + "token_endpoint_auth_method": "none", + }); + cimd_fixture::serve_uncacheable(VOLATILE, &volatile_doc.to_string()); + assert_eq!(check(VOLATILE, "http://localhost:3118/callback").await, ClientCheck::Allowed); + cimd_fixture::fail(VOLATILE, "origin down"); + let verdict = check(VOLATILE, "http://localhost:3118/callback").await; + assert!(matches!(verdict, ClientCheck::MetadataUnavailable(_)), "{verdict:?}"); + + // A document served as anything but application/json is not a document. + const HTML: &str = "https://cimd-html.claude.ai/client.json"; + let html_doc = json!({ "client_id": HTML, "redirect_uris": ["http://127.0.0.1/cb"] }); + cimd_fixture::serve_as(HTML, &html_doc.to_string(), "text/html; charset=utf-8"); + assert_eq!(check(HTML, "http://127.0.0.1:7/cb").await, ClientCheck::Refused); + + // DCR clients are untouched: an unregistered ordinary id is refused. + assert_eq!(check("client-unknown", "http://127.0.0.1:1/cb").await, ClientCheck::Refused); + } + + /// The AS advertises CIMD — which is what makes Claude and ChatGPT select it + /// over DCR, and would be an outage without the implementation behind it — + /// alongside the `none` that both require to go with it, and only where the + /// deployment opted in. + #[tokio::test] + async fn as_metadata_advertises_cimd_only_where_enabled() { + let metadata = |store: super::AuthStore| async { + let resp = super::authorization_server_metadata(axum::extract::State(store)).await; + let body = axum::body::to_bytes(resp.into_body(), 64 * 1024).await.expect("body"); + serde_json::from_slice::(&body).expect("JSON") + }; + let on = metadata(test_store()).await; + assert_eq!(on["client_id_metadata_document_supported"], serde_json::json!(true)); + assert_eq!(on["token_endpoint_auth_methods_supported"], serde_json::json!(["none"])); + let off = metadata(test_store().with_cimd(false)).await; + assert_eq!(off["client_id_metadata_document_supported"], serde_json::json!(false)); + assert_eq!(off["token_endpoint_auth_methods_supported"], serde_json::json!(["none"])); + } + + /// `/oauth/authorize` with a URL `client_id` off the vendor trust policy: the + /// same "not approved" page (or JSON) a hosted redirect off the allow-list + /// gets — 403, naming the contact — and no fetch. + #[tokio::test] + async fn authorize_points_an_unvetted_cimd_origin_at_the_contact() { + use axum::extract::{Query, State}; + let store = test_store(); + const STRANGER: &str = "https://cimd-authorize-stranger.test/client.json"; + super::cimd_fixture::fail(STRANGER, "must not be fetched"); + let query = || super::AuthorizeQuery { + response_type: Some("code".into()), + client_id: STRANGER.into(), + redirect_uri: "http://127.0.0.1:1/cb".into(), + state: Some("xyz".into()), + code_challenge: Some("E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM".into()), + code_challenge_method: Some("S256".into()), + scope: None, + resource: None, + }; + let body_of = |resp: axum::response::Response| async { + let content_type = + resp.headers()[axum::http::header::CONTENT_TYPE].to_str().unwrap().to_owned(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + (content_type, String::from_utf8(bytes.to_vec()).unwrap()) + }; + let resp = super::authorize(State(store.clone()), html_headers(), Query(query())).await; + assert_eq!(resp.status(), axum::http::StatusCode::FORBIDDEN); + let (content_type, html) = body_of(resp).await; + assert!(content_type.starts_with("text/html"), "{content_type}"); + assert!(html.contains(super::CONTACT), "the page must name the contact"); + assert!(!html.contains(STRANGER), "the page reflects nothing"); + let resp = super::authorize(State(store.clone()), json_headers(), Query(query())).await; + assert_eq!(resp.status(), axum::http::StatusCode::FORBIDDEN); + let (content_type, json) = body_of(resp).await; + assert!(content_type.contains("json"), "{content_type}"); + assert!(json.contains("invalid_client") && json.contains(super::CONTACT), "{json}"); + assert_eq!(super::cimd_fixture::hits(STRANGER), 0, "nothing is fetched off-policy"); + } + + /// The transient verdict at the endpoint. A vetted CIMD client whose document + /// could not be fetched RIGHT NOW — the origin unreachable, or answering with + /// a status the client may retry (425 Too Early here) — is told to retry: + /// `503 temporarily_unavailable` to a programmatic caller, the sign-in error + /// page to a browser. Not `invalid_client`, which would have the user re-add + /// the connector. Neither body reflects the `client_id` URL or the cause, and + /// nothing is remembered: the next request fetches again. + #[tokio::test] + async fn authorize_tells_a_cimd_client_to_retry_when_its_document_is_unavailable() { + use axum::extract::{Query, State}; + let store = test_store(); + const DOWN: &str = "https://cimd-authorize-down.claude.ai/client.json"; + const EARLY: &str = "https://cimd-authorize-early.claude.ai/client.json"; + super::cimd_fixture::fail(DOWN, "connection reset by peer"); + super::cimd_fixture::answer(EARLY, 425); + let query = |client_id: &str| super::AuthorizeQuery { + response_type: Some("code".into()), + client_id: client_id.into(), + redirect_uri: "http://127.0.0.1:1/cb".into(), + state: Some("xyz".into()), + code_challenge: Some("E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM".into()), + code_challenge_method: Some("S256".into()), + scope: None, + resource: None, + }; + let body_of = |resp: axum::response::Response| async { + let content_type = + resp.headers()[axum::http::header::CONTENT_TYPE].to_str().unwrap().to_owned(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + (content_type, String::from_utf8(bytes.to_vec()).unwrap()) + }; + for (id, cause) in [(DOWN, "reset"), (EARLY, "425")] { + let resp = + super::authorize(State(store.clone()), json_headers(), Query(query(id))).await; + assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE, "{id}"); + let (content_type, json) = body_of(resp).await; + assert!(content_type.contains("json"), "{content_type}"); + assert!(json.contains("temporarily_unavailable"), "{json}"); + assert!(!json.contains("invalid_client"), "a retry, not a client to re-add: {json}"); + assert!( + !json.contains(id) && !json.contains(cause), + "the body reflects nothing: {json}" + ); + let resp = + super::authorize(State(store.clone()), html_headers(), Query(query(id))).await; + assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE, "{id}"); + let (content_type, html) = body_of(resp).await; + assert!(content_type.starts_with("text/html"), "{content_type}"); + assert!(html.contains("Try again in a moment"), "{html}"); + assert!(!html.contains(id) && !html.contains(cause), "the page reflects nothing"); + assert_eq!(super::cimd_fixture::hits(id), 2, "{id}: not remembered, so fetched again"); + } + } + /// `OAUTH_ALLOWED_REDIRECT_PREFIXES` entries parse to `(host, path)` only for a /// bare `https://host/path`; a port, query, fragment, or userinfo is refused /// (dropped) rather than silently discarded, and so is a non-https or root-path @@ -2585,6 +4445,14 @@ mod tests { } fn test_store_cfg(require_resource: bool) -> super::AuthStore { + // As deployed with `OAUTH_CIMD_ENABLED=1`, and with CIMD state of its own so + // tests do not see each other's flights; the off case sets this itself. + new_store(require_resource).with_cimd(true) + } + + /// A store exactly as [`super::AuthStore::new`] builds it: CIMD per the + /// environment (off under `cargo test`), CIMD state shared process-wide. + fn new_store(require_resource: bool) -> super::AuthStore { use candid::Principal; use imcp2_core::identities::{Identities, IiInstance}; let agent = @@ -2610,6 +4478,19 @@ mod tests { ) } + /// Every store in the process shares one CIMD state: the bundled binary + /// mounts a store per II instance, and the in-flight bounds must hold across + /// them — two mounts must not mean twice the fetches — as must the cache. + #[test] + fn cimd_state_is_shared_by_every_store() { + use std::sync::Arc; + let (a, b) = (new_store(false), new_store(false)); + assert!(Arc::ptr_eq(&a.cimd, &b.cimd), "stores must share the process's CIMD state"); + // The test stores keep their own, so tests do not see each other's flights. + let (c, d) = (test_store(), test_store()); + assert!(!Arc::ptr_eq(&c.cimd, &d.cimd) && !Arc::ptr_eq(&c.cimd, &a.cimd)); + } + /// A request header map that accepts HTML — i.e. a browser hitting the /// front-channel `/oauth/authorize`. fn html_headers() -> axum::http::HeaderMap { @@ -2736,14 +4617,13 @@ mod tests { let stamp = || async { store.clients.registrations.read().await["client-x"].last_used }; backdate().await; - assert!(store.validate_client("client-x", redirect).await); + assert_eq!(store.validate_client("client-x", redirect).await, super::ClientCheck::Allowed); assert!(stamp().await > 0, "an accepted redirect refreshes the LRU stamp"); backdate().await; - assert!( - !store - .validate_client("client-x", "https://claude.ai/api/mcp/auth_callback/nope") - .await + assert_eq!( + store.validate_client("client-x", "https://claude.ai/api/mcp/auth_callback/nope").await, + super::ClientCheck::Refused ); assert_eq!(stamp().await, 0, "a rejected redirect must not refresh the stamp"); }