From 58f1d9e911f489f83c47fb54f5290c6228f972a4 Mon Sep 17 00:00:00 2001 From: Duncan Date: Wed, 29 Jul 2026 18:45:32 -0400 Subject: [PATCH 1/3] fix(relay): advertise the NIP-11 max_limit the REQ path actually enforces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NIP-11 advertised `max_limit: 10_000`, but the effective websocket page ceiling was 1,000: the REQ path never sets `EventQuery::max_limit`, so `query_events` applied its `unwrap_or(1000)` clamp to every historical query. A client trusting the advertisement asks for 10,000, silently receives 1,000, and reads that short page as exhaustion — dropping up to 9,000 events with no error. `MAX_HISTORICAL_LIMIT = 2_000` in the REQ path was dead for the same reason: nothing could survive the DB clamp. Both call sites now use the DB clamp default directly, so one constant is the single source of truth for the advertised ceiling and the enforced one. No behavior change — 1,000 was already the real limit on every path. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-db/src/event.rs | 15 +++++--- crates/buzz-db/src/lib.rs | 2 +- crates/buzz-relay/src/handlers/req.rs | 51 ++++++++++++++++++++++++--- crates/buzz-relay/src/nip11.rs | 7 +++- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index 520fd1536f..855aca43db 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -17,6 +17,13 @@ use buzz_core::{CommunityId, StoredEvent}; use crate::error::{DbError, Result}; +/// Largest page [`query_events`] will return when [`EventQuery::max_limit`] is +/// unset — the effective ceiling on any client-requested `limit`. +/// +/// This is the value the relay advertises as NIP-11 `limitation.max_limit`, so +/// the advertised ceiling and the enforced one cannot drift. +pub const DEFAULT_MAX_PAGE_LIMIT: i64 = 1_000; + /// Optional filters for [`query_events`]. #[derive(Debug, Clone)] pub struct EventQuery { @@ -67,9 +74,9 @@ pub struct EventQuery { /// channel-less global events. Applied before SQL `LIMIT` so access-filtered /// historical pages have exact exhaustion semantics. pub channel_ids: Option>, - /// Override the default limit clamp (1000). Used by COUNT fallback path - /// which needs to fetch all matching events for post-filter counting. - /// When None, the default clamp of 1000 applies. + /// Override the default page clamp ([`DEFAULT_MAX_PAGE_LIMIT`]). Used by + /// the COUNT fallback path, which needs to fetch all matching events for + /// post-filter counting. When None, the default clamp applies. pub max_limit: Option, /// Persona visibility reader: when set, append an SQL visibility clause /// for kind 30175 before ORDER/LIMIT so private personas are excluded from @@ -344,7 +351,7 @@ pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result RelayLimitation { max_message_length: Some(max_message_length as u64), max_subscriptions: Some(1024), max_filters: Some(10), - max_limit: Some(10_000), + max_limit: Some(buzz_db::DEFAULT_MAX_PAGE_LIMIT as u32), max_subid_length: Some(256), min_pow_difficulty: None, auth_required: true, From afed220ca6f1817b12287556c09aec3f8f012c57 Mon Sep 17 00:00:00 2001 From: Duncan Date: Thu, 30 Jul 2026 11:03:25 -0400 Subject: [PATCH 2/3] fix(relay): derive the NIP-50 scan budget from the page ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search path clamped its emission target to the advertised ceiling, but how far it could scan to reach that target was bounded separately by a bare 10-page loop over 100-hit pages. The product happened to equal the ceiling, so raising the ceiling — or shrinking a page — would leave search emitting short pages while still advertising the larger number: the same silent under-delivery the ceiling exists to prevent. The page count is now ceiling-divided from the shared limit over a named page size, so scan capacity tracks the advertisement by construction, and a guard test asserts the relation holds with no spare page of slack. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-relay/src/api/bridge.rs | 21 +++++++++ crates/buzz-relay/src/handlers/req.rs | 68 +++++++++++++++++++++------ 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 2e00d6bd2f..bc0678a3bd 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -3022,6 +3022,27 @@ mod tests { assert_eq!(extract_page_offset(&raw, None), None); } + /// Offsets are sized from the *clamped* limit the DB will honor, not from + /// what the client asked for. `filter_to_query_params` clamps an absent or + /// over-ceiling `limit` to `DEFAULT_MAX_PAGE_LIMIT` (guarded in + /// `handlers::req::tests::req_filter_limit_clamps_to_advertised_nip11_max_limit`) + /// and that clamped value is what arrives here — so page N starts exactly + /// N-1 full pages in. Sizing from an unclamped limit would step past rows + /// the previous page never returned. + #[test] + fn extract_page_offset_sizes_pages_from_clamped_limit() { + let clamped = buzz_db::DEFAULT_MAX_PAGE_LIMIT; + + assert_eq!( + extract_page_offset(&serde_json::json!({ "page": 2 }), Some(clamped)), + Some(clamped) + ); + assert_eq!( + extract_page_offset(&serde_json::json!({ "page": 3 }), Some(clamped)), + Some(clamped * 2) + ); + } + #[test] fn extract_depth_limit_valid() { let raw = serde_json::json!({ "depth_limit": 3 }); diff --git a/crates/buzz-relay/src/handlers/req.rs b/crates/buzz-relay/src/handlers/req.rs index c08f101e9f..6b3704ab8c 100644 --- a/crates/buzz-relay/src/handlers/req.rs +++ b/crates/buzz-relay/src/handlers/req.rs @@ -415,10 +415,18 @@ pub async fn handle_req( ); } -/// Handle a NIP-50 search REQ: query Postgres FTS, fetch full events, deliver results, EOSE. -/// Search subscriptions are one-shot — no persistent subscription is registered. +/// FTS candidate hits fetched per page. Pages are always full regardless of the +/// requested limit — post-filtering may discard many hits, so the scan needs +/// headroom to fill the limit. +const SEARCH_PAGE_SIZE: u32 = 100; + /// Maximum FTS pages to fetch per filter (prevents unbounded loops). -const MAX_SEARCH_PAGES: u32 = 10; +/// +/// Derived from the advertised page ceiling rather than fixed, so the search +/// path can always reach the limit NIP-11 promises. A bare page count would let +/// a future ceiling change leave search silently emitting fewer events than the +/// relay advertises — the same class of lie this ceiling exists to prevent. +const MAX_SEARCH_PAGES: u32 = (buzz_db::DEFAULT_MAX_PAGE_LIMIT as u32).div_ceil(SEARCH_PAGE_SIZE); /// Resolve request-local channel access, repairing a stale cache-negative. /// @@ -500,6 +508,8 @@ pub(crate) fn build_search_channel_scope_filter( }) } +/// Handle a NIP-50 search REQ: query Postgres FTS, fetch full events, deliver results, EOSE. +/// Search subscriptions are one-shot — no persistent subscription is registered. #[allow(clippy::too_many_arguments)] async fn handle_search_req( sub_id: &str, @@ -586,9 +596,6 @@ async fn handle_search_req( // or exhausted the search result set. This ensures post-filtering // doesn't silently reduce the result count below the requested limit. let mut emitted: u32 = 0; - // Always fetch full pages (100) regardless of limit — post-filtering - // may discard many hits, so we need headroom to fill the requested limit. - let per_page: u32 = 100; for page in 1..=MAX_SEARCH_PAGES { if emitted >= limit { @@ -604,7 +611,7 @@ async fn handle_search_req( since, until, page, - per_page, + per_page: SEARCH_PAGE_SIZE, mode: buzz_search::SearchMode::FullText, }; @@ -616,9 +623,9 @@ async fn handle_search_req( } }; - // A short page is the last page: FTS returns up to `per_page` hits, - // so fewer than that means the result set is exhausted. - let exhausted = search_result.hits.len() < per_page as usize; + // A short page is the last page: FTS returns up to a full page of + // hits, so fewer than that means the result set is exhausted. + let exhausted = search_result.hits.len() < SEARCH_PAGE_SIZE as usize; let page_empty = search_result.hits.is_empty(); let hit_ids: Vec<[u8; 32]> = @@ -1415,9 +1422,9 @@ mod tests { ) } - #[test] - fn req_filter_limit_clamps_to_advertised_nip11_max_limit() { - let advertised = crate::nip11::RelayInfo::build( + /// NIP-11 `limitation.max_limit` as this relay actually advertises it. + fn advertised_max_limit() -> i64 { + crate::nip11::RelayInfo::build( None, None, false, @@ -1427,7 +1434,12 @@ mod tests { .limitation .expect("limitation") .max_limit - .expect("max_limit") as i64; + .expect("max_limit") as i64 + } + + #[test] + fn req_filter_limit_clamps_to_advertised_nip11_max_limit() { + let advertised = advertised_max_limit(); let community = buzz_core::tenant::CommunityId::from_uuid(uuid::Uuid::new_v4()); @@ -1457,6 +1469,34 @@ mod tests { assert_eq!(modest.limit, Some(10)); } + /// The NIP-50 search path clamps its emission target to the advertised + /// ceiling like every other REQ, but its ability to *reach* that target is + /// bounded a second time by how many FTS pages it will scan. If that scan + /// budget falls below the ceiling, search under-delivers against NIP-11 + /// while the clamp above still looks correct — so the budget is asserted + /// against the advertised value, not just against the DB constant it is + /// derived from. + #[test] + fn search_scan_capacity_covers_advertised_nip11_max_limit() { + let advertised = advertised_max_limit(); + let capacity = i64::from(MAX_SEARCH_PAGES) * i64::from(SEARCH_PAGE_SIZE); + + assert!( + capacity >= advertised, + "NIP-50 scans at most {capacity} hits ({MAX_SEARCH_PAGES} pages of \ + {SEARCH_PAGE_SIZE}) but NIP-11 advertises {advertised} — search \ + would silently emit a short page" + ); + + // The budget is derived, not hand-tuned: one page under the derived + // count must be insufficient, or the ceiling could rise without the + // page count following it. + assert!( + capacity - i64::from(SEARCH_PAGE_SIZE) < advertised, + "scan budget has a spare page of slack — derive it from the ceiling" + ); + } + #[test] fn count_fallback_fetches_one_extra_candidate() { let mut query = From 2271584873766dbad1b06cf4233ae0dd2511e518 Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Thu, 30 Jul 2026 17:19:10 -0400 Subject: [PATCH 3/3] docs(relay): frame NIP-50 scan budget as a candidate floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scan-budget comments and test framing claimed the derivation lets search reach the advertised limit. It cannot: the budget bounds FTS candidates scanned, and post-filtering (NIP-01 match, channel access, reader visibility, dedup) discards an unpredictable share before emission. NIP-11 defines max_limit as a clamp on the request, not a promised response count, so a short search result was never a conformance violation. The derivation stays — it is a resource policy that tracks the ceiling — but its justification no longer overreaches. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-relay/src/handlers/req.rs | 47 ++++++++++++++++----------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/crates/buzz-relay/src/handlers/req.rs b/crates/buzz-relay/src/handlers/req.rs index 6b3704ab8c..048a9c6cf9 100644 --- a/crates/buzz-relay/src/handlers/req.rs +++ b/crates/buzz-relay/src/handlers/req.rs @@ -415,17 +415,23 @@ pub async fn handle_req( ); } -/// FTS candidate hits fetched per page. Pages are always full regardless of the -/// requested limit — post-filtering may discard many hits, so the scan needs -/// headroom to fill the limit. +/// FTS candidate hits fetched per page. Pages are always full regardless of +/// the requested limit — post-filtering discards an unpredictable share of +/// hits, so the scan fetches candidates in full pages rather than sizing +/// pages to the request. const SEARCH_PAGE_SIZE: u32 = 100; /// Maximum FTS pages to fetch per filter (prevents unbounded loops). /// -/// Derived from the advertised page ceiling rather than fixed, so the search -/// path can always reach the limit NIP-11 promises. A bare page count would let -/// a future ceiling change leave search silently emitting fewer events than the -/// relay advertises — the same class of lie this ceiling exists to prevent. +/// Derived from the advertised page ceiling rather than fixed: the scan +/// budget is a resource policy — at most one advertised page ceiling's worth +/// of candidates per filter — and deriving it keeps the budget tracking the +/// ceiling if the ceiling ever moves. This bounds candidates *scanned*, not +/// events *emitted*: post-filtering (NIP-01 match, channel access, reader +/// visibility, dedup) can discard any number of candidates, so a result +/// smaller than the requested limit remains possible and is not a NIP-11 +/// violation — `max_limit` promises a clamp on the request, not a count in +/// the response. const MAX_SEARCH_PAGES: u32 = (buzz_db::DEFAULT_MAX_PAGE_LIMIT as u32).div_ceil(SEARCH_PAGE_SIZE); /// Resolve request-local channel access, repairing a stale cache-negative. @@ -592,9 +598,10 @@ async fn handle_search_req( let since = filter.since.map(|s| s.as_secs() as i64); let until = filter.until.map(|u| u.as_secs() as i64); - // Paginate: keep fetching pages until we've emitted `limit` results - // or exhausted the search result set. This ensures post-filtering - // doesn't silently reduce the result count below the requested limit. + // Paginate: keep fetching pages until we've emitted `limit` results or + // exhausted the search result set. Post-filtering discards an unpredictable + // share of each page, so continuing past short yields gives the scan a + // chance — not a guarantee — of filling the requested limit. let mut emitted: u32 = 0; for page in 1..=MAX_SEARCH_PAGES { @@ -1470,12 +1477,14 @@ mod tests { } /// The NIP-50 search path clamps its emission target to the advertised - /// ceiling like every other REQ, but its ability to *reach* that target is - /// bounded a second time by how many FTS pages it will scan. If that scan - /// budget falls below the ceiling, search under-delivers against NIP-11 - /// while the clamp above still looks correct — so the budget is asserted - /// against the advertised value, not just against the DB constant it is - /// derived from. + /// ceiling like every other REQ, but the number of candidates it will scan + /// is bounded a second time by the page budget. This pins the resource + /// policy: the budget covers exactly one advertised page ceiling's worth of + /// candidates — no less (a ceiling raise must not silently shrink the scan + /// relative to what clients may request) and no hand-tuned spare (the budget + /// must stay derived, not drift back into a magic number). It deliberately + /// does NOT claim search fills the emitted limit — post-filtering can + /// discard any number of candidates. #[test] fn search_scan_capacity_covers_advertised_nip11_max_limit() { let advertised = advertised_max_limit(); @@ -1483,9 +1492,9 @@ mod tests { assert!( capacity >= advertised, - "NIP-50 scans at most {capacity} hits ({MAX_SEARCH_PAGES} pages of \ - {SEARCH_PAGE_SIZE}) but NIP-11 advertises {advertised} — search \ - would silently emit a short page" + "NIP-50 scans at most {capacity} candidates ({MAX_SEARCH_PAGES} pages of \ + {SEARCH_PAGE_SIZE}) but NIP-11 advertises {advertised} — the scan budget \ + no longer covers the advertised ceiling" ); // The budget is derived, not hand-tuned: one page under the derived